diff --git a/.circleci/config.yml b/.circleci/config.yml index 83acf0ac1c8..1485f517164 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3050,28 +3050,29 @@ jobs: - run: name: Run Docker container with bad DATABASE_URL command: | + set +e docker run --name my-app \ -p 4000:4000 \ -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \ -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ -e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \ myapp:latest \ - --port 4000 > docker_output.log 2>&1 || true + --port 4000 > docker_output.log 2>&1 + echo "$?" > docker_exit_code + set -e - run: name: Display Docker logs command: cat docker_output.log - run: - name: Check for expected error + name: Proxy must refuse to serve on an unreachable database command: | - if grep -q "Error: P1001: Can't reach database server at" docker_output.log && \ - (grep -q "Database setup failed after multiple retries" docker_output.log || \ - grep -q "ERROR: Application startup failed. Exiting." docker_output.log); then - echo "Expected error found. Test passed." - else - echo "Expected error not found. Test failed." - cat docker_output.log - exit 1 - fi + fail() { echo "FAILED: $1"; cat docker_output.log; exit 1; } + exit_code="$(cat docker_exit_code)" + [ "$exit_code" -ne 0 ] || fail "proxy exited 0 with an unreachable database" + grep -q "P1001" docker_output.log || fail "log does not name the unreachable database server" + ! grep -q "Application startup complete" docker_output.log || fail "proxy reached serving state" + ! docker exec my-app true 2>/dev/null || fail "container is still running" + echo "Proxy refused to serve (exit $exit_code) and never reached startup. Test passed." provider_replay_harness: docker: diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 06d369eabcd..592d8edf6b8 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -130,6 +130,10 @@ jobs: echo "File content around line 43:" head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10 + - name: Check MCP operation boundary + if: steps.changes.outputs.decision != 'skip' + run: uv run --no-sync python scripts/check_mcp_operation_boundary.py + - name: Run Ruff linting if: steps.changes.outputs.decision != 'skip' run: | diff --git a/Makefile b/Makefile index 0e9d2bbf82c..ab7fab6aa99 100644 --- a/Makefile +++ b/Makefile @@ -164,6 +164,7 @@ lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE) # Linting targets lint-ruff: $(LINT_DEP_INSTALL) + $(UV_RUN) python scripts/check_mcp_operation_boundary.py cd litellm && $(UV_RUN) ruff check . && cd .. $(UV_RUN) ruff check --config ruff-tests.toml tests diff --git a/docker/README.md b/docker/README.md index 26d8c9a37b0..376dc7b2d97 100644 --- a/docker/README.md +++ b/docker/README.md @@ -2,6 +2,17 @@ This guide provides instructions for building and running the LiteLLM application using Docker and Docker Compose. +> **Just want to run LiteLLM?** This guide builds from source. To run the published +> image instead, use `docker-compose.quickstart.yml` in this directory — the +> two-service stack (gateway + Postgres) that the +> [Docker quickstart](https://docs.litellm.ai/docs/proxy/docker_quick_start) documents: +> +> ```bash +> curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker/docker-compose.quickstart.yml +> printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env +> docker compose -f docker-compose.quickstart.yml up -d +> ``` + ## Prerequisites - Docker diff --git a/docker/docker-compose.quickstart.yml b/docker/docker-compose.quickstart.yml new file mode 100644 index 00000000000..11631603a72 --- /dev/null +++ b/docker/docker-compose.quickstart.yml @@ -0,0 +1,41 @@ +# LiteLLM quickstart stack: the gateway plus a Postgres database that stores +# models, virtual keys, and spend logs. Used by +# https://docs.litellm.ai/docs/proxy/docker_quick_start +# +# curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker/docker-compose.quickstart.yml +# printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env +# docker compose -f docker-compose.quickstart.yml up -d +# +# Compose reads .env from this directory. Keep it: regenerating LITELLM_SALT_KEY +# makes credentials already stored in the database unreadable. For anything +# beyond local evaluation, pin the image to a specific release tag. +services: + litellm: + image: docker.litellm.ai/berriai/litellm:main-stable + ports: + - "4000:4000" + environment: + LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:?set it in .env - see the header of this file} + LITELLM_SALT_KEY: ${LITELLM_SALT_KEY:?set it in .env - see the header of this file} + DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm + STORE_MODEL_IN_DB: "True" + depends_on: + db: + condition: service_healthy + + db: + image: postgres:16 + environment: + POSTGRES_USER: litellm + POSTGRES_PASSWORD: litellm + POSTGRES_DB: litellm + healthcheck: + test: ["CMD-SHELL", "pg_isready -U litellm"] + interval: 5s + timeout: 5s + retries: 10 + volumes: + - postgres_data:/var/lib/postgresql/data + +volumes: + postgres_data: diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260921000000_add_password_reset_columns/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260921000000_add_password_reset_columns/migration.sql new file mode 100644 index 00000000000..960b0d4d7eb --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260921000000_add_password_reset_columns/migration.sql @@ -0,0 +1,3 @@ +ALTER TABLE "LiteLLM_UserTable" ADD COLUMN IF NOT EXISTS "password_reset_required" BOOLEAN; + +ALTER TABLE "LiteLLM_UserTable" ADD COLUMN IF NOT EXISTS "last_breach_check_at" TIMESTAMP(3); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 2d7e557a9d1..368864f9cd1 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -246,6 +246,8 @@ model LiteLLM_UserTable { organization_id String? object_permission_id String? password String? + password_reset_required Boolean? + last_breach_check_at DateTime? teams String[] @default([]) user_role String? max_budget Float? diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index b37202dce1b..b13bce2ed67 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -1377,6 +1377,18 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fancy-regex" version = "0.17.0" @@ -1428,6 +1440,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1892,11 +1910,32 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash", +] + [[package]] name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" +dependencies = [ + "hashbrown 0.17.1", +] [[package]] name = "heck" @@ -2277,6 +2316,12 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +[[package]] +name = "iter-read" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071ed4cc1afd86650602c7b11aa2e1ce30762a1c27193201cb5cee9c6ebb1294" + [[package]] name = "itertools" version = "0.13.0" @@ -2435,6 +2480,17 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libsqlite3-sys" +version = "0.38.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -2540,6 +2596,36 @@ dependencies = [ "url", ] +[[package]] +name = "litellm-cache-disk" +version = "0.1.0" +dependencies = [ + "litellm-cache", + "py_literal", + "rand 0.8.7", + "rstest", + "rusqlite", + "serde-pickle", + "serde_json", + "tempfile", + "tokio", +] + +[[package]] +name = "litellm-cache-gcs" +version = "0.1.0" +dependencies = [ + "futures-util", + "litellm-auth-gcp", + "litellm-auth-types", + "litellm-cache", + "percent-encoding", + "reqwest 0.12.28", + "serde_json", + "tokio", + "wiremock", +] + [[package]] name = "litellm-cache-memory" version = "0.1.0" @@ -2743,6 +2829,8 @@ dependencies = [ "litellm-auth-gcp", "litellm-cache", "litellm-cache-azure-blob", + "litellm-cache-disk", + "litellm-cache-gcs", "litellm-cache-memory", "litellm-cache-redis", "litellm-cache-response", @@ -4033,6 +4121,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror 2.0.19", +] + [[package]] name = "rstest" version = "0.26.1" @@ -4073,6 +4171,21 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "rusqlite" +version = "0.40.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +dependencies = [ + "bitflags 2.13.1", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -4330,6 +4443,19 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-pickle" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b641fdc8bcf2781ee78b30c599700d64ad4f412976143e4c5d0b9df906bb4843" +dependencies = [ + "byteorder", + "iter-read", + "num-bigint 0.4.8", + "num-traits", + "serde", +] + [[package]] name = "serde_core" version = "1.0.229" @@ -4557,6 +4683,18 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "sse-stream" version = "0.2.6" @@ -5303,6 +5441,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "veil" version = "0.3.0" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 7d94d31db3e..3ab1e592c6a 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -32,6 +32,8 @@ litellm-cache = { path = "crates/cache" } litellm-cache-azure-blob = { path = "crates/cache-azure-blob" } litellm-cache-memory = { path = "crates/cache-memory" } litellm-cache-redis = { path = "crates/cache-redis" } +litellm-cache-gcs = { path = "crates/cache-gcs" } +litellm-cache-disk = { path = "crates/cache-disk" } litellm-cache-response = { path = "crates/cache-response" } litellm-token-counter = { path = "crates/token-counter" } litellm-token-counter-fast = { path = "crates/token-counter-fast" } @@ -69,6 +71,7 @@ base64 = "0.22" moka = { version = "0.12.16", features = ["future"] } strum = { version = "0.28.0", features = ["derive"] } url = "2.5.8" +percent-encoding = "2.3" webpki-roots = "1" time = { version = "0.3.53", features = ["parsing"] } criterion = "0.8.2" diff --git a/litellm-rust/crates/auth-gcp/src/lib.rs b/litellm-rust/crates/auth-gcp/src/lib.rs index 8aeddae9efc..534d85acdb0 100644 --- a/litellm-rust/crates/auth-gcp/src/lib.rs +++ b/litellm-rust/crates/auth-gcp/src/lib.rs @@ -128,6 +128,14 @@ impl VertexAuth { } } + pub async fn access_token( + &self, + config: &VertexConfig, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + ) -> Result { + self.load_provider(config, env_lookup).await?.token().await + } + pub async fn validate_environment( &self, headers: Vec<(String, String)>, diff --git a/litellm-rust/crates/cache-disk/Cargo.toml b/litellm-rust/crates/cache-disk/Cargo.toml new file mode 100644 index 00000000000..b96994b3b55 --- /dev/null +++ b/litellm-rust/crates/cache-disk/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "litellm-cache-disk" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-cache.workspace = true +py_literal = "0.4.0" +rand.workspace = true +rusqlite = { version = "0.40", features = ["bundled"] } +serde-pickle = "1.2" +serde_json.workspace = true +tokio.workspace = true + +[dev-dependencies] +rstest.workspace = true +tempfile = "3.27.0" diff --git a/litellm-rust/crates/cache-disk/src/adapter.rs b/litellm-rust/crates/cache-disk/src/adapter.rs new file mode 100644 index 00000000000..b6d318d5509 --- /dev/null +++ b/litellm-rust/crates/cache-disk/src/adapter.rs @@ -0,0 +1,10 @@ +use litellm_cache::Error; + +use crate::StoredValue; + +pub trait ValueAdapter: Send + Sync + 'static { + fn read(&self, value: StoredValue) -> Result>, Error>; + fn write(&self, payload: Vec) -> StoredValue; + fn counter_seed(&self, value: Option) -> Result; + fn counter_value(&self, value: f64) -> StoredValue; +} diff --git a/litellm-rust/crates/cache-disk/src/cache.rs b/litellm-rust/crates/cache-disk/src/cache.rs new file mode 100644 index 00000000000..8e1223309b4 --- /dev/null +++ b/litellm-rust/crates/cache-disk/src/cache.rs @@ -0,0 +1,301 @@ +use std::{ + path::Path, + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use litellm_cache::{ + BaseCache, BatchCache, BatchEntry, CacheCodec, CacheConnectionResult, CacheConnectionStatus, + CounterCache, DeleteCache, Error, ExactCacheContext, FlushCache, +}; + +use crate::{DiskStore, DiskcacheSqliteStore, PythonDiskCacheAdapter, StoredValue, ValueAdapter}; + +pub struct DiskCache { + store: Arc, + adapter: Arc, + codec: S, +} + +impl DiskCache { + pub fn open(directory: impl AsRef, codec: S) -> Result { + Ok(Self { + store: Arc::new(DiskcacheSqliteStore::open(directory)?), + adapter: Arc::new(PythonDiskCacheAdapter), + codec, + }) + } +} + +impl DiskCache { + pub fn with_store(store: D, codec: S) -> Self { + Self { + store: Arc::new(store), + adapter: Arc::new(PythonDiskCacheAdapter), + codec, + } + } +} + +impl DiskCache { + pub fn with_adapter(store: D, adapter: A, codec: S) -> Self { + Self { + store: Arc::new(store), + adapter: Arc::new(adapter), + codec, + } + } + + pub fn directory(&self) -> &Path { + self.store.directory() + } + + fn decode_stored(&self, value: StoredValue) -> Result, Error> { + let Some(bytes) = self.adapter.read(value)? else { + return Ok(None); + }; + self.codec.decode(&bytes).map(Some) + } + + async fn run_blocking(store: Arc, operation: F) -> Result + where + T: Send + 'static, + F: FnOnce(&D) -> Result + Send + 'static, + { + tokio::task::spawn_blocking(move || operation(&store)) + .await + .map_err(|_| Error::Unavailable)? + } +} + +impl BaseCache for DiskCache { + type Value = S::Value; + type Context = ExactCacheContext; + + fn get_ttl(&self, context: &Self::Context) -> Option { + context.ttl + } + + fn set_cache( + &self, + key: &str, + value: Self::Value, + context: &Self::Context, + ) -> Result<(), Error> { + let value = self.adapter.write(self.codec.encode(&value)?); + let expire_time = context.ttl.map(|ttl| unix_now() + ttl.as_secs_f64()); + self.store.set(key, value, expire_time, unix_now()) + } + + fn get_cache(&self, key: &str, _: &Self::Context) -> Result, Error> { + self.store + .get(key, unix_now())? + .map(|value| self.decode_stored(value)) + .transpose() + .map(|value| value.flatten()) + } + + async fn async_set_cache( + &self, + key: &str, + value: Self::Value, + context: ExactCacheContext, + ) -> Result<(), Error> { + let value = self.adapter.write(self.codec.encode(&value)?); + let ttl = context.ttl; + let key = key.to_string(); + Self::run_blocking(Arc::clone(&self.store), move |store| { + let expire_time = ttl.map(|ttl| unix_now() + ttl.as_secs_f64()); + store.set(&key, value, expire_time, unix_now()) + }) + .await + } + + async fn async_get_cache( + &self, + key: &str, + _: &ExactCacheContext, + ) -> Result, Error> { + let key = key.to_string(); + let value = Self::run_blocking(Arc::clone(&self.store), move |store| { + store.get(&key, unix_now()) + }) + .await?; + value + .map(|value| self.decode_stored(value)) + .transpose() + .map(|value| value.flatten()) + } + + async fn async_set_cache_pipeline( + &self, + entries: Vec<(String, Self::Value)>, + context: ExactCacheContext, + ) -> Result<(), Error> { + let entries = entries + .into_iter() + .map(|(key, value)| { + self.codec + .encode(&value) + .map(|value| (key, self.adapter.write(value))) + }) + .collect::, _>>()?; + let expire_after = context.ttl; + Self::run_blocking(Arc::clone(&self.store), move |store| { + for (key, value) in entries { + let expire_time = expire_after.map(|ttl| unix_now() + ttl.as_secs_f64()); + store.set(&key, value, expire_time, unix_now())?; + } + Ok(()) + }) + .await + } + + async fn disconnect(&self) -> Result<(), Error> { + Ok(()) + } + + async fn test_connection(&self) -> Result { + let result = Self::run_blocking(Arc::clone(&self.store), |store| { + store.probe().map(|_| CacheConnectionResult { + status: CacheConnectionStatus::Success, + message: "Disk cache connection test successful".into(), + error: None, + }) + }) + .await; + Ok(match result { + Ok(result) => result, + Err(error) => CacheConnectionResult { + status: CacheConnectionStatus::Failed, + message: format!("Disk cache connection failed: {error}"), + error: Some(error.to_string()), + }, + }) + } +} + +impl BatchCache for DiskCache { + fn batch_get_cache( + &self, + keys: &[String], + context: &ExactCacheContext, + ) -> Result>, Error> { + keys.iter() + .map(|key| match self.get_cache(key, context) { + Ok(Some(value)) => Ok(BatchEntry::Hit(value)), + Ok(None) => Ok(BatchEntry::Miss), + Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid), + Err(error) => Err(error), + }) + .collect() + } + + async fn async_batch_get_cache( + &self, + keys: Vec, + _: ExactCacheContext, + ) -> Result>, Error> { + let values = Self::run_blocking(Arc::clone(&self.store), move |store| { + keys.into_iter() + .map(|key| store.get(&key, unix_now()).map(|value| (key, value))) + .collect::, _>>() + }) + .await?; + values + .into_iter() + .map(|(_, value)| match value { + None => Ok(BatchEntry::Miss), + Some(value) => match self.decode_stored(value) { + Ok(Some(value)) => Ok(BatchEntry::Hit(value)), + Ok(None) => Ok(BatchEntry::Miss), + Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid), + Err(error) => Err(error), + }, + }) + .collect() + } +} + +impl DeleteCache for DiskCache { + fn delete_cache(&self, key: &str) -> Result<(), Error> { + self.store.pop(key, unix_now()).map(|_| ()) + } + + async fn async_delete_cache(&self, key: &str) -> Result<(), Error> { + let key = key.to_string(); + Self::run_blocking(Arc::clone(&self.store), move |store| { + store.pop(&key, unix_now()).map(|_| ()) + }) + .await + } +} + +impl FlushCache for DiskCache { + fn flush_cache(&self) -> Result<(), Error> { + self.store.clear() + } + + async fn async_flush_cache(&self) -> Result<(), Error> { + Self::run_blocking(Arc::clone(&self.store), |store| store.clear()).await + } +} + +impl, D: DiskStore, A: ValueAdapter> CounterCache + for DiskCache +{ + fn increment_cache( + &self, + key: &str, + amount: f64, + context: ExactCacheContext, + ) -> Result { + increment( + self.adapter.as_ref(), + self.store.as_ref(), + key, + amount, + context.ttl, + ) + } + + async fn async_increment( + &self, + key: &str, + amount: f64, + context: ExactCacheContext, + ) -> Result { + let key = key.to_string(); + let adapter = Arc::clone(&self.adapter); + Self::run_blocking(Arc::clone(&self.store), move |store| { + increment(adapter.as_ref(), store, &key, amount, context.ttl) + }) + .await + } +} + +fn increment( + adapter: &A, + store: &D, + key: &str, + amount: f64, + ttl: Option, +) -> Result { + let mut result = None; + let mut apply = |current: Option| { + let initial = adapter.counter_seed(current)?; + let value = initial + amount; + let stored = adapter.counter_value(value); + result = Some(value); + Ok((stored, ttl.map(|ttl| unix_now() + ttl.as_secs_f64()))) + }; + store.update(key, unix_now(), &mut apply)?; + result.ok_or(Error::InvalidEntry) +} + +fn unix_now() -> f64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64() +} diff --git a/litellm-rust/crates/cache-disk/src/lib.rs b/litellm-rust/crates/cache-disk/src/lib.rs new file mode 100644 index 00000000000..9b2ffc24915 --- /dev/null +++ b/litellm-rust/crates/cache-disk/src/lib.rs @@ -0,0 +1,11 @@ +mod adapter; +mod cache; +mod python; +mod sqlite; +mod store; + +pub use adapter::ValueAdapter; +pub use cache::DiskCache; +pub use python::PythonDiskCacheAdapter; +pub use sqlite::DiskcacheSqliteStore; +pub use store::{DiskStore, StoredValue}; diff --git a/litellm-rust/crates/cache-disk/src/python/mod.rs b/litellm-rust/crates/cache-disk/src/python/mod.rs new file mode 100644 index 00000000000..7a370db357c --- /dev/null +++ b/litellm-rust/crates/cache-disk/src/python/mod.rs @@ -0,0 +1,77 @@ +mod value; + +use litellm_cache::Error; +use py_literal::Value; + +use crate::{StoredValue, ValueAdapter}; + +#[derive(Clone, Copy, Debug, Default)] +pub struct PythonDiskCacheAdapter; + +impl PythonDiskCacheAdapter { + fn python_get_cache(value: StoredValue) -> Result, Error> { + let value = match value { + StoredValue::Bytes(value) => Value::Bytes(value), + StoredValue::Text(value) => Value::String(value), + StoredValue::Integer(value) => Value::Integer(value.into()), + StoredValue::Float(value) => Value::Float(value), + StoredValue::Pickle(value) => value::from_pickle(&value)?, + }; + if !value::is_truthy(&value) { + return Ok(None); + } + match value { + Value::String(text) => Ok(Some( + value::from_json_text(&text).unwrap_or(Value::String(text)), + )), + Value::Bytes(bytes) => match std::str::from_utf8(&bytes) { + Ok(text) => Ok(Some( + value::from_json_text(text).unwrap_or(Value::Bytes(bytes)), + )), + Err(_) => Ok(Some(Value::Bytes(bytes))), + }, + value => Ok(Some(value)), + } + } +} + +impl ValueAdapter for PythonDiskCacheAdapter { + fn read(&self, value: StoredValue) -> Result>, Error> { + match value { + StoredValue::Text(value) => Ok((!value.is_empty()).then(|| value.into_bytes())), + StoredValue::Bytes(value) => Ok((!value.is_empty()).then_some(value)), + value => { + let Some(value) = Self::python_get_cache(value)? else { + return Ok(None); + }; + value::to_json(&value).map(Some) + } + } + } + + fn write(&self, payload: Vec) -> StoredValue { + StoredValue::Bytes(payload) + } + + fn counter_seed(&self, value: Option) -> Result { + let Some(value) = value else { + return Ok(0.0); + }; + let Some(value) = Self::python_get_cache(value)? else { + return Ok(0.0); + }; + Ok(if value::is_int(&value) { + value::to_f64(&value).unwrap_or(0.0) + } else { + 0.0 + }) + } + + fn counter_value(&self, value: f64) -> StoredValue { + if value.fract() == 0.0 && value >= i64::MIN as f64 && value <= i64::MAX as f64 { + StoredValue::Integer(value as i64) + } else { + StoredValue::Float(value) + } + } +} diff --git a/litellm-rust/crates/cache-disk/src/python/value.rs b/litellm-rust/crates/cache-disk/src/python/value.rs new file mode 100644 index 00000000000..645eba7b757 --- /dev/null +++ b/litellm-rust/crates/cache-disk/src/python/value.rs @@ -0,0 +1,173 @@ +use litellm_cache::Error; +use py_literal::Value; +use serde_json::{Map, Number}; + +pub(crate) fn from_pickle(bytes: &[u8]) -> Result { + let value = serde_pickle::value_from_slice(bytes, Default::default()) + .map_err(|_| Error::InvalidEntry)?; + from_pickle_value(value) +} + +fn from_pickle_value(value: serde_pickle::Value) -> Result { + match value { + serde_pickle::Value::None => Ok(Value::None), + serde_pickle::Value::Bool(value) => Ok(Value::Boolean(value)), + serde_pickle::Value::I64(value) => integer(value.to_string()), + serde_pickle::Value::Int(value) => integer(value.to_string()), + serde_pickle::Value::F64(value) => Ok(Value::Float(value)), + serde_pickle::Value::String(value) => Ok(Value::String(value)), + serde_pickle::Value::Bytes(value) => Ok(Value::Bytes(value)), + serde_pickle::Value::List(values) => values + .into_iter() + .map(from_pickle_value) + .collect::, _>>() + .map(Value::List), + serde_pickle::Value::Tuple(values) => values + .into_iter() + .map(from_pickle_value) + .collect::, _>>() + .map(Value::Tuple), + serde_pickle::Value::Set(values) => values + .into_iter() + .map(from_pickle_hashable) + .collect::, _>>() + .map(Value::Set), + serde_pickle::Value::FrozenSet(values) => values + .into_iter() + .map(from_pickle_hashable) + .collect::, _>>() + .map(Value::Set), + serde_pickle::Value::Dict(values) => values + .into_iter() + .map(|(key, value)| Ok((from_pickle_hashable(key)?, from_pickle_value(value)?))) + .collect::, Error>>() + .map(Value::Dict), + } +} + +fn from_pickle_hashable(value: serde_pickle::HashableValue) -> Result { + Ok(match value { + serde_pickle::HashableValue::None => Value::None, + serde_pickle::HashableValue::Bool(value) => Value::Boolean(value), + serde_pickle::HashableValue::I64(value) => integer(value.to_string())?, + serde_pickle::HashableValue::Int(value) => integer(value.to_string())?, + serde_pickle::HashableValue::F64(value) => Value::Float(value), + serde_pickle::HashableValue::Bytes(value) => Value::Bytes(value), + serde_pickle::HashableValue::String(value) => Value::String(value), + serde_pickle::HashableValue::Tuple(values) => Value::Tuple( + values + .into_iter() + .map(from_pickle_hashable) + .collect::, _>>()?, + ), + serde_pickle::HashableValue::FrozenSet(values) => Value::Set( + values + .into_iter() + .map(from_pickle_hashable) + .collect::, _>>()?, + ), + }) +} + +fn integer(value: String) -> Result { + value.parse().map_err(|_| Error::InvalidEntry) +} + +pub(crate) fn from_json(value: serde_json::Value) -> Value { + match value { + serde_json::Value::Null => Value::None, + serde_json::Value::Bool(value) => Value::Boolean(value), + serde_json::Value::Number(value) => { + if value.is_i64() || value.is_u64() { + integer(value.to_string()) + .unwrap_or(Value::Float(value.as_f64().unwrap_or(f64::NAN))) + } else { + Value::Float(value.as_f64().unwrap_or(f64::NAN)) + } + } + serde_json::Value::String(value) => Value::String(value), + serde_json::Value::Array(values) => { + Value::List(values.into_iter().map(from_json).collect()) + } + serde_json::Value::Object(values) => Value::Dict( + values + .into_iter() + .map(|(key, value)| (Value::String(key), from_json(value))) + .collect(), + ), + } +} + +pub(crate) fn from_json_text(value: &str) -> Result { + serde_json::from_str(value) + .map(from_json) + .map_err(|_| Error::InvalidEntry) +} + +pub(crate) fn is_truthy(value: &Value) -> bool { + match value { + Value::None => false, + Value::Boolean(value) => *value, + Value::Integer(value) => value.to_string() != "0", + Value::Float(value) => *value != 0.0, + Value::Complex(value) => value.re != 0.0 || value.im != 0.0, + Value::String(value) => !value.is_empty(), + Value::Bytes(value) => !value.is_empty(), + Value::Tuple(value) | Value::List(value) | Value::Set(value) => !value.is_empty(), + Value::Dict(value) => !value.is_empty(), + } +} + +pub(crate) fn is_int(value: &Value) -> bool { + matches!(value, Value::Integer(_) | Value::Boolean(_)) +} + +pub(crate) fn to_f64(value: &Value) -> Option { + match value { + Value::Integer(value) => value.to_string().parse().ok(), + Value::Boolean(value) => Some(if *value { 1.0 } else { 0.0 }), + _ => None, + } +} + +pub(crate) fn to_json(value: &Value) -> Result, Error> { + serde_json::to_vec(&to_json_value(value)?).map_err(|_| Error::InvalidEntry) +} + +fn to_json_value(value: &Value) -> Result { + Ok(match value { + Value::None => serde_json::Value::Null, + Value::Boolean(value) => serde_json::Value::Bool(*value), + Value::Integer(value) => serde_json::Value::Number( + value + .to_string() + .parse::() + .map_err(|_| Error::InvalidEntry)?, + ), + Value::Float(value) => { + serde_json::Value::Number(Number::from_f64(*value).ok_or(Error::InvalidEntry)?) + } + Value::Complex(_) | Value::Bytes(_) => return Err(Error::InvalidEntry), + Value::String(value) => serde_json::Value::String(value.clone()), + Value::Tuple(values) | Value::List(values) | Value::Set(values) => { + serde_json::Value::Array( + values + .iter() + .map(to_json_value) + .collect::, _>>()?, + ) + } + Value::Dict(values) => { + let values = values + .iter() + .map(|(key, value)| { + let Value::String(key) = key else { + return Err(Error::InvalidEntry); + }; + Ok((key.clone(), to_json_value(value)?)) + }) + .collect::, _>>()?; + serde_json::Value::Object(values) + } + }) +} diff --git a/litellm-rust/crates/cache-disk/src/sqlite.rs b/litellm-rust/crates/cache-disk/src/sqlite.rs new file mode 100644 index 00000000000..9a36f8af6ad --- /dev/null +++ b/litellm-rust/crates/cache-disk/src/sqlite.rs @@ -0,0 +1,817 @@ +use std::{ + collections::HashMap, + fs::{self, OpenOptions}, + io::Write, + path::{Path, PathBuf}, + sync::Mutex, +}; + +use litellm_cache::Error; +use rand::RngCore; +use rusqlite::{Connection, OptionalExtension, params, types::Value}; + +use crate::{DiskStore, StoredValue}; + +const MODE_RAW: i64 = 1; +const MODE_BINARY: i64 = 2; +const MODE_TEXT: i64 = 3; +const MODE_PICKLE: i64 = 4; + +const DEFAULT_DISK_MIN_FILE_SIZE: i64 = 2_i64.pow(15); +const DEFAULT_SIZE_LIMIT: i64 = 2_i64.pow(30); +const DEFAULT_CULL_LIMIT: i64 = 10; + +pub struct DiskcacheSqliteStore { + directory: PathBuf, + connection: Mutex, + min_file_size: usize, + eviction_policy: String, + size_limit: i64, + cull_limit: i64, + statistics: bool, +} + +struct StoredColumns { + size: i64, + mode: i64, + filename: Option, + value: Option, +} + +struct Row { + rowid: i64, + mode: i64, + filename: Option, + value: Value, +} + +impl DiskcacheSqliteStore { + pub fn open(directory: impl AsRef) -> Result { + let directory = directory.as_ref().to_path_buf(); + fs::create_dir_all(&directory).map_err(|_| Error::Unavailable)?; + let directory = std::path::absolute(&directory).map_err(|_| Error::Unavailable)?; + let database = directory.join("cache.db"); + let connection = Connection::open(database).map_err(|_| Error::Unavailable)?; + connection + .busy_timeout(std::time::Duration::from_secs(60)) + .map_err(|_| Error::Unavailable)?; + + let mut settings = read_settings(&connection)?; + for (key, value) in default_settings() { + settings.entry(key).or_insert(value); + } + for (key, value) in settings + .iter() + .filter(|(key, _)| key.starts_with("sqlite_")) + { + apply_pragma(&connection, key, value)?; + } + + connection + .execute_batch( + "CREATE TABLE IF NOT EXISTS Settings ( + key TEXT NOT NULL UNIQUE, + value + )", + ) + .map_err(|_| Error::Unavailable)?; + for (key, value) in &settings { + if !matches!(key.as_str(), "count" | "size" | "hits" | "misses") { + connection + .execute( + "INSERT OR REPLACE INTO Settings VALUES (?, ?)", + params![key, value], + ) + .map_err(|_| Error::Unavailable)?; + } + } + for (key, value) in [ + ("count", Value::Integer(0)), + ("size", Value::Integer(0)), + ("hits", Value::Integer(0)), + ("misses", Value::Integer(0)), + ] { + connection + .execute( + "INSERT OR IGNORE INTO Settings VALUES (?, ?)", + params![key, value], + ) + .map_err(|_| Error::Unavailable)?; + } + connection + .execute_batch( + "CREATE TABLE IF NOT EXISTS Cache ( + rowid INTEGER PRIMARY KEY, + key BLOB, + raw INTEGER, + store_time REAL, + expire_time REAL, + access_time REAL, + access_count INTEGER DEFAULT 0, + tag BLOB, + size INTEGER DEFAULT 0, + mode INTEGER DEFAULT 0, + filename TEXT, + value BLOB + ); + CREATE UNIQUE INDEX IF NOT EXISTS Cache_key_raw ON Cache(key, raw); + CREATE INDEX IF NOT EXISTS Cache_expire_time ON Cache(expire_time);", + ) + .map_err(|_| Error::Unavailable)?; + + let eviction_policy = setting_string(&settings, "eviction_policy") + .unwrap_or_else(|| "least-recently-stored".to_string()); + match eviction_policy.as_str() { + "none" => {} + "least-recently-stored" => { + connection + .execute_batch( + "CREATE INDEX IF NOT EXISTS Cache_store_time ON Cache(store_time)", + ) + .map_err(|_| Error::Unavailable)?; + } + "least-recently-used" => { + connection + .execute_batch( + "CREATE INDEX IF NOT EXISTS Cache_access_time ON Cache(access_time)", + ) + .map_err(|_| Error::Unavailable)?; + } + "least-frequently-used" => { + connection + .execute_batch( + "CREATE INDEX IF NOT EXISTS Cache_access_count ON Cache(access_count)", + ) + .map_err(|_| Error::Unavailable)?; + } + _ => return Err(Error::Unavailable), + } + connection + .execute_batch( + "CREATE TRIGGER IF NOT EXISTS Settings_count_insert + AFTER INSERT ON Cache FOR EACH ROW BEGIN + UPDATE Settings SET value = value + 1 + WHERE key = \"count\"; END; + CREATE TRIGGER IF NOT EXISTS Settings_count_delete + AFTER DELETE ON Cache FOR EACH ROW BEGIN + UPDATE Settings SET value = value - 1 + WHERE key = \"count\"; END; + CREATE TRIGGER IF NOT EXISTS Settings_size_insert + AFTER INSERT ON Cache FOR EACH ROW BEGIN + UPDATE Settings SET value = value + NEW.size + WHERE key = \"size\"; END; + CREATE TRIGGER IF NOT EXISTS Settings_size_update + AFTER UPDATE ON Cache FOR EACH ROW BEGIN + UPDATE Settings + SET value = value + NEW.size - OLD.size + WHERE key = \"size\"; END; + CREATE TRIGGER IF NOT EXISTS Settings_size_delete + AFTER DELETE ON Cache FOR EACH ROW BEGIN + UPDATE Settings SET value = value - OLD.size + WHERE key = \"size\"; END;", + ) + .map_err(|_| Error::Unavailable)?; + + let min_file_size = setting_i64(&settings, "disk_min_file_size") + .unwrap_or(DEFAULT_DISK_MIN_FILE_SIZE) + .try_into() + .map_err(|_| Error::Unavailable)?; + let size_limit = setting_i64(&settings, "size_limit").unwrap_or(DEFAULT_SIZE_LIMIT); + let cull_limit = setting_i64(&settings, "cull_limit").unwrap_or(DEFAULT_CULL_LIMIT); + let statistics = setting_i64(&settings, "statistics").unwrap_or_default() != 0; + + Ok(Self { + directory, + connection: Mutex::new(connection), + min_file_size, + eviction_policy, + size_limit, + cull_limit, + statistics, + }) + } + + fn set_locked( + &self, + connection: &Connection, + key: &str, + columns: StoredColumns, + expire_time: Option, + now: f64, + ) -> Result, Error> { + let mut cleanup = Vec::new(); + if let Some(old_filename) = connection + .query_row( + "SELECT filename FROM Cache WHERE key = ? AND raw = 1", + params![key], + |row| row.get::<_, Option>(0), + ) + .optional() + .map_err(|_| Error::Unavailable)? + .flatten() + { + cleanup.push(old_filename); + } + let (size, mode, filename, value) = + (columns.size, columns.mode, columns.filename, columns.value); + let rowid = connection + .query_row( + "SELECT rowid FROM Cache WHERE key = ? AND raw = 1", + params![key], + |row| row.get::<_, i64>(0), + ) + .optional() + .map_err(|_| Error::Unavailable)?; + if let Some(rowid) = rowid { + connection + .execute( + "UPDATE Cache SET store_time = ?, expire_time = ?, access_time = ?, + access_count = 0, tag = NULL, size = ?, mode = ?, filename = ?, value = ? + WHERE rowid = ?", + params![now, expire_time, now, size, mode, filename, value, rowid], + ) + .map_err(|_| Error::Unavailable)?; + } else { + connection + .execute( + "INSERT INTO Cache( + key, raw, store_time, expire_time, access_time, access_count, + tag, size, mode, filename, value + ) VALUES (?, 1, ?, ?, ?, 0, NULL, ?, ?, ?, ?)", + params![key, now, expire_time, now, size, mode, filename, value], + ) + .map_err(|_| Error::Unavailable)?; + } + cleanup.extend(self.cull(connection, now)?); + Ok(cleanup) + } + + fn cull(&self, connection: &Connection, now: f64) -> Result, Error> { + if self.cull_limit <= 0 { + return Ok(Vec::new()); + } + let mut cleanup = Vec::new(); + let expired = connection + .prepare( + "SELECT rowid, filename FROM Cache + WHERE expire_time IS NOT NULL AND expire_time < ? + ORDER BY expire_time LIMIT ?", + ) + .map_err(|_| Error::Unavailable)? + .query_map(params![now, self.cull_limit], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, Option>(1)?)) + }) + .map_err(|_| Error::Unavailable)? + .collect::, _>>() + .map_err(|_| Error::Unavailable)?; + for (_, filename) in &expired { + if let Some(filename) = filename { + cleanup.push(filename.clone()); + } + } + for (rowid, _) in &expired { + connection + .execute("DELETE FROM Cache WHERE rowid = ?", params![rowid]) + .map_err(|_| Error::Unavailable)?; + } + let remaining = self.cull_limit - i64::try_from(expired.len()).unwrap_or(self.cull_limit); + if remaining <= 0 || self.volume(connection)? < self.size_limit { + return Ok(cleanup); + } + let order = match self.eviction_policy.as_str() { + "none" => return Ok(cleanup), + "least-recently-stored" => "store_time", + "least-recently-used" => "access_time", + "least-frequently-used" => "access_count", + _ => return Err(Error::Unavailable), + }; + let rows = connection + .prepare(&format!( + "SELECT rowid, filename FROM Cache ORDER BY {order} LIMIT ?" + )) + .map_err(|_| Error::Unavailable)? + .query_map(params![remaining], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, Option>(1)?)) + }) + .map_err(|_| Error::Unavailable)? + .collect::, _>>() + .map_err(|_| Error::Unavailable)?; + for (_, filename) in &rows { + if let Some(filename) = filename { + cleanup.push(filename.clone()); + } + } + for (rowid, _) in rows { + connection + .execute("DELETE FROM Cache WHERE rowid = ?", params![rowid]) + .map_err(|_| Error::Unavailable)?; + } + Ok(cleanup) + } + + fn volume(&self, connection: &Connection) -> Result { + let page_count: i64 = connection + .query_row("PRAGMA page_count", [], |row| row.get(0)) + .map_err(|_| Error::Unavailable)?; + let page_size: i64 = connection + .query_row("PRAGMA page_size", [], |row| row.get(0)) + .map_err(|_| Error::Unavailable)?; + let size: i64 = connection + .query_row("SELECT value FROM Settings WHERE key = 'size'", [], |row| { + row.get(0) + }) + .map_err(|_| Error::Unavailable)?; + Ok(page_count.saturating_mul(page_size).saturating_add(size)) + } +} + +impl DiskStore for DiskcacheSqliteStore { + fn directory(&self) -> &Path { + &self.directory + } + + fn get(&self, key: &str, now: f64) -> Result, Error> { + let connection = self.connection.lock().map_err(|_| Error::Unavailable)?; + let select = "SELECT rowid, expire_time, mode, filename, value FROM Cache + WHERE key = ? AND raw = 1 AND (expire_time IS NULL OR expire_time > ?)"; + let row = connection + .query_row(select, params![key, now], row_from_query) + .optional() + .map_err(|_| Error::Unavailable)?; + if !self.statistics && !has_get_update(&self.eviction_policy) { + return row + .map(|row| fetch_row(&self.directory, row)) + .transpose() + .map(|value| value.flatten()); + } + transactional(&connection, |connection| { + let row = connection + .query_row(select, params![key, now], row_from_query) + .optional() + .map_err(|_| Error::Unavailable)?; + let Some(row) = row else { + if self.statistics { + connection + .execute( + "UPDATE Settings SET value = value + 1 WHERE key = 'misses'", + [], + ) + .map_err(|_| Error::Unavailable)?; + } + return Ok(None); + }; + let rowid = row.rowid; + let value = fetch_row(&self.directory, row); + let hit = value.as_ref().is_ok_and(Option::is_some); + if hit && self.statistics { + connection + .execute( + "UPDATE Settings SET value = value + 1 WHERE key = 'hits'", + [], + ) + .map_err(|_| Error::Unavailable)?; + } else if !hit && self.statistics { + connection + .execute( + "UPDATE Settings SET value = value + 1 WHERE key = 'misses'", + [], + ) + .map_err(|_| Error::Unavailable)?; + } + if has_get_update(&self.eviction_policy) && hit { + let update = match self.eviction_policy.as_str() { + "least-recently-used" => "UPDATE Cache SET access_time = ? WHERE rowid = ?", + "least-frequently-used" => { + "UPDATE Cache SET access_count = access_count + 1 WHERE rowid = ?" + } + _ => return Err(Error::Unavailable), + }; + if self.eviction_policy == "least-recently-used" { + connection + .execute(update, params![now, rowid]) + .map_err(|_| Error::Unavailable)?; + } else { + connection + .execute(update, params![rowid]) + .map_err(|_| Error::Unavailable)?; + } + } + value + }) + } + + fn set( + &self, + key: &str, + value: StoredValue, + expire_time: Option, + now: f64, + ) -> Result<(), Error> { + let columns = store_value(&self.directory, self.min_file_size, value)?; + let new_filename = columns.filename.clone(); + let connection = self.connection.lock().map_err(|_| Error::Unavailable)?; + let result = transactional(&connection, |connection| { + self.set_locked(connection, key, columns, expire_time, now) + }); + match result { + Ok(cleanup) => { + cleanup_files(&self.directory, cleanup); + Ok(()) + } + Err(error) => { + if let Some(filename) = new_filename { + remove_file(&self.directory, &filename); + } + Err(error) + } + } + } + + fn pop(&self, key: &str, now: f64) -> Result, Error> { + let connection = self.connection.lock().map_err(|_| Error::Unavailable)?; + let selected = transactional(&connection, |connection| { + let row = connection + .query_row( + "SELECT rowid, expire_time, mode, filename, value FROM Cache + WHERE key = ? AND raw = 1 + AND (expire_time IS NULL OR expire_time > ?)", + params![key, now], + row_from_query, + ) + .optional() + .map_err(|_| Error::Unavailable)?; + let Some(row) = row else { + return Ok(None); + }; + connection + .execute("DELETE FROM Cache WHERE rowid = ?", params![row.rowid]) + .map_err(|_| Error::Unavailable)?; + Ok(Some(row)) + })?; + let Some(row) = selected else { + return Ok(None); + }; + let filename = row.filename.clone(); + let result = fetch_row(&self.directory, row)?; + if let Some(filename) = filename { + remove_file(&self.directory, &filename); + } + Ok(result) + } + + fn clear(&self) -> Result<(), Error> { + let connection = self.connection.lock().map_err(|_| Error::Unavailable)?; + let mut last_rowid = 0_i64; + loop { + let batch = transactional(&connection, |connection| { + let rows = connection + .prepare( + "SELECT rowid, filename FROM Cache + WHERE rowid > ? ORDER BY rowid LIMIT 100", + ) + .map_err(|_| Error::Unavailable)? + .query_map(params![last_rowid], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, Option>(1)?)) + }) + .map_err(|_| Error::Unavailable)? + .collect::, _>>() + .map_err(|_| Error::Unavailable)?; + if rows.is_empty() { + return Ok(rows); + } + let ids = rows + .iter() + .map(|(rowid, _)| rowid.to_string()) + .collect::>() + .join(","); + connection + .execute(&format!("DELETE FROM Cache WHERE rowid IN ({ids})"), []) + .map_err(|_| Error::Unavailable)?; + Ok(rows) + })?; + if batch.is_empty() { + return Ok(()); + } + last_rowid = batch.last().map(|(rowid, _)| *rowid).unwrap_or(last_rowid); + cleanup_files( + &self.directory, + batch + .into_iter() + .filter_map(|(_, filename)| filename) + .collect(), + ); + } + } + + fn update( + &self, + key: &str, + now: f64, + apply: &mut dyn FnMut(Option) -> Result<(StoredValue, Option), Error>, + ) -> Result<(), Error> { + let connection = self.connection.lock().map_err(|_| Error::Unavailable)?; + let mut created_filename = None; + let result = transactional(&connection, |connection| { + let current = connection + .query_row( + "SELECT rowid, expire_time, mode, filename, value FROM Cache + WHERE key = ? AND raw = 1 + AND (expire_time IS NULL OR expire_time > ?)", + params![key, now], + row_from_query, + ) + .optional() + .map_err(|_| Error::Unavailable)? + .map(|row| fetch_row(&self.directory, row)) + .transpose()? + .flatten(); + let (value, expire_time) = apply(current)?; + let columns = store_value(&self.directory, self.min_file_size, value)?; + created_filename = columns.filename.clone(); + let cleanup = self.set_locked(connection, key, columns, expire_time, now)?; + Ok(cleanup) + }); + match result { + Ok(cleanup) => { + cleanup_files(&self.directory, cleanup); + Ok(()) + } + Err(error) => { + if let Some(filename) = created_filename { + remove_file(&self.directory, &filename); + } + Err(error) + } + } + } + + fn probe(&self) -> Result<(), Error> { + let connection = self.connection.lock().map_err(|_| Error::Unavailable)?; + connection + .query_row( + "SELECT value FROM Settings WHERE key = 'count'", + [], + |row| row.get::<_, i64>(0), + ) + .map(|_| ()) + .map_err(|_| Error::Unavailable) + } +} + +fn default_settings() -> HashMap { + HashMap::from([ + ("statistics".to_string(), Value::Integer(0)), + ("tag_index".to_string(), Value::Integer(0)), + ( + "eviction_policy".to_string(), + Value::Text("least-recently-stored".to_string()), + ), + ("size_limit".to_string(), Value::Integer(DEFAULT_SIZE_LIMIT)), + ("cull_limit".to_string(), Value::Integer(DEFAULT_CULL_LIMIT)), + ("sqlite_auto_vacuum".to_string(), Value::Integer(1)), + ("sqlite_cache_size".to_string(), Value::Integer(8192)), + ( + "sqlite_journal_mode".to_string(), + Value::Text("wal".to_string()), + ), + ( + "sqlite_mmap_size".to_string(), + Value::Integer(2_i64.pow(26)), + ), + ("sqlite_synchronous".to_string(), Value::Integer(1)), + ( + "disk_min_file_size".to_string(), + Value::Integer(DEFAULT_DISK_MIN_FILE_SIZE), + ), + ("disk_pickle_protocol".to_string(), Value::Integer(5)), + ]) +} + +fn read_settings(connection: &Connection) -> Result, Error> { + let mut statement = match connection.prepare("SELECT key, value FROM Settings") { + Ok(statement) => statement, + Err(_) => return Ok(HashMap::new()), + }; + statement + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .map_err(|_| Error::Unavailable)? + .collect::, _>>() + .map_err(|_| Error::Unavailable) +} + +fn apply_pragma(connection: &Connection, key: &str, value: &Value) -> Result<(), Error> { + let pragma = key.strip_prefix("sqlite_").ok_or(Error::Unavailable)?; + match value { + Value::Integer(value) => connection + .pragma_update(None, pragma, value) + .map_err(|_| Error::Unavailable), + Value::Text(value) => connection + .pragma_update(None, pragma, value) + .map_err(|_| Error::Unavailable), + _ => Err(Error::Unavailable), + } +} + +fn setting_i64(settings: &HashMap, key: &str) -> Option { + match settings.get(key) { + Some(Value::Integer(value)) => Some(*value), + _ => None, + } +} + +fn setting_string(settings: &HashMap, key: &str) -> Option { + match settings.get(key) { + Some(Value::Text(value)) => Some(value.clone()), + _ => None, + } +} + +fn has_get_update(policy: &str) -> bool { + matches!(policy, "least-recently-used" | "least-frequently-used") +} + +fn row_from_query(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(Row { + rowid: row.get(0)?, + mode: row.get(2)?, + filename: row.get(3)?, + value: row.get(4)?, + }) +} + +fn fetch_row(directory: &Path, row: Row) -> Result, Error> { + match row.mode { + MODE_RAW => match row.value { + Value::Blob(value) => Ok(Some(StoredValue::Bytes(value))), + Value::Text(value) => Ok(Some(StoredValue::Text(value))), + Value::Integer(value) => Ok(Some(StoredValue::Integer(value))), + Value::Real(value) => Ok(Some(StoredValue::Float(value))), + Value::Null => Err(Error::InvalidEntry), + }, + MODE_BINARY | MODE_PICKLE => { + let bytes = match row.value { + Value::Blob(value) => value, + Value::Null => { + let Some(value) = read_file(directory, row.filename.as_deref())? else { + return Ok(None); + }; + value + } + _ => return Err(Error::InvalidEntry), + }; + Ok(Some(if row.mode == MODE_BINARY { + StoredValue::Bytes(bytes) + } else { + StoredValue::Pickle(bytes) + })) + } + MODE_TEXT => { + let bytes = match row.value { + Value::Null => { + let Some(value) = read_file(directory, row.filename.as_deref())? else { + return Ok(None); + }; + value + } + Value::Blob(value) => value, + Value::Text(value) => value.into_bytes(), + _ => return Err(Error::InvalidEntry), + }; + Ok(Some(StoredValue::Text( + String::from_utf8(bytes).map_err(|_| Error::InvalidEntry)?, + ))) + } + _ => Err(Error::InvalidEntry), + } +} + +fn read_file(directory: &Path, filename: Option<&str>) -> Result>, Error> { + let Some(filename) = filename else { + return Err(Error::InvalidEntry); + }; + match fs::read(directory.join(filename)) { + Ok(value) => Ok(Some(value)), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(_) => Err(Error::Unavailable), + } +} + +fn store_value( + directory: &Path, + min_file_size: usize, + value: StoredValue, +) -> Result { + match value { + StoredValue::Integer(value) => Ok(StoredColumns { + size: 0, + mode: MODE_RAW, + filename: None, + value: Some(Value::Integer(value)), + }), + StoredValue::Float(value) => Ok(StoredColumns { + size: 0, + mode: MODE_RAW, + filename: None, + value: Some(Value::Real(value)), + }), + StoredValue::Text(value) if value.chars().count() < min_file_size => Ok(StoredColumns { + size: 0, + mode: MODE_RAW, + filename: None, + value: Some(Value::Text(value)), + }), + StoredValue::Text(value) => { + let bytes = value.into_bytes(); + let filename = write_file(directory, &bytes)?; + Ok(StoredColumns { + size: i64::try_from(bytes.len()).map_err(|_| Error::Unavailable)?, + mode: MODE_TEXT, + filename: Some(filename), + value: None, + }) + } + StoredValue::Bytes(value) if value.len() < min_file_size => Ok(StoredColumns { + size: 0, + mode: MODE_RAW, + filename: None, + value: Some(Value::Blob(value)), + }), + StoredValue::Bytes(value) => { + let filename = write_file(directory, &value)?; + Ok(StoredColumns { + size: i64::try_from(value.len()).map_err(|_| Error::Unavailable)?, + mode: MODE_BINARY, + filename: Some(filename), + value: None, + }) + } + StoredValue::Pickle(value) if value.len() < min_file_size => Ok(StoredColumns { + size: 0, + mode: MODE_PICKLE, + filename: None, + value: Some(Value::Blob(value)), + }), + StoredValue::Pickle(value) => { + let filename = write_file(directory, &value)?; + Ok(StoredColumns { + size: i64::try_from(value.len()).map_err(|_| Error::Unavailable)?, + mode: MODE_PICKLE, + filename: Some(filename), + value: None, + }) + } + } +} + +fn write_file(directory: &Path, bytes: &[u8]) -> Result { + let mut random = [0_u8; 16]; + rand::rngs::OsRng.fill_bytes(&mut random); + let hex = random + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + let filename = format!("{}/{}/{}.val", &hex[..2], &hex[2..4], &hex[4..]); + let path = directory.join(&filename); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).map_err(|_| Error::Unavailable)?; + } + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map_err(|_| Error::Unavailable)?; + file.write_all(bytes).map_err(|_| Error::Unavailable)?; + Ok(filename) +} + +fn cleanup_files(directory: &Path, filenames: Vec) { + for filename in filenames { + remove_file(directory, &filename); + } +} + +fn remove_file(directory: &Path, filename: &str) { + let path = directory.join(filename); + let _ = fs::remove_file(&path); +} + +fn transactional( + connection: &Connection, + operation: impl FnOnce(&Connection) -> Result, +) -> Result { + connection + .execute_batch("BEGIN IMMEDIATE") + .map_err(|_| Error::Unavailable)?; + match operation(connection) { + Ok(value) => { + connection + .execute_batch("COMMIT") + .map_err(|_| Error::Unavailable)?; + Ok(value) + } + Err(error) => { + let _ = connection.execute_batch("ROLLBACK"); + Err(error) + } + } +} diff --git a/litellm-rust/crates/cache-disk/src/store.rs b/litellm-rust/crates/cache-disk/src/store.rs new file mode 100644 index 00000000000..ed167317cf0 --- /dev/null +++ b/litellm-rust/crates/cache-disk/src/store.rs @@ -0,0 +1,33 @@ +use std::path::Path; + +use litellm_cache::Error; + +#[derive(Clone, Debug, PartialEq)] +pub enum StoredValue { + Bytes(Vec), + Text(String), + Integer(i64), + Float(f64), + Pickle(Vec), +} + +pub trait DiskStore: Send + Sync + 'static { + fn directory(&self) -> &Path; + fn get(&self, key: &str, now: f64) -> Result, Error>; + fn set( + &self, + key: &str, + value: StoredValue, + expire_time: Option, + now: f64, + ) -> Result<(), Error>; + fn pop(&self, key: &str, now: f64) -> Result, Error>; + fn clear(&self) -> Result<(), Error>; + fn update( + &self, + key: &str, + now: f64, + apply: &mut dyn FnMut(Option) -> Result<(StoredValue, Option), Error>, + ) -> Result<(), Error>; + fn probe(&self) -> Result<(), Error>; +} diff --git a/litellm-rust/crates/cache-disk/tests/cache.rs b/litellm-rust/crates/cache-disk/tests/cache.rs new file mode 100644 index 00000000000..dd1f2b1f04e --- /dev/null +++ b/litellm-rust/crates/cache-disk/tests/cache.rs @@ -0,0 +1,431 @@ +use std::{ + fs, + path::{Path, PathBuf}, + sync::Arc, + thread, + time::Duration, +}; + +use litellm_cache::{ + BaseCache, BatchCache, BatchEntry, CacheCodec, CounterCache, DeleteCache, ExactCacheContext, + FlushCache, JsonCodec, +}; +use litellm_cache_disk::{DiskCache, DiskStore, DiskcacheSqliteStore, StoredValue, ValueAdapter}; +use rstest::{fixture, rstest}; +use rusqlite::Connection; +use serde_json::{Value, json}; +use tempfile::TempDir; + +struct Sandbox { + directory: TempDir, +} + +#[fixture] +fn sandbox() -> Sandbox { + Sandbox { + directory: tempfile::tempdir().unwrap(), + } +} + +impl Sandbox { + fn store(&self) -> DiskcacheSqliteStore { + DiskcacheSqliteStore::open(self.directory.path()).unwrap() + } + + fn cache(&self) -> DiskCache> + where + JsonCodec: CacheCodec, + { + DiskCache::open(self.directory.path(), JsonCodec::new()).unwrap() + } + + fn db(&self) -> Connection { + Connection::open(self.directory.path().join("cache.db")).unwrap() + } + + fn value_files(&self) -> Vec { + fn visit(directory: &Path, files: &mut Vec) { + for entry in fs::read_dir(directory).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + visit(&path, files); + } else if path.extension().is_some_and(|extension| extension == "val") { + files.push(path); + } + } + } + + let mut files = Vec::new(); + visit(self.directory.path(), &mut files); + files + } +} + +#[rstest] +fn relative_store_directory_is_absolutized(sandbox: Sandbox) { + let relative = PathBuf::from(format!( + ".litellm-cache-disk-{}", + sandbox + .directory + .path() + .file_name() + .unwrap() + .to_string_lossy() + )); + let store = DiskcacheSqliteStore::open(&relative).unwrap(); + assert!(store.directory().is_absolute()); + assert!(store.directory().ends_with(&relative)); + let directory = store.directory().to_path_buf(); + drop(store); + fs::remove_dir_all(directory).unwrap(); +} + +#[derive(Clone, Copy, Debug, Default)] +struct TextAdapter; + +impl ValueAdapter for TextAdapter { + fn read(&self, value: StoredValue) -> Result>, litellm_cache::Error> { + match value { + StoredValue::Text(value) => Ok(Some(value.into_bytes())), + _ => Ok(None), + } + } + + fn write(&self, payload: Vec) -> StoredValue { + StoredValue::Text(String::from_utf8(payload).unwrap()) + } + + fn counter_seed(&self, _: Option) -> Result { + Ok(0.0) + } + + fn counter_value(&self, value: f64) -> StoredValue { + if value.fract() == 0.0 { + StoredValue::Integer(value as i64) + } else { + StoredValue::Float(value) + } + } +} + +#[rstest] +fn roundtrip_persists_and_reopens(sandbox: Sandbox) { + let context = ExactCacheContext::default(); + let opened = sandbox.cache::(); + opened + .set_cache("key", json!({"answer": 42}), &context) + .unwrap(); + assert_eq!( + opened.get_cache("key", &context).unwrap(), + Some(json!({"answer": 42})) + ); + drop(opened); + let reopened = sandbox.cache::(); + assert_eq!( + reopened.get_cache("key", &context).unwrap(), + Some(json!({"answer": 42})) + ); +} + +#[rstest] +fn ttl_and_expired_culling_match_cache_contract(sandbox: Sandbox) { + let store = sandbox.store(); + store + .set( + "expired", + StoredValue::Bytes(b"old".to_vec()), + Some(10.0), + 0.0, + ) + .unwrap(); + assert_eq!(store.get("expired", 10.0).unwrap(), None); + store + .set("new", StoredValue::Bytes(b"new".to_vec()), None, 11.0) + .unwrap(); + assert_eq!( + sandbox + .db() + .query_row("SELECT COUNT(*) FROM Cache", [], |row| row.get::<_, i64>(0)) + .unwrap(), + 1 + ); + assert_eq!( + sandbox + .db() + .query_row( + "SELECT value FROM Settings WHERE key = 'count'", + [], + |row| row.get::<_, i64>(0) + ) + .unwrap(), + 1 + ); +} + +#[rstest] +fn batch_preserves_order_and_classifies_misses_and_invalid_values(sandbox: Sandbox) { + let store = sandbox.store(); + store + .set( + "hit", + StoredValue::Bytes(br#"{"ok":true}"#.to_vec()), + None, + 0.0, + ) + .unwrap(); + store + .set( + "invalid", + StoredValue::Pickle(vec![0x80, 0x05, 0x2e]), + None, + 0.0, + ) + .unwrap(); + let entries = sandbox + .cache::() + .batch_get_cache( + &["hit".into(), "missing".into(), "invalid".into()], + &ExactCacheContext::default(), + ) + .unwrap(); + assert_eq!( + entries, + vec![ + BatchEntry::Hit(json!({"ok": true})), + BatchEntry::Miss, + BatchEntry::Invalid + ] + ); +} + +#[rstest] +#[case(StoredValue::Bytes(Vec::new()))] +#[case(StoredValue::Text(String::new()))] +#[case(StoredValue::Integer(0))] +#[case(StoredValue::Float(0.0))] +#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e]))] +#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x89, 0x2e]))] +#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x00, 0x2e]))] +#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e]))] +#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x94, 0x2e]))] +#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x5d, 0x94, 0x2e]))] +#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x7d, 0x94, 0x2e]))] +#[case(StoredValue::Pickle(vec![0x80, 0x05, 0x29, 0x2e]))] +fn falsy_values_are_misses(sandbox: Sandbox, #[case] value: StoredValue) { + sandbox.store().set("key", value, None, 0.0).unwrap(); + assert_eq!( + sandbox + .cache::() + .get_cache("key", &ExactCacheContext::default()) + .unwrap(), + None + ); +} + +#[rstest] +#[case(Some(StoredValue::Integer(2)), 1.5, 3.5, "real")] +#[case(Some(StoredValue::Integer(2)), 1.0, 3.0, "integer")] +#[case(Some(StoredValue::Float(3.5)), 1.0, 1.0, "integer")] +#[case(Some(StoredValue::Text("not a number".into())), 2.0, 2.0, "integer")] +#[case(Some(StoredValue::Text("5".into())), 2.0, 7.0, "integer")] +#[case(Some(StoredValue::Text("3.5".into())), 2.0, 2.0, "integer")] +#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x88, 0x2e])), 1.0, 2.0, "integer")] +#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x02, 0x2e])), 1.0, 3.0, "integer")] +#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x8c, 0x01, 0x61, 0x94, 0x4b, 0x01, 0x73, 0x2e])), 1.0, 1.0, "integer")] +#[case(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e])), 4.0, 4.0, "integer")] +fn counters_follow_python_initialization( + sandbox: Sandbox, + #[case] initial: Option, + #[case] amount: f64, + #[case] expected: f64, + #[case] sqlite_type: &str, +) { + if let Some(initial) = initial { + sandbox.store().set("counter", initial, None, 0.0).unwrap(); + } + let cache = sandbox.cache::(); + assert_eq!( + cache + .increment_cache("counter", amount, ExactCacheContext::default()) + .unwrap(), + expected + ); + assert_eq!( + sandbox + .db() + .query_row( + "SELECT typeof(value) FROM Cache WHERE key = 'counter'", + [], + |row| row.get::<_, String>(0) + ) + .unwrap(), + sqlite_type + ); +} + +#[rstest] +fn counters_are_atomic_across_concurrent_callers(sandbox: Sandbox) { + let cache = Arc::new(sandbox.cache::()); + let workers = (0..8) + .map(|_| { + let cache = Arc::clone(&cache); + thread::spawn(move || { + for _ in 0..25 { + cache + .increment_cache("counter", 1.0, ExactCacheContext::default()) + .unwrap(); + } + }) + }) + .collect::>(); + for worker in workers { + worker.join().unwrap(); + } + assert_eq!( + cache + .increment_cache("counter", 0.0, ExactCacheContext::default()) + .unwrap(), + 200.0 + ); +} + +#[rstest] +fn fractional_then_integer_increment_follows_python_behavior(sandbox: Sandbox) { + let cache = sandbox.cache::(); + assert_eq!( + cache + .increment_cache("counter", 3.5, ExactCacheContext::default()) + .unwrap(), + 3.5 + ); + assert_eq!( + cache + .increment_cache("counter", 1.0, ExactCacheContext::default()) + .unwrap(), + 1.0 + ); +} + +#[rstest] +fn increment_ttl_replacement_clears_expiry_without_ttl(sandbox: Sandbox) { + let cache = sandbox.cache::(); + cache + .increment_cache( + "counter", + 1.0, + ExactCacheContext { + ttl: Some(Duration::from_secs(60)), + }, + ) + .unwrap(); + assert!( + sandbox + .db() + .query_row( + "SELECT expire_time IS NOT NULL FROM Cache WHERE key = 'counter'", + [], + |row| row.get::<_, bool>(0) + ) + .unwrap() + ); + cache + .increment_cache("counter", 1.0, ExactCacheContext::default()) + .unwrap(); + assert!( + !sandbox + .db() + .query_row( + "SELECT expire_time IS NOT NULL FROM Cache WHERE key = 'counter'", + [], + |row| row.get::<_, bool>(0) + ) + .unwrap() + ); +} + +#[rstest] +fn custom_adapter_controls_storage_and_reads(sandbox: Sandbox) { + let cache = DiskCache::with_adapter(sandbox.store(), TextAdapter, JsonCodec::::new()); + cache + .set_cache("key", json!({"answer": 42}), &ExactCacheContext::default()) + .unwrap(); + assert!(matches!( + sandbox.store().get("key", 0.0).unwrap(), + Some(StoredValue::Text(_)) + )); + assert_eq!( + cache + .get_cache("key", &ExactCacheContext::default()) + .unwrap(), + Some(json!({"answer": 42})) + ); +} + +#[rstest] +fn delete_flush_and_spilled_file_replacement_clean_up_storage(sandbox: Sandbox) { + let large = vec![b'x'; 32 * 1024]; + sandbox + .store() + .set("large", StoredValue::Bytes(large.clone()), None, 0.0) + .unwrap(); + assert_eq!(sandbox.value_files().len(), 1); + sandbox + .store() + .set( + "large", + StoredValue::Bytes(vec![b'y'; 32 * 1024]), + None, + 0.0, + ) + .unwrap(); + assert_eq!(sandbox.value_files().len(), 1); + sandbox.store().pop("large", 0.0).unwrap(); + assert!(sandbox.value_files().is_empty()); + sandbox + .store() + .set("a", StoredValue::Bytes(large.clone()), None, 0.0) + .unwrap(); + sandbox + .store() + .set("b", StoredValue::Bytes(large), None, 0.0) + .unwrap(); + sandbox.store().clear().unwrap(); + assert!(sandbox.value_files().is_empty()); +} + +#[rstest] +#[tokio::test] +async fn async_operations_connection_and_delete_match_sync_operations(sandbox: Sandbox) { + let cache = sandbox.cache::(); + let context = ExactCacheContext { + ttl: Some(Duration::from_secs(60)), + }; + cache + .async_set_cache("a", json!(1), context.clone()) + .await + .unwrap(); + cache + .async_set_cache_pipeline( + vec![("b".into(), json!(2)), ("c".into(), json!(3))], + context.clone(), + ) + .await + .unwrap(); + assert_eq!( + cache.async_get_cache("a", &context).await.unwrap(), + Some(json!(1)) + ); + assert_eq!( + cache + .async_batch_get_cache(vec!["c".into(), "missing".into()], context.clone()) + .await + .unwrap(), + vec![BatchEntry::Hit(json!(3)), BatchEntry::Miss] + ); + cache.async_delete_cache("a").await.unwrap(); + cache.async_flush_cache().await.unwrap(); + assert_eq!( + cache.test_connection().await.unwrap().status, + litellm_cache::CacheConnectionStatus::Success + ); +} diff --git a/litellm-rust/crates/cache-disk/tests/python_compat.rs b/litellm-rust/crates/cache-disk/tests/python_compat.rs new file mode 100644 index 00000000000..9cbef8573bd --- /dev/null +++ b/litellm-rust/crates/cache-disk/tests/python_compat.rs @@ -0,0 +1,113 @@ +use litellm_cache::Error; +use litellm_cache_disk::{PythonDiskCacheAdapter, StoredValue, ValueAdapter}; +use rstest::rstest; + +enum ReadExpectation { + Bytes(&'static [u8]), + Miss, + Invalid, +} + +#[rstest] +#[case::pickled_dictionary_with_string_keys( + StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x8c, 0x01, 0x61, 0x94, 0x4b, 0x01, 0x73, 0x2e]), + ReadExpectation::Bytes(br#"{"a":1}"#) +)] +#[case::pickled_list_of_integers( + StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5d, 0x94, 0x28, 0x4b, 0x01, 0x4b, 0x02, 0x65, 0x2e]), + ReadExpectation::Bytes(br#"[1,2]"#) +)] +#[case::pickled_tuple_of_integers( + StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x01, 0x4b, 0x02, 0x86, 0x94, 0x2e]), + ReadExpectation::Bytes(br#"[1,2]"#) +)] +#[case::pickled_set_of_integers( + StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8f, 0x94, 0x28, 0x4b, 0x01, 0x4b, 0x02, 0x90, 0x2e]), + ReadExpectation::Bytes(br#"[1,2]"#) +)] +#[case::pickled_response_envelope( + StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x28, 0x8c, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x94, 0x47, 0x3f, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x94, 0x8c, 0x08, 0x7b, 0x22, 0x61, 0x22, 0x3a, 0x20, 0x31, 0x7d, 0x94, 0x75, 0x2e]), + ReadExpectation::Bytes(br#"{"response":"{\"a\": 1}","timestamp":1.5}"#) +)] +#[case::non_json_text( + StoredValue::Text("not json".into()), + ReadExpectation::Bytes(b"not json") +)] +#[case::json_text( + StoredValue::Text("{\"a\": 1}".into()), + ReadExpectation::Bytes(br#"{"a": 1}"#) +)] +#[case::non_utf8_bytes( + StoredValue::Bytes(vec![0xff, 0xfe]), + ReadExpectation::Bytes(&[0xff, 0xfe]) +)] +#[case::integer_seven(StoredValue::Integer(7), ReadExpectation::Bytes(b"7"))] +#[case::float_one_point_five(StoredValue::Float(1.5), ReadExpectation::Bytes(b"1.5"))] +#[case::pickled_true( + StoredValue::Pickle(vec![0x80, 0x05, 0x88, 0x2e]), + ReadExpectation::Bytes(b"true") +)] +#[case::pickled_negative_integer( + StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0xfd, 0xff, 0xff, 0xff, 0x2e]), + ReadExpectation::Bytes(b"-3") +)] +#[case::pickled_bytes( + StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x05, 0x62, 0x79, 0x74, 0x65, 0x73, 0x94, 0x2e]), + ReadExpectation::Invalid +)] +#[case::pickled_dictionary_with_integer_key( + StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x4b, 0x01, 0x8c, 0x01, 0x61, 0x94, 0x73, 0x2e]), + ReadExpectation::Invalid +)] +#[case::pickled_complex( + StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x2e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x08, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e, 0x73, 0x94, 0x8c, 0x07, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x94, 0x93, 0x94, 0x47, 0x3f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x86, 0x94, 0x52, 0x94, 0x2e]), + ReadExpectation::Invalid +)] +#[case::truncated_pickle( + StoredValue::Pickle(vec![0x80, 0x05, 0x2e]), + ReadExpectation::Invalid +)] +#[case::empty_bytes(StoredValue::Bytes(Vec::new()), ReadExpectation::Miss)] +#[case::empty_text(StoredValue::Text(String::new()), ReadExpectation::Miss)] +#[case::zero_integer(StoredValue::Integer(0), ReadExpectation::Miss)] +#[case::zero_float(StoredValue::Float(0.0), ReadExpectation::Miss)] +#[case::pickled_none(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e]), ReadExpectation::Miss)] +#[case::pickled_false(StoredValue::Pickle(vec![0x80, 0x05, 0x89, 0x2e]), ReadExpectation::Miss)] +#[case::pickled_zero(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x00, 0x2e]), ReadExpectation::Miss)] +#[case::pickled_zero_float(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e]), ReadExpectation::Miss)] +#[case::pickled_empty_string(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x94, 0x2e]), ReadExpectation::Miss)] +#[case::pickled_empty_list(StoredValue::Pickle(vec![0x80, 0x05, 0x5d, 0x94, 0x2e]), ReadExpectation::Miss)] +#[case::pickled_empty_dictionary(StoredValue::Pickle(vec![0x80, 0x05, 0x7d, 0x94, 0x2e]), ReadExpectation::Miss)] +#[case::pickled_empty_tuple(StoredValue::Pickle(vec![0x80, 0x05, 0x29, 0x2e]), ReadExpectation::Miss)] +fn python_read_cases(#[case] row: StoredValue, #[case] expected: ReadExpectation) { + let result = PythonDiskCacheAdapter.read(row); + match expected { + ReadExpectation::Bytes(expected) => assert_eq!(result.unwrap().unwrap(), expected), + ReadExpectation::Miss => assert_eq!(result.unwrap(), None), + ReadExpectation::Invalid => assert!(matches!(result, Err(Error::InvalidEntry))), + } +} + +#[rstest] +#[case::integer_two(Some(StoredValue::Integer(2)), 2.0)] +#[case::float_three_point_five(Some(StoredValue::Float(3.5)), 0.0)] +#[case::text_not_a_number(Some(StoredValue::Text("not a number".into())), 0.0)] +#[case::text_five(Some(StoredValue::Text("5".into())), 5.0)] +#[case::text_three_point_five(Some(StoredValue::Text("3.5".into())), 0.0)] +#[case::pickled_true(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x88, 0x2e])), 1.0)] +#[case::pickled_two(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4b, 0x02, 0x2e])), 2.0)] +#[case::pickled_dictionary(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x95, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x94, 0x8c, 0x01, 0x61, 0x94, 0x4b, 0x01, 0x73, 0x2e])), 0.0)] +#[case::missing(None, 0.0)] +#[case::pickled_none(Some(StoredValue::Pickle(vec![0x80, 0x05, 0x4e, 0x2e])), 0.0)] +fn python_counter_seed_cases(#[case] row: Option, #[case] expected: f64) { + assert_eq!(PythonDiskCacheAdapter.counter_seed(row).unwrap(), expected); +} + +#[rstest] +#[case::integer_three(3.0, StoredValue::Integer(3))] +#[case::fractional_three_point_five(3.5, StoredValue::Float(3.5))] +#[case::negative_zero(-0.0, StoredValue::Integer(0))] +#[case::large_float(1e300, StoredValue::Float(1e300))] +fn python_counter_value_cases(#[case] value: f64, #[case] expected: StoredValue) { + assert_eq!(PythonDiskCacheAdapter.counter_value(value), expected); +} diff --git a/litellm-rust/crates/cache-gcs/Cargo.toml b/litellm-rust/crates/cache-gcs/Cargo.toml new file mode 100644 index 00000000000..4ec60bcfa3b --- /dev/null +++ b/litellm-rust/crates/cache-gcs/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "litellm-cache-gcs" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +futures-util.workspace = true +litellm-auth-gcp.workspace = true +litellm-auth-types.workspace = true +litellm-cache.workspace = true +percent-encoding.workspace = true +reqwest.workspace = true +tokio.workspace = true + +[dev-dependencies] +serde_json.workspace = true +tokio.workspace = true +wiremock = "0.6.5" diff --git a/litellm-rust/crates/cache-gcs/src/cache.rs b/litellm-rust/crates/cache-gcs/src/cache.rs new file mode 100644 index 00000000000..65282ac99d5 --- /dev/null +++ b/litellm-rust/crates/cache-gcs/src/cache.rs @@ -0,0 +1,260 @@ +use std::{future::Future, sync::Arc, time::Duration}; + +use futures_util::future::try_join_all; +use litellm_cache::{ + BaseCache, BatchCache, BatchEntry, CacheCodec, CacheConnectionResult, Error, ExactCacheContext, + FlushCache, +}; +use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_encode}; +use reqwest::Client; + +use crate::{GcpTokenSource, TokenSource}; + +pub const DEFAULT_ENDPOINT: &str = "https://storage.googleapis.com"; + +const OBJECT_NAME_ENCODE_SET: &AsciiSet = &NON_ALPHANUMERIC + .remove(b'-') + .remove(b'_') + .remove(b'.') + .remove(b'~'); + +pub fn key_prefix(gcs_path: Option<&str>) -> String { + match gcs_path { + Some(path) if !path.is_empty() => format!("{}/", path.trim_end_matches('/')), + _ => String::new(), + } +} + +#[derive(Clone, Debug)] +pub struct GcsConfig { + pub bucket_name: String, + pub gcs_path: Option, + pub path_service_account: Option, + pub endpoint: String, +} + +impl GcsConfig { + pub fn new(bucket_name: impl Into) -> Self { + Self { + bucket_name: bucket_name.into(), + gcs_path: None, + path_service_account: None, + endpoint: DEFAULT_ENDPOINT.to_string(), + } + } +} + +pub struct GcsCache { + config: GcsConfig, + key_prefix: String, + client: Client, + token: Arc, + codec: S, +} + +impl GcsCache { + pub fn new(config: GcsConfig, codec: S) -> Result { + let token = Arc::new(GcpTokenSource::new(config.path_service_account.clone())); + Self::with_token_source(config, codec, token) + } + + pub fn with_token_source( + config: GcsConfig, + codec: S, + token: Arc, + ) -> Result { + let client = Client::builder().build().map_err(|_| Error::Unavailable)?; + let key_prefix = key_prefix(config.gcs_path.as_deref()); + Ok(Self { + config, + key_prefix, + client, + token, + codec, + }) + } + + pub fn bucket_name(&self) -> &str { + &self.config.bucket_name + } + + pub fn key_prefix(&self) -> &str { + &self.key_prefix + } + + pub fn path_service_account(&self) -> Option<&str> { + self.config.path_service_account.as_deref() + } + + pub fn object_name(&self, key: &str) -> String { + format!("{}{}", self.key_prefix, key) + } + + fn encoded_object_name(&self, key: &str) -> String { + percent_encode(self.object_name(key).as_bytes(), OBJECT_NAME_ENCODE_SET).to_string() + } + + fn endpoint(&self, path: &str) -> String { + format!("{}{}", self.config.endpoint.trim_end_matches('/'), path) + } + + async fn async_set(&self, key: &str, value: S::Value) -> Result<(), Error> { + let token = self.token.bearer_token().await?; + let payload = self.codec.encode(&value)?; + let url = self.endpoint(&format!( + "/upload/storage/v1/b/{}/o?uploadType=media&name={}", + self.config.bucket_name, + self.encoded_object_name(key) + )); + let response = self + .client + .post(url) + .bearer_auth(token) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(payload) + .send() + .await + .map_err(|_| Error::Unavailable)?; + if !response.status().is_success() { + return Err(Error::Unavailable); + } + Ok(()) + } + + async fn async_get(&self, key: &str) -> Result, Error> { + let token = self.token.bearer_token().await?; + let url = self.endpoint(&format!( + "/storage/v1/b/{}/o/{}?alt=media", + self.config.bucket_name, + self.encoded_object_name(key) + )); + let response = self + .client + .get(url) + .bearer_auth(token) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .send() + .await + .map_err(|_| Error::Unavailable)?; + if response.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(None); + } + if !response.status().is_success() { + return Err(Error::Unavailable); + } + let body = response.bytes().await.map_err(|_| Error::Unavailable)?; + self.codec + .decode(&body) + .map(Some) + .map_err(|_| Error::InvalidEntry) + } + + fn run_sync(future: F) -> Result + where + F: Future> + Send, + T: Send, + { + let run = || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|_| Error::Unavailable) + .and_then(|runtime| runtime.block_on(future)) + }; + if let Ok(handle) = tokio::runtime::Handle::try_current() { + if handle.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread { + return tokio::task::block_in_place(run); + } + return std::thread::scope(|scope| { + scope + .spawn(run) + .join() + .map_err(|_| Error::Unavailable) + .and_then(|result| result) + }); + } + run() + } +} + +impl BaseCache for GcsCache { + type Value = S::Value; + type Context = ExactCacheContext; + + fn get_ttl(&self, _: &Self::Context) -> Option { + None + } + + fn set_cache(&self, key: &str, value: Self::Value, _: &Self::Context) -> Result<(), Error> { + Self::run_sync(self.async_set(key, value)) + } + + fn get_cache(&self, key: &str, _: &Self::Context) -> Result, Error> { + Self::run_sync(self.async_get(key)) + } + + async fn async_set_cache( + &self, + key: &str, + value: Self::Value, + _: Self::Context, + ) -> Result<(), Error> { + self.async_set(key, value).await + } + + async fn async_get_cache( + &self, + key: &str, + _: &Self::Context, + ) -> Result, Error> { + self.async_get(key).await + } + + async fn async_set_cache_pipeline( + &self, + entries: Vec<(String, Self::Value)>, + context: Self::Context, + ) -> Result<(), Error> { + try_join_all(entries.into_iter().map(|(key, value)| { + let context = context.clone(); + async move { self.async_set_cache(&key, value, context).await } + })) + .await + .map(|_| ()) + } + + async fn disconnect(&self) -> Result<(), Error> { + Ok(()) + } + + async fn test_connection(&self) -> Result { + Err(Error::UnsupportedOperation) + } +} + +impl BatchCache for GcsCache { + async fn async_batch_get_cache( + &self, + keys: Vec, + context: Self::Context, + ) -> Result>, Error> { + try_join_all(keys.into_iter().map(|key| { + let context = context.clone(); + async move { + match self.async_get_cache(&key, &context).await { + Ok(Some(value)) => Ok(BatchEntry::Hit(value)), + Ok(None) => Ok(BatchEntry::Miss), + Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid), + Err(error) => Err(error), + } + } + })) + .await + } +} + +impl FlushCache for GcsCache { + fn flush_cache(&self) -> Result<(), Error> { + Ok(()) + } +} diff --git a/litellm-rust/crates/cache-gcs/src/lib.rs b/litellm-rust/crates/cache-gcs/src/lib.rs new file mode 100644 index 00000000000..cbb61cf0685 --- /dev/null +++ b/litellm-rust/crates/cache-gcs/src/lib.rs @@ -0,0 +1,5 @@ +mod cache; +mod token; + +pub use cache::{DEFAULT_ENDPOINT, GcsCache, GcsConfig, key_prefix}; +pub use token::{GcpTokenSource, StaticTokenSource, TokenSource}; diff --git a/litellm-rust/crates/cache-gcs/src/token.rs b/litellm-rust/crates/cache-gcs/src/token.rs new file mode 100644 index 00000000000..adb601c276c --- /dev/null +++ b/litellm-rust/crates/cache-gcs/src/token.rs @@ -0,0 +1,44 @@ +use std::{future::Future, pin::Pin}; + +use litellm_auth_gcp::{VertexAuth, VertexConfig}; +use litellm_auth_types::{InputSource, SecretValue, Sourced}; +use litellm_cache::Error; + +pub trait TokenSource: Send + Sync + 'static { + fn bearer_token(&self) -> Pin> + Send + '_>>; +} + +pub struct GcpTokenSource { + auth: VertexAuth, + config: VertexConfig, +} + +impl GcpTokenSource { + pub fn new(path_service_account: Option) -> Self { + let credentials = path_service_account + .map(|path| Sourced::new(SecretValue::new(path), InputSource::Deployment)); + Self { + auth: VertexAuth::default(), + config: VertexConfig::new(credentials, None, None), + } + } +} + +impl TokenSource for GcpTokenSource { + fn bearer_token(&self) -> Pin> + Send + '_>> { + Box::pin(async move { + self.auth + .access_token(&self.config, &|name| std::env::var(name).ok()) + .await + .map_err(|_| Error::Unavailable) + }) + } +} + +pub struct StaticTokenSource(pub String); + +impl TokenSource for StaticTokenSource { + fn bearer_token(&self) -> Pin> + Send + '_>> { + Box::pin(async move { Ok(self.0.clone()) }) + } +} diff --git a/litellm-rust/crates/cache-gcs/tests/cache.rs b/litellm-rust/crates/cache-gcs/tests/cache.rs new file mode 100644 index 00000000000..45eecf01cec --- /dev/null +++ b/litellm-rust/crates/cache-gcs/tests/cache.rs @@ -0,0 +1,324 @@ +use std::{sync::Arc, time::Duration}; + +use litellm_cache::{ + BaseCache, BatchCache, BatchEntry, CacheContext, Error, ExactCacheContext, FlushCache, + JsonCodec, +}; +use litellm_cache_gcs::{GcsCache, GcsConfig, StaticTokenSource, TokenSource, key_prefix}; +use serde_json::json; +use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{body_bytes, header, method, path, query_param}, +}; + +fn config(server: &MockServer, gcs_path: Option<&str>) -> GcsConfig { + GcsConfig { + bucket_name: "bucket".into(), + gcs_path: gcs_path.map(str::to_string), + path_service_account: None, + endpoint: server.uri(), + } +} + +fn cache(server: &MockServer, gcs_path: Option<&str>) -> GcsCache> { + GcsCache::with_token_source( + config(server, gcs_path), + JsonCodec::new(), + Arc::new(StaticTokenSource("tok".into())), + ) + .unwrap() +} + +#[tokio::test] +async fn set_writes_encoded_object_and_headers() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/upload/storage/v1/b/bucket/o")) + .and(query_param("uploadType", "media")) + .and(header("authorization", "Bearer tok")) + .and(header("content-type", "application/json")) + .and(body_bytes(br#"{"value":"entry"}"#)) + .respond_with(ResponseTemplate::new(200)) + .expect(1) + .mount(&server) + .await; + cache(&server, Some("cache/")) + .set_cache( + "team:a b/c", + json!({"value": "entry"}), + &ExactCacheContext::default(), + ) + .unwrap(); + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 1); + assert_eq!( + requests[0].url.query(), + Some("uploadType=media&name=cache%2Fteam%3Aa%20b%2Fc") + ); +} + +#[tokio::test] +async fn get_maps_statuses_and_decode_failures() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/storage/v1/b/bucket/o/hit")) + .and(query_param("alt", "media")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"}))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/storage/v1/b/bucket/o/missing")) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/storage/v1/b/bucket/o/server-error")) + .respond_with(ResponseTemplate::new(500)) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/storage/v1/b/bucket/o/invalid")) + .respond_with(ResponseTemplate::new(200).set_body_string("not json")) + .mount(&server) + .await; + + let cache = cache(&server, None); + assert_eq!( + cache + .get_cache("hit", &ExactCacheContext::default()) + .unwrap(), + Some(json!({"value": "entry"})) + ); + assert_eq!( + cache + .get_cache("missing", &ExactCacheContext::default()) + .unwrap(), + None + ); + assert_eq!( + cache + .get_cache("server-error", &ExactCacheContext::default()) + .unwrap_err(), + Error::Unavailable + ); + assert_eq!( + cache + .get_cache("invalid", &ExactCacheContext::default()) + .unwrap_err(), + Error::InvalidEntry + ); +} + +#[test] +fn key_prefix_normalizes_paths() { + assert_eq!(key_prefix(None), ""); + assert_eq!(key_prefix(Some("a/b/")), "a/b/"); + assert_eq!(key_prefix(Some("a/b")), "a/b/"); + assert_eq!(key_prefix(Some("")), ""); +} + +#[tokio::test] +async fn object_names_use_python_quote_encoding() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/upload/storage/v1/b/bucket/o")) + .and(query_param("uploadType", "media")) + .respond_with(ResponseTemplate::new(200)) + .expect(2) + .mount(&server) + .await; + let cache = cache(&server, Some("p/")); + cache + .set_cache( + "a~b-c_d.e/f g%h", + json!({"value": "punctuation"}), + &ExactCacheContext::default(), + ) + .unwrap(); + cache + .set_cache( + "ключ", + json!({"value": "utf8"}), + &ExactCacheContext::default(), + ) + .unwrap(); + let requests = server.received_requests().await.unwrap(); + let queries: Vec<_> = requests + .iter() + .filter_map(|request| request.url.query()) + .collect(); + assert!(queries.contains(&"uploadType=media&name=p%2Fa~b-c_d.e%2Ff%20g%25h")); + assert!(queries.contains(&"uploadType=media&name=p%2F%D0%BA%D0%BB%D1%8E%D1%87")); +} + +#[tokio::test] +async fn ignores_ttl_and_writes_pipeline_concurrently() { + let server = MockServer::start().await; + for key in ["one", "two", "three"] { + Mock::given(method("POST")) + .and(path("/upload/storage/v1/b/bucket/o")) + .and(query_param("uploadType", "media")) + .and(query_param("name", key)) + .respond_with(ResponseTemplate::new(200)) + .expect(1) + .mount(&server) + .await; + } + let cache = cache(&server, None); + assert_eq!(cache.get_ttl(&ExactCacheContext::default()), None); + assert_eq!( + cache.get_ttl(&ExactCacheContext::default().with_ttl(Some(Duration::from_secs(5)))), + None + ); + cache + .async_set_cache_pipeline( + vec![ + ("one".into(), json!({"key": "one"})), + ("two".into(), json!({"key": "two"})), + ("three".into(), json!({"key": "three"})), + ], + ExactCacheContext::default().with_ttl(Some(Duration::from_secs(5))), + ) + .await + .unwrap(); +} + +#[tokio::test] +async fn async_batch_get_preserves_hits_misses_and_invalid_entries() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/storage/v1/b/bucket/o/hit")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"}))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/storage/v1/b/bucket/o/missing")) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/storage/v1/b/bucket/o/invalid")) + .respond_with(ResponseTemplate::new(200).set_body_string("not json")) + .mount(&server) + .await; + + assert_eq!( + cache(&server, None) + .async_batch_get_cache( + vec!["hit".into(), "missing".into(), "invalid".into()], + ExactCacheContext::default(), + ) + .await + .unwrap(), + vec![ + BatchEntry::Hit(json!({"value": "entry"})), + BatchEntry::Miss, + BatchEntry::Invalid, + ] + ); +} + +#[tokio::test] +async fn lifecycle_operations_are_noops_and_connection_test_is_unsupported() { + let server = MockServer::start().await; + let cache = cache(&server, None); + assert_eq!(cache.flush_cache(), Ok(())); + assert_eq!(cache.disconnect().await, Ok(())); + assert_eq!( + cache.test_connection().await, + Err(Error::UnsupportedOperation) + ); +} + +#[test] +fn sync_operations_work_without_an_active_runtime() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + let server = runtime.block_on(MockServer::start()); + runtime.block_on( + Mock::given(method("POST")) + .and(path("/upload/storage/v1/b/bucket/o")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server), + ); + runtime.block_on( + Mock::given(method("GET")) + .and(path("/storage/v1/b/bucket/o/key")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"}))) + .mount(&server), + ); + let cache = cache(&server, None); + cache + .set_cache( + "key", + json!({"value": "entry"}), + &ExactCacheContext::default(), + ) + .unwrap(); + assert_eq!( + cache + .get_cache("key", &ExactCacheContext::default()) + .unwrap(), + Some(json!({"value": "entry"})) + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn sync_operations_work_inside_a_multi_thread_runtime() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/upload/storage/v1/b/bucket/o")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/storage/v1/b/bucket/o/key")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"}))) + .mount(&server) + .await; + let cache = cache(&server, None); + cache + .set_cache( + "key", + json!({"value": "entry"}), + &ExactCacheContext::default(), + ) + .unwrap(); + assert_eq!( + cache + .get_cache("key", &ExactCacheContext::default()) + .unwrap(), + Some(json!({"value": "entry"})) + ); +} + +struct FailingTokenSource; + +impl TokenSource for FailingTokenSource { + fn bearer_token( + &self, + ) -> std::pin::Pin> + Send + '_>> + { + Box::pin(async { Err(Error::Unavailable) }) + } +} + +#[tokio::test] +async fn token_source_failure_skips_http() { + let server = MockServer::start().await; + let cache = GcsCache::with_token_source( + config(&server, None), + JsonCodec::::new(), + Arc::new(FailingTokenSource), + ) + .unwrap(); + assert_eq!( + cache + .get_cache("key", &ExactCacheContext::default()) + .unwrap_err(), + Error::Unavailable + ); + assert_eq!(server.received_requests().await.unwrap().len(), 0); +} diff --git a/litellm-rust/crates/cache/src/error.rs b/litellm-rust/crates/cache/src/error.rs index ff3ff6572d4..1a381d0afd8 100644 --- a/litellm-rust/crates/cache/src/error.rs +++ b/litellm-rust/crates/cache/src/error.rs @@ -6,4 +6,6 @@ pub enum Error { InvalidEntry, #[error("flushing Redis requires an explicit namespace")] UnscopedFlush, + #[error("operation is not supported by this cache")] + UnsupportedOperation, } diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 5528e12ee2d..4a9ea633ac5 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -24,6 +24,8 @@ litellm-cache.workspace = true litellm-cache-azure-blob.workspace = true litellm-cache-memory.workspace = true litellm-cache-redis.workspace = true +litellm-cache-gcs.workspace = true +litellm-cache-disk.workspace = true litellm-cache-response.workspace = true serde.workspace = true litellm-auth.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/cache/config.rs b/litellm-rust/crates/python-bridge/src/cache/config.rs index 5c962f2bc7a..6c180708335 100644 --- a/litellm-rust/crates/python-bridge/src/cache/config.rs +++ b/litellm-rust/crates/python-bridge/src/cache/config.rs @@ -1,4 +1,4 @@ -use std::time::Duration; +use std::{path::PathBuf, time::Duration}; use litellm_cache::CacheType; use litellm_cache_redis::{RedisNode, RedisTopology}; @@ -26,6 +26,10 @@ pub(super) struct MemoryCacheConfig { pub(super) max_entry_bytes: usize, } +pub(super) struct DiskCacheConfig { + pub(super) directory: PathBuf, +} + #[derive(Debug, PartialEq)] pub(super) enum RedisProtocol { Resp2, @@ -75,6 +79,13 @@ pub(super) struct RedisCacheConfig { pub(super) connection: RedisConnectionConfig, } +#[derive(Debug, PartialEq)] +pub(super) struct GcsCacheConfig { + pub(super) bucket_name: String, + pub(super) key_prefix: String, + pub(super) path_service_account: Option, +} + pub(super) struct AzureBlobCacheConfig { pub(super) account_url: String, pub(super) container: String, @@ -94,6 +105,8 @@ const REDIS_PY_DEFAULT_MAX_CONNECTIONS: usize = 1 << 31; pub(super) enum CacheBackendConfig { Memory(MemoryCacheConfig), Redis(Box), + Gcs(GcsCacheConfig), + Disk(DiskCacheConfig), AzureBlob(AzureBlobCacheConfig), } @@ -109,6 +122,8 @@ pub(super) enum UnsupportedCacheConfig { RedisCredentials, RedisConnection, RedisOption, + GcsBucket, + DiskStore, } impl UnsupportedCacheConfig { @@ -119,6 +134,8 @@ impl UnsupportedCacheConfig { Self::RedisCredentials => "native Redis credentials require Python", Self::RedisConnection => "native Redis connection type is not implemented", Self::RedisOption => "native Redis configuration requires Python", + Self::GcsBucket => "native GCS cache requires a configured bucket name", + Self::DiskStore => "native disk cache requires the built-in diskcache store", } } } @@ -161,6 +178,20 @@ impl NativeCacheConfig { }))), Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)), }, + Some(CacheType::Gcs) => match project_gcs(&backend)? { + Ok(backend) => Ok(CacheConfigProjection::Native(Box::new(Self { + policy, + backend: CacheBackendConfig::Gcs(backend), + }))), + Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)), + }, + Some(CacheType::Disk) => match project_disk(&backend)? { + Ok(backend) => Ok(CacheConfigProjection::Native(Box::new(Self { + policy, + backend: CacheBackendConfig::Disk(backend), + }))), + Err(reason) => Ok(CacheConfigProjection::Unsupported(reason)), + }, Some(CacheType::AzureBlob) => project_azure_blob(&backend).map(|backend| { CacheConfigProjection::Native(Box::new(Self { policy, @@ -171,9 +202,7 @@ impl NativeCacheConfig { CacheType::RedisSemantic | CacheType::ValkeySemantic | CacheType::S3 - | CacheType::Disk - | CacheType::QdrantSemantic - | CacheType::Gcs, + | CacheType::QdrantSemantic, ) | None => Ok(CacheConfigProjection::Unsupported( UnsupportedCacheConfig::Backend, @@ -185,7 +214,9 @@ impl NativeCacheConfig { let default_ttl = match &self.backend { CacheBackendConfig::Memory(config) => Some(config.default_ttl), CacheBackendConfig::Redis(config) => Some(config.default_ttl), - CacheBackendConfig::AzureBlob(_) => None, + CacheBackendConfig::Disk(_) + | CacheBackendConfig::AzureBlob(_) + | CacheBackendConfig::Gcs(_) => None, }; if service.default_ttl() != default_ttl { return Some("facade and native backend default TTLs must match"); @@ -212,6 +243,42 @@ impl NativeCacheConfig { CacheBackendConfig::Redis(config) => (service.namespace() != config.namespace.as_deref()) .then_some("facade and native backend namespaces must match"), + CacheBackendConfig::Gcs(_) if service.kind() != "gcs" => { + Some("facade and native backend types must match") + } + CacheBackendConfig::Gcs(config) + if service + .gcs_backend() + .is_none_or(|backend| backend.bucket_name() != config.bucket_name) => + { + Some("facade and native backend buckets must match") + } + CacheBackendConfig::Gcs(config) + if service + .gcs_backend() + .is_none_or(|backend| backend.key_prefix() != config.key_prefix) => + { + Some("facade and native backend key prefixes must match") + } + CacheBackendConfig::Gcs(config) + if service.gcs_backend().is_none_or(|backend| { + backend.path_service_account() != config.path_service_account.as_deref() + }) => + { + Some("facade and native backend credentials must match") + } + CacheBackendConfig::Gcs(_) => None, + CacheBackendConfig::Disk(_) if service.kind() != "disk" => { + Some("facade and native backend types must match") + } + CacheBackendConfig::Disk(config) => { + let Some(directory) = service.directory() else { + return Some("facade and native backend types must match"); + }; + let native = std::fs::canonicalize(directory).ok(); + let facade = std::fs::canonicalize(&config.directory).ok(); + (native != facade).then_some("facade and native backend directories must match") + } CacheBackendConfig::AzureBlob(config) => match service.azure_blob_identity() { None => Some("facade and native backend types must match"), Some((account_url, container)) @@ -252,6 +319,38 @@ fn project_memory(backend: &Bound<'_, PyAny>) -> PyResult { }) } +#[inline(never)] +fn project_gcs( + backend: &Bound<'_, PyAny>, +) -> PyResult> { + let bucket_name = match backend.getattr("bucket_name")?.extract::>() { + Ok(Some(bucket_name)) if !bucket_name.is_empty() => bucket_name, + _ => return Ok(Err(UnsupportedCacheConfig::GcsBucket)), + }; + Ok(Ok(GcsCacheConfig { + bucket_name, + key_prefix: backend.getattr("key_prefix")?.extract::()?, + path_service_account: backend + .getattr("path_service_account")? + .extract::>()?, + })) +} + +#[inline(never)] +fn project_disk( + backend: &Bound<'_, PyAny>, +) -> PyResult> { + let store = backend.getattr("disk_cache")?; + if !instance_class_is(&store, "diskcache.core", "Cache")? + || !instance_class_is(&store.getattr("_disk")?, "diskcache.core", "Disk")? + { + return Ok(Err(UnsupportedCacheConfig::DiskStore)); + } + Ok(Ok(DiskCacheConfig { + directory: PathBuf::from(store.getattr("directory")?.extract::()?), + })) +} + #[inline(never)] fn project_redis( backend: &Bound<'_, PyAny>, @@ -639,8 +738,8 @@ mod tests { use litellm_cache_redis::{RedisNode, RedisTopology}; use super::{ - CacheBackendConfig, CacheConfigProjection, CertificateRequirement, NativeCacheConfig, - RedisProtocol, + CacheBackendConfig, CacheConfigProjection, CachePolicy, CertificateRequirement, + DiskCacheConfig, GcsCacheConfig, NativeCacheConfig, RedisProtocol, UnsupportedCacheConfig, }; use crate::cache::native::NativeResponseCache; @@ -715,6 +814,71 @@ mod tests { }); } + #[test] + fn projects_gcs_configuration() { + Python::initialize(); + Python::attach(|py| { + let facade = facade( + py, + "backend = SimpleNamespace(bucket_name='bucket', key_prefix='cache/', path_service_account='credentials.json')\n\ + facade = SimpleNamespace(type='gcs', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)", + ); + let CacheConfigProjection::Native(config) = + NativeCacheConfig::project(&facade).unwrap() + else { + panic!("GCS cache should be supported"); + }; + let CacheBackendConfig::Gcs(gcs) = config.backend else { + panic!("expected GCS configuration"); + }; + assert_eq!( + gcs, + GcsCacheConfig { + bucket_name: "bucket".into(), + key_prefix: "cache/".into(), + path_service_account: Some("credentials.json".into()), + } + ); + let matching = NativeResponseCache::gcs( + litellm_cache_gcs::GcsConfig { + bucket_name: "bucket".into(), + gcs_path: Some("cache/".into()), + path_service_account: Some("credentials.json".into()), + endpoint: litellm_cache_gcs::DEFAULT_ENDPOINT.into(), + }, + Some("token".into()), + ) + .unwrap(); + let matching_config = NativeCacheConfig { + policy: config.policy, + backend: CacheBackendConfig::Gcs(gcs), + }; + assert_eq!(matching_config.service_mismatch(&matching), None); + }); + } + + #[test] + fn rejects_gcs_without_a_bucket_name() { + Python::initialize(); + Python::attach(|py| { + let facade = facade( + py, + "backend = SimpleNamespace(bucket_name=None, key_prefix='', path_service_account=None)\n\ + facade = SimpleNamespace(type='gcs', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)", + ); + let CacheConfigProjection::Unsupported(reason) = + NativeCacheConfig::project(&facade).unwrap() + else { + panic!("GCS cache without a bucket should be unsupported"); + }; + assert!(matches!(&reason, UnsupportedCacheConfig::GcsBucket)); + assert_eq!( + reason.message(), + "native GCS cache requires a configured bucket name" + ); + }); + } + #[test] fn projects_resolved_redis_tls_configuration() { Python::initialize(); @@ -775,6 +939,85 @@ mod tests { assert_eq!(reason.message(), "native Redis credentials require Python"); }); } + #[test] + fn projects_builtin_disk_configuration_and_rejects_custom_stores() { + Python::initialize(); + Python::attach(|py| { + let root = + std::env::temp_dir().join(format!("litellm-disk-config-{}", std::process::id())); + let directory = root.to_string_lossy(); + let disk_facade = facade( + py, + &format!( + "Cache = type('Cache', (), {{'__module__': 'diskcache.core'}})\n\ + Disk = type('Disk', (), {{'__module__': 'diskcache.core'}})\n\ + store = Cache()\n\ + store._disk = Disk()\n\ + store.directory = {directory:?}\n\ + backend = SimpleNamespace(disk_cache=store)\n\ + facade = SimpleNamespace(type='disk', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)" + ), + ); + let CacheConfigProjection::Native(config) = + NativeCacheConfig::project(&disk_facade).unwrap() + else { + panic!("disk cache should be supported"); + }; + let CacheBackendConfig::Disk(disk) = config.backend else { + panic!("expected disk configuration"); + }; + assert_eq!(disk.directory, root); + let matching = NativeResponseCache::disk(&directory).unwrap(); + assert_eq!( + (NativeCacheConfig { + policy: config.policy, + backend: CacheBackendConfig::Disk(disk), + }) + .service_mismatch(&matching), + None + ); + let other = NativeResponseCache::disk(&root.join("other").to_string_lossy()).unwrap(); + let mismatch = NativeCacheConfig { + policy: CachePolicy { + mode: "default-on".into(), + ttl: None, + namespace: None, + supported_call_types: None, + redis_flush_size: None, + semantic_cache_scope: "key".into(), + }, + backend: CacheBackendConfig::Disk(DiskCacheConfig { + directory: root.clone(), + }), + }; + assert_eq!( + mismatch.service_mismatch(&other), + Some("facade and native backend directories must match") + ); + + let custom = facade( + py, + &format!( + "CustomCache = type('CustomCache', (), {{'__module__': 'mypkg'}})\n\ + CustomDisk = type('CustomDisk', (), {{'__module__': 'mypkg'}})\n\ + store = CustomCache()\n\ + store._disk = CustomDisk()\n\ + store.directory = {directory:?}\n\ + backend = SimpleNamespace(disk_cache=store)\n\ + facade = SimpleNamespace(type='disk', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)" + ), + ); + let CacheConfigProjection::Unsupported(reason) = + NativeCacheConfig::project(&custom).unwrap() + else { + panic!("custom disk store must stay on Python"); + }; + assert_eq!( + reason.message(), + "native disk cache requires the built-in diskcache store" + ); + }); + } #[test] fn projects_cluster_startup_nodes_as_redis_topology() { diff --git a/litellm-rust/crates/python-bridge/src/cache/facade.rs b/litellm-rust/crates/python-bridge/src/cache/facade.rs index 0e20676938f..0fa4511b1c1 100644 --- a/litellm-rust/crates/python-bridge/src/cache/facade.rs +++ b/litellm-rust/crates/python-bridge/src/cache/facade.rs @@ -34,6 +34,11 @@ struct RedisPoolGuard { attributes: RedisPoolAttributes, } +struct DiskStoreGuard { + reference: Py, + directory: String, +} + struct AzureBlobClientGuard { sync_client: Py, async_client: Py, @@ -46,7 +51,6 @@ enum ConnectionGuard { RedisPool(RedisPoolGuard), AzureBlob(AzureBlobClientGuard), } - struct RedisPoolAttributes { pool: &'static str, connection_class: &'static str, @@ -68,6 +72,7 @@ const CLUSTER_POOL: RedisPoolAttributes = RedisPoolAttributes { pub(super) struct FacadeGuard { outer: ObjectGuard, backend: ObjectGuard, + disk_store: Option, connection: ConnectionGuard, } @@ -218,6 +223,26 @@ impl RedisPoolGuard { } } +impl DiskStoreGuard { + fn capture(backend: &Bound<'_, PyAny>) -> PyResult { + let store = backend.getattr("disk_cache")?; + Ok(Self { + reference: store.clone().unbind(), + directory: store.getattr("directory")?.extract()?, + }) + } + + fn matches(&self, py: Python<'_>, backend: &Bound<'_, PyAny>) -> PyResult { + let store = backend.getattr("disk_cache")?; + Ok(self.reference.bind(py).is(&store) + && self.directory == store.getattr("directory")?.extract::()?) + } + + fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + visit.call(&self.reference) + } +} + impl AzureBlobClientGuard { fn capture(backend: &Bound<'_, PyAny>) -> PyResult { let sync_client = backend.getattr("container_client")?; @@ -295,6 +320,8 @@ impl FacadeGuard { "RedisClusterCache", "redis", ), + ("gcs", _) => ("litellm.caching.gcs_cache", "GCSCache", "gcs"), + ("disk", _) => ("litellm.caching.disk_cache", "DiskCache", "disk"), ("azure-blob", _) => ( "litellm.caching.azure_blob_cache", "AzureBlobCache", @@ -343,8 +370,14 @@ impl FacadeGuard { "max_size_per_item", "redis_kwargs", "redis_flush_size", + "bucket_name", + "key_prefix", + "path_service_account", ], )?, + disk_store: (kind == "disk") + .then(|| DiskStoreGuard::capture(&backend)) + .transpose()?, connection: ConnectionGuard::capture(kind, cluster, &backend)?, }) } @@ -357,12 +390,20 @@ impl FacadeGuard { if !self.backend.matches(py, &backend)? { return Ok(false); } + if let Some(guard) = &self.disk_store + && !guard.matches(py, &backend)? + { + return Ok(false); + } self.connection.matches(py, &backend) } pub(super) fn traverse(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { self.outer.traverse(&visit)?; self.backend.traverse(&visit)?; + if let Some(guard) = &self.disk_store { + guard.traverse(&visit)?; + } self.connection.traverse(&visit) } } diff --git a/litellm-rust/crates/python-bridge/src/cache/handle.rs b/litellm-rust/crates/python-bridge/src/cache/handle.rs index 002cd6b7a33..64ac3696e80 100644 --- a/litellm-rust/crates/python-bridge/src/cache/handle.rs +++ b/litellm-rust/crates/python-bridge/src/cache/handle.rs @@ -2,6 +2,8 @@ use litellm_cache_redis::{RedisNode, RedisTopology}; use litellm_host_python::{release_gil, run_sync_value}; use pyo3::{PyTraverseError, PyVisit, exceptions::PyRuntimeError, prelude::*}; +use litellm_cache_gcs::{DEFAULT_ENDPOINT, GcsConfig}; + use super::{cache_error, facade::FacadeGuard, native::NativeResponseCache, request::duration}; #[pyclass(frozen, name = "_CacheTestHandle")] @@ -64,6 +66,43 @@ impl CacheTestHandle { }) } + #[staticmethod] + #[pyo3(signature = (bucket_name, *, gcs_path=None, path_service_account=None, endpoint=None, token=None))] + fn gcs( + py: Python<'_>, + bucket_name: String, + gcs_path: Option, + path_service_account: Option, + endpoint: Option, + token: Option, + ) -> PyResult { + let config = GcsConfig { + bucket_name, + gcs_path, + path_service_account, + endpoint: endpoint.unwrap_or_else(|| DEFAULT_ENDPOINT.to_string()), + }; + let service = release_gil(py, move || NativeResponseCache::gcs(config, token)) + .map_err(cache_error)?; + Ok(Self { + service, + guard: None, + pid: std::process::id(), + }) + } + + #[staticmethod] + #[pyo3(signature = (directory))] + fn disk(py: Python<'_>, directory: String) -> PyResult { + let service = + release_gil(py, move || NativeResponseCache::disk(&directory)).map_err(cache_error)?; + Ok(Self { + service, + guard: None, + pid: std::process::id(), + }) + } + #[staticmethod] #[pyo3(signature = (account_url, container))] fn azure_blob(py: Python<'_>, account_url: String, container: String) -> PyResult { diff --git a/litellm-rust/crates/python-bridge/src/cache/mod.rs b/litellm-rust/crates/python-bridge/src/cache/mod.rs index aec08610f6e..ac4494150d9 100644 --- a/litellm-rust/crates/python-bridge/src/cache/mod.rs +++ b/litellm-rust/crates/python-bridge/src/cache/mod.rs @@ -10,7 +10,7 @@ mod resolver; use litellm_cache::Error; use pyo3::{ - exceptions::{PyRuntimeError, PyValueError}, + exceptions::{PyNotImplementedError, PyRuntimeError, PyValueError}, prelude::*, }; @@ -21,6 +21,7 @@ pub(crate) use self::{ fn cache_error(error: Error) -> PyErr { match error { Error::InvalidEntry => PyValueError::new_err(error.to_string()), + Error::UnsupportedOperation => PyNotImplementedError::new_err(error.to_string()), _ => PyRuntimeError::new_err(error.to_string()), } } diff --git a/litellm-rust/crates/python-bridge/src/cache/native.rs b/litellm-rust/crates/python-bridge/src/cache/native.rs index 2cd72a7ce14..b10f9e2a443 100644 --- a/litellm-rust/crates/python-bridge/src/cache/native.rs +++ b/litellm-rust/crates/python-bridge/src/cache/native.rs @@ -1,7 +1,9 @@ -use std::{sync::Arc, time::Duration}; +use std::{path::Path, sync::Arc, time::Duration}; use litellm_cache::{CacheCodec, CacheConnectionResult, Error}; use litellm_cache_azure_blob::AzureBlobCache; +use litellm_cache_disk::DiskCache; +use litellm_cache_gcs::{GcsCache, GcsConfig, StaticTokenSource}; use litellm_cache_memory::InMemoryCache; use litellm_cache_redis::{RedisCache, RedisTopology}; use litellm_cache_response::{ @@ -16,6 +18,8 @@ pub(super) enum NativeResponseCache { cache: Arc>>, buffer: Option>, }, + Gcs(Arc>>), + Disk(Arc>>), AzureBlob(Arc>>), } @@ -47,6 +51,22 @@ impl NativeResponseCache { buffer: None, }) } + pub fn disk(directory: &str) -> Result { + let cache = DiskCache::open(directory, ResponseCacheCodec)?; + Ok(Self::Disk(Arc::new(ResponseCache::new(Arc::new(cache))))) + } + + pub fn gcs(config: GcsConfig, token: Option) -> Result { + let backend = match token { + Some(token) => GcsCache::with_token_source( + config, + ResponseCacheCodec, + Arc::new(StaticTokenSource(token)), + )?, + None => GcsCache::new(config, ResponseCacheCodec)?, + }; + Ok(Self::Gcs(Arc::new(ResponseCache::new(Arc::new(backend))))) + } pub async fn azure_blob(account_url: &str, container: &str) -> Result { let backend = AzureBlobCache::connect( @@ -67,7 +87,7 @@ impl NativeResponseCache { cache.backend().account_url(), cache.backend().container_name(), )), - Self::Memory(_) | Self::Redis { .. } => None, + Self::Memory(_) | Self::Redis { .. } | Self::Disk(_) | Self::Gcs(_) => None, } } } @@ -77,6 +97,8 @@ impl NativeResponseCache { match self { Self::Memory(_) => "memory", Self::Redis { .. } => "redis", + Self::Gcs(_) => "gcs", + Self::Disk(_) => "disk", Self::AzureBlob(_) => "azure-blob", } } @@ -85,20 +107,23 @@ impl NativeResponseCache { match self { Self::Memory(cache) => cache.default_ttl(), Self::Redis { cache, .. } => cache.default_ttl(), + Self::Gcs(cache) => cache.default_ttl(), + Self::Disk(cache) => cache.default_ttl(), Self::AzureBlob(cache) => cache.default_ttl(), } } pub fn namespace(&self) -> Option<&str> { match self { - Self::Memory(_) | Self::AzureBlob(_) => None, + Self::Memory(_) | Self::Disk(_) | Self::AzureBlob(_) => None, Self::Redis { cache, .. } => cache.backend().namespace(), + Self::Gcs(_) => None, } } pub fn topology(&self) -> Option<&RedisTopology> { match self { - Self::Memory(_) | Self::AzureBlob(_) => None, + Self::Memory(_) | Self::Disk(_) | Self::AzureBlob(_) | Self::Gcs(_) => None, Self::Redis { cache, .. } => Some(cache.backend().topology()), } } @@ -106,14 +131,14 @@ impl NativeResponseCache { pub fn capacity(&self) -> Option { match self { Self::Memory(cache) => Some(cache.backend().max_size_in_memory()), - Self::Redis { .. } | Self::AzureBlob(_) => None, + Self::Redis { .. } | Self::Disk(_) | Self::AzureBlob(_) | Self::Gcs(_) => None, } } pub fn max_entry_bytes(&self) -> Option { match self { Self::Memory(cache) => cache.backend().max_entry_bytes(), - Self::Redis { .. } | Self::AzureBlob(_) => None, + Self::Redis { .. } | Self::Disk(_) | Self::AzureBlob(_) | Self::Gcs(_) => None, } } @@ -127,6 +152,13 @@ impl NativeResponseCache { } } + pub fn directory(&self) -> Option<&Path> { + match self { + Self::Disk(cache) => Some(cache.backend().directory()), + Self::Memory(_) | Self::Redis { .. } | Self::AzureBlob(_) | Self::Gcs(_) => None, + } + } + pub fn lookup( &self, request: &ResponseCacheRequest, @@ -135,6 +167,8 @@ impl NativeResponseCache { match self { Self::Memory(cache) => cache.lookup(request, now), Self::Redis { cache, .. } => cache.lookup(request, now), + Self::Gcs(cache) => cache.lookup(request, now), + Self::Disk(cache) => cache.lookup(request, now), Self::AzureBlob(cache) => cache.lookup(request, now), } } @@ -148,6 +182,8 @@ impl NativeResponseCache { match self { Self::Memory(cache) => cache.store(request, response, now), Self::Redis { cache, .. } => cache.store(request, response, now), + Self::Gcs(cache) => cache.store(request, response, now), + Self::Disk(cache) => cache.store(request, response, now), Self::AzureBlob(cache) => cache.store(request, response, now), } } @@ -160,6 +196,8 @@ impl NativeResponseCache { match self { Self::Memory(cache) => cache.lookup_batch(requests, now), Self::Redis { cache, .. } => cache.lookup_batch(requests, now), + Self::Gcs(cache) => cache.lookup_batch(requests, now), + Self::Disk(cache) => cache.lookup_batch(requests, now), Self::AzureBlob(cache) => cache.lookup_batch(requests, now), } } @@ -172,6 +210,8 @@ impl NativeResponseCache { match self { Self::Memory(cache) => cache.async_lookup(request, now).await, Self::Redis { cache, .. } => cache.async_lookup(request, now).await, + Self::Gcs(cache) => cache.async_lookup(request, now).await, + Self::Disk(cache) => cache.async_lookup(request, now).await, Self::AzureBlob(cache) => cache.async_lookup(request, now).await, } } @@ -192,6 +232,8 @@ impl NativeResponseCache { cache, buffer: Some(buffer), } => buffer.async_store(cache, request, response, now).await, + Self::Gcs(cache) => cache.async_store(request, response, now).await, + Self::Disk(cache) => cache.async_store(request, response, now).await, Self::AzureBlob(cache) => cache.async_store(request, response, now).await, } } @@ -204,6 +246,8 @@ impl NativeResponseCache { match self { Self::Memory(cache) => cache.async_lookup_batch(requests, now).await, Self::Redis { cache, .. } => cache.async_lookup_batch(requests, now).await, + Self::Gcs(cache) => cache.async_lookup_batch(requests, now).await, + Self::Disk(cache) => cache.async_lookup_batch(requests, now).await, Self::AzureBlob(cache) => cache.async_lookup_batch(requests, now).await, } } @@ -216,6 +260,8 @@ impl NativeResponseCache { match self { Self::Memory(cache) => cache.async_store_batch(entries, now).await, Self::Redis { cache, .. } => cache.async_store_batch(entries, now).await, + Self::Gcs(cache) => cache.async_store_batch(entries, now).await, + Self::Disk(cache) => cache.async_store_batch(entries, now).await, Self::AzureBlob(cache) => cache.async_store_batch(entries, now).await, } } @@ -229,6 +275,8 @@ impl NativeResponseCache { } cache.async_flush().await } + Self::Gcs(cache) => cache.async_flush().await, + Self::Disk(cache) => cache.async_flush().await, Self::AzureBlob(cache) => cache.async_flush().await, } } @@ -237,7 +285,16 @@ impl NativeResponseCache { match self { Self::Memory(cache) => cache.test_connection().await, Self::Redis { cache, .. } => cache.test_connection().await, + Self::Gcs(cache) => cache.test_connection().await, + Self::Disk(cache) => cache.test_connection().await, Self::AzureBlob(cache) => cache.test_connection().await, } } + + pub fn gcs_backend(&self) -> Option<&GcsCache> { + match self { + Self::Gcs(cache) => Some(cache.backend()), + _ => None, + } + } } diff --git a/litellm/constants.py b/litellm/constants.py index 88cc5b04743..895abf75047 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -2121,3 +2121,6 @@ BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY: Final = "batch_enqueued_token_limit" # Shared read-only empty mapping, for defaulting optional Mapping parameters without # constructing a fresh mutable dict at each call site. EMPTY_MAPPING: Final = MappingProxyType({}) + +# API endpoint for breached password k-anonymity search +HIBP_RANGE_API_BASE: Final = "https://api.pwnedpasswords.com/range" diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 1ef1011591e..ad8e29ad4ac 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -238,7 +238,7 @@ _TOOL_CALL_PAYLOAD_EVENT_TYPES: Final = _TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES | f _OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"}) _OUTPUT_TEXT_EVENT_TYPES: Final = frozenset({"response.output_text.delta", "response.output_text.done"}) _PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType( - {"function_call_output": "output", "message": "content"} + {"function_call_output": "output", "custom_tool_call_output": "output", "message": "content"} ) _EMPTY_RESPONSES_REQUEST: Final[ResponsesAPIOptionalRequestParams] = {} diff --git a/litellm/llms/xai/chat/transformation.py b/litellm/llms/xai/chat/transformation.py index 91bf697487d..33ee727dfab 100644 --- a/litellm/llms/xai/chat/transformation.py +++ b/litellm/llms/xai/chat/transformation.py @@ -118,6 +118,7 @@ class XAIChatConfig(OpenAIGPTConfig): base_openai_params: Final = [ "logit_bias", "logprobs", + "max_completion_tokens", "max_tokens", "n", "parallel_tool_calls", diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6b49b1d47a5..beb90c2fb8c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -43037,21 +43037,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 8.95578e-07, + "input_cost_per_token": 8.92272e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.791156e-06, + "output_cost_per_token": 1.784544e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.46315e-08, + "cache_read_input_token_cost": 7.4356e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -68212,13 +68212,13 @@ "supports_web_search": false }, "openrouter/z-ai/glm-5.3-flash": { - "input_cost_per_token": 7.5e-08, - "output_cost_per_token": 2.5e-07, - "cache_read_input_token_cost": 2e-08, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5e-07, + "cache_read_input_token_cost": 5e-08, "litellm_provider": "openrouter", "max_input_tokens": 1310720, - "max_output_tokens": 102400, - "max_tokens": 102400, + "max_output_tokens": 943718, + "max_tokens": 943718, "mode": "chat", "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, @@ -73252,14 +73252,14 @@ "supports_web_search": true }, "openrouter/~z-ai/glm-flash-latest": { - "cache_read_input_token_cost": 2e-08, - "input_cost_per_token": 7.5e-08, + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 1.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 1310720, - "max_output_tokens": 102400, - "max_tokens": 102400, + "max_output_tokens": 943718, + "max_tokens": 943718, "mode": "chat", - "output_cost_per_token": 2.5e-07, + "output_cost_per_token": 5e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, diff --git a/litellm/models/user.py b/litellm/models/user.py index 82f78c28078..92aca87d303 100644 --- a/litellm/models/user.py +++ b/litellm/models/user.py @@ -24,6 +24,8 @@ class LiteLLM_UserTable(LiteLLMPydanticObjectBase): organization_id: str | None = None object_permission_id: str | None = None password: str | None = Field(default=None, exclude=True) + password_reset_required: bool | None = None + last_breach_check_at: datetime | None = None teams: list[str] = [] user_role: str | None = None max_budget: float | None = None diff --git a/litellm/proxy/_experimental/mcp_server/contracts.py b/litellm/proxy/_experimental/mcp_server/contracts.py new file mode 100644 index 00000000000..c3129d171ad --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/contracts.py @@ -0,0 +1,95 @@ +from collections.abc import Mapping +from copy import deepcopy +from dataclasses import dataclass, field +from datetime import datetime +from types import MappingProxyType +from typing import Final, Protocol + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.types.mcp_server.mcp_server_manager import MCPServer + + +def copy_caller(auth: UserAPIKeyAuth | None) -> UserAPIKeyAuth | None: + if auth is None: + return None + span: Final = auth.parent_otel_span + return deepcopy(auth, {id(span): span} if span is not None else None) # mutable-ok: deepcopy mutates its memo + + +@dataclass(frozen=True, slots=True) +class OperationContext: + _caller: UserAPIKeyAuth | None = field(repr=False) + mcp_auth_header: str | None = field(default=None, repr=False) + mcp_servers: tuple[str, ...] | None = None + mcp_server_auth_headers: Mapping[str, Mapping[str, str]] | None = field(default=None, repr=False) + oauth2_headers: Mapping[str, str] | None = field(default=None, repr=False) + raw_headers: Mapping[str, str] | None = field(default=None, repr=False) + client_ip: str | None = None + mcp_proxy_mode: bool = False + + def __post_init__(self) -> None: + object.__setattr__(self, "_caller", copy_caller(self._caller)) + object.__setattr__(self, "mcp_servers", tuple(self.mcp_servers) if self.mcp_servers is not None else None) + object.__setattr__( + self, + "oauth2_headers", + MappingProxyType(dict(self.oauth2_headers)) if self.oauth2_headers is not None else None, + ) + object.__setattr__( + self, "raw_headers", MappingProxyType(dict(self.raw_headers)) if self.raw_headers is not None else None + ) + object.__setattr__( + self, + "mcp_server_auth_headers", + MappingProxyType( + {key: MappingProxyType(dict(value)) for key, value in self.mcp_server_auth_headers.items()} + ) + if self.mcp_server_auth_headers is not None + else None, + ) + + @property + def user_api_key_auth(self) -> UserAPIKeyAuth | None: + return copy_caller(self._caller) + + def legacy_auth( + self, + ) -> tuple[ + UserAPIKeyAuth | None, + str | None, + list[str] | None, # mutable-ok: detached legacy server-list payload + dict[str, dict[str, str]] | None, # mutable-ok: legacy auth dispatch requires concrete dict headers + dict[str, str] | None, # mutable-ok: detached legacy header payload + dict[str, str] | None, # mutable-ok: detached legacy header payload + str | None, + ]: + return ( + self.user_api_key_auth, + self.mcp_auth_header, + list(self.mcp_servers) if self.mcp_servers is not None else None, # mutable-ok: legacy policy list input + { + key: dict(value) for key, value in self.mcp_server_auth_headers.items() + } # mutable-ok: legacy auth dispatch checks concrete dict headers + if self.mcp_server_auth_headers is not None + else None, + dict(self.oauth2_headers) + if self.oauth2_headers is not None + else None, # mutable-ok: legacy OAuth header input + dict(self.raw_headers) if self.raw_headers is not None else None, # mutable-ok: legacy request header input + self.client_ip, + ) + + +class ProgressCallback(Protocol): + async def __call__(self, progress: float, total: float | None, /) -> None: ... + + +@dataclass(frozen=True, slots=True) +class AuthorizedToolCall: + name: str + arguments: Mapping[str, object] + allowed_mcp_servers: tuple[MCPServer, ...] + start_time: datetime + host_progress_callback: ProgressCallback | None + guardrail_context: Mapping[str, object] | None + logging_data: Mapping[str, object] diff --git a/litellm/proxy/_experimental/mcp_server/legacy_callbacks.py b/litellm/proxy/_experimental/mcp_server/legacy_callbacks.py new file mode 100644 index 00000000000..9e321062643 --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/legacy_callbacks.py @@ -0,0 +1,83 @@ +from collections.abc import Mapping +from typing import Final, Protocol + +from mcp.client.session import ClientRequestContext +from mcp.types import ( + CreateMessageRequestParams, + CreateMessageResult, + CreateMessageResultWithTools, + ElicitRequestParams, + ElicitResult, + ErrorData, +) + +from litellm.proxy._experimental.mcp_server.contracts import OperationContext +from litellm.proxy._types import UserAPIKeyAuth + + +class SamplingCallback(Protocol): + async def __call__( + self, context: ClientRequestContext, params: CreateMessageRequestParams, / + ) -> CreateMessageResult | CreateMessageResultWithTools | ErrorData: ... + + +class ElicitationCallback(Protocol): + async def __call__(self, context: object, params: ElicitRequestParams, /) -> ElicitResult | ErrorData: ... + + +def create_sampling_callback( + user_api_key_auth: UserAPIKeyAuth | None = None, + raw_headers: Mapping[str, str] | None = None, + client_ip: str | None = None, + operation_context: OperationContext | None = None, +) -> SamplingCallback: + from litellm.proxy._experimental.mcp_server.server import get_active_auth_context + + auth: Final = get_active_auth_context() if operation_context is None and user_api_key_auth is None else None + captured: Final = ( + operation_context + if operation_context is not None + else OperationContext( + _caller=user_api_key_auth if user_api_key_auth is not None else (auth.user_api_key_auth if auth else None), + raw_headers=raw_headers if raw_headers is not None else (auth.raw_headers if auth else None), + client_ip=client_ip if client_ip is not None else (auth.client_ip if auth else None), + ) + ) + + async def callback( + context: ClientRequestContext, params: CreateMessageRequestParams + ) -> CreateMessageResult | CreateMessageResultWithTools | ErrorData: + import litellm + from litellm.proxy._experimental.mcp_server.sampling_handler import handle_sampling_create_message + + return await handle_sampling_create_message( + context=context, + params=params, + default_model=getattr(litellm, "default_mcp_sampling_model", None), + user_api_key_auth=captured.user_api_key_auth, + raw_headers=dict(captured.raw_headers) + if captured.raw_headers is not None + else None, # mutable-ok: handler consumes an owned request header dict + client_ip=captured.client_ip, + ) + + return callback + + +def create_elicitation_callback() -> ElicitationCallback: + from litellm.proxy._experimental.mcp_server.server import get_active_mcp_session + + downstream_session: Final = get_active_mcp_session() + downstream_capabilities: Final = getattr(downstream_session, "capabilities", None) + + async def callback(context: object, params: ElicitRequestParams) -> ElicitResult | ErrorData: + from litellm.proxy._experimental.mcp_server.elicitation_handler import handle_elicitation_request + + return await handle_elicitation_request( + context=context, + params=params, + downstream_session=downstream_session, + downstream_capabilities=downstream_capabilities, + ) + + return callback diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index b49bac8a4cd..4a2713cb19c 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -73,6 +73,7 @@ from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( MCPServerAccess, _is_mcp_admitted_user_subject, ) +from litellm.proxy._experimental.mcp_server.contracts import OperationContext from litellm.proxy._experimental.mcp_server.elicitation_handler import ( MCP_ELICITATION_AVAILABLE, ) @@ -195,9 +196,6 @@ from litellm.types.mcp_server.mcp_server_manager import ( from litellm.types.utils import CallTypes if TYPE_CHECKING: - from mcp.client.session import ClientRequestContext - from mcp.types import CreateMessageRequestParams - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.types.mcp_server.mcp_toolset import MCPToolset @@ -1218,7 +1216,7 @@ async def _resolve_byok_mcp_auth_header( if not mcp_server.is_byok: return mcp_auth_header - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._experimental.mcp_server.operations import ( _check_byok_credential, _get_byok_credential, ) @@ -1577,77 +1575,25 @@ def _normalize_mcp_server_cost_info(mcp_info: MCPInfo) -> None: mcp_info["mcp_server_cost_info"] = normalized -def _create_sampling_callback(user_api_key_auth: UserAPIKeyAuth | None = None): - """ - Create a sampling callback for MCP ClientSession. - Returns a callable that handles sampling/createMessage requests from - upstream MCP servers by routing them through litellm.acompletion(). - """ +def _create_sampling_callback( + user_api_key_auth: UserAPIKeyAuth | None = None, + raw_headers: Mapping[str, str] | None = None, + client_ip: str | None = None, + operation_context: OperationContext | None = None, +): if not MCP_SAMPLING_AVAILABLE: return None + from litellm.proxy._experimental.mcp_server.legacy_callbacks import create_sampling_callback - async def _sampling_callback( - context: "ClientRequestContext", - params: "CreateMessageRequestParams", - ): - import litellm - from litellm.proxy._experimental.mcp_server.sampling_handler import ( - handle_sampling_create_message, - ) - from litellm.proxy._experimental.mcp_server.server import ( - get_active_auth_context, - ) - - auth_context: Final = get_active_auth_context() - resolved_auth: Final = user_api_key_auth or (auth_context.user_api_key_auth if auth_context else None) - # Forward original HTTP headers and client IP so that - # header-dependent guardrails, tag-based routing, trace - # correlation, and forward_llm_provider_auth_headers work - # correctly for sampling sub-calls. - _raw_headers: Final = getattr(auth_context, "raw_headers", None) - _client_ip: Final = getattr(auth_context, "client_ip", None) - - return await handle_sampling_create_message( - context=context, - params=params, - default_model=getattr(litellm, "default_mcp_sampling_model", None), - user_api_key_auth=resolved_auth, - raw_headers=_raw_headers, - client_ip=_client_ip, - ) - - return _sampling_callback + return create_sampling_callback(user_api_key_auth, raw_headers, client_ip, operation_context) def _create_elicitation_callback(): - """ - Create an elicitation callback for MCP ClientSession. - Returns a callable that handles elicitation/create requests from - upstream MCP servers. In gateway mode, this relays to the downstream - client; in tool bridge mode, it returns a decline response. - """ if not MCP_ELICITATION_AVAILABLE: return None + from litellm.proxy._experimental.mcp_server.legacy_callbacks import create_elicitation_callback - async def _elicitation_callback(context, params): - from litellm.proxy._experimental.mcp_server.elicitation_handler import ( - handle_elicitation_request, - ) - from litellm.proxy._experimental.mcp_server.server import get_active_mcp_session - - # In Gateway mode, we relay the elicitation request to the downstream client - # that triggered the current operation. - downstream_session: Final = get_active_mcp_session() - downstream_capabilities = getattr(downstream_session, "capabilities", None) if downstream_session else None - - return await handle_elicitation_request( - context=context, - params=params, - downstream_session=downstream_session, - downstream_capabilities=downstream_capabilities, - ) - - return _elicitation_callback + return create_elicitation_callback() def _record_mcp_guardrail_evaluations( @@ -3386,17 +3332,13 @@ class MCPServerManager: listable but uninvokable. Empty inside a toolset scope: toolset_mcp_route / dynamic_mcp_route set - ``_mcp_active_toolset_id`` before calling the handler, pinning the request to the toolset's + the caller's server-only ``mcp_toolset_id`` before calling the handler, pinning the request to the toolset's own servers (checking op.mcp_toolsets==[] instead would false-positive on DB-default rows where Postgres initialises the column to ARRAY[]::TEXT[]). ``allow_all_server_ids`` / ``submitted_server_ids`` are injectable so the server union, which precomputes both for its fallback path, does not compute them twice.""" - from litellm.proxy._experimental.mcp_server.mcp_context import ( # noqa: PLC0415 - _mcp_active_toolset_id, - ) - - if _mcp_active_toolset_id.get() is not None: + if user_api_key_auth is not None and user_api_key_auth.mcp_toolset_id is not None: return set() if allow_all_server_ids is None: allow_all_server_ids = self.get_allow_all_keys_server_ids() @@ -4164,6 +4106,8 @@ class MCPServerManager: subject_token: str | None = None, user_api_key_auth: UserAPIKeyAuth | None = None, cred_provider: UpstreamCredentialProvider | None = None, + raw_headers: Mapping[str, str] | None = None, + client_ip: str | None = None, ) -> MCPClient: """ Create an MCPClient instance for the given server. @@ -4212,7 +4156,13 @@ class MCPServerManager: # Create sampling and elicitation callbacks for this client sampling_cb = ( - _create_sampling_callback(user_api_key_auth=user_api_key_auth) if resolved_server.allow_sampling else None + _create_sampling_callback( + operation_context=OperationContext( + _caller=user_api_key_auth, raw_headers=raw_headers, client_ip=client_ip + ) + ) + if resolved_server.allow_sampling + else None ) elicitation_cb: Final = _create_elicitation_callback() if resolved_server.allow_elicitation else None @@ -4357,6 +4307,7 @@ class MCPServerManager: raw_headers: dict[str, str] | None = None, user_api_key_auth: UserAPIKeyAuth | None = None, oauth2_headers: dict[str, str] | None = None, + client_ip: str | None = None, ) -> list[MCPTool]: """ Helper method to get tools from a single MCP server with prefixed names. @@ -4446,6 +4397,8 @@ class MCPServerManager: stdio_env=stdio_env, subject_token=subject_token, user_api_key_auth=user_api_key_auth, + raw_headers=raw_headers, + client_ip=client_ip, ) ## HANDLE OPENAPI TOOLS @@ -4556,6 +4509,7 @@ class MCPServerManager: extra_headers: dict[str, str] | None = None, add_prefix: bool = True, raw_headers: dict[str, str] | None = None, + client_ip: str | None = None, ) -> list[Prompt]: try: headers: Final = ( @@ -4576,6 +4530,8 @@ class MCPServerManager: stdio_env=stdio_env, subject_token=subject_token, user_api_key_auth=user_api_key_auth, + raw_headers=raw_headers, + client_ip=client_ip, ) credential_fingerprint: Final = await client.discovery_auth_fingerprint() key: Final = self._discovery_key( @@ -4599,6 +4555,7 @@ class MCPServerManager: extra_headers: dict[str, str] | None = None, add_prefix: bool = True, raw_headers: dict[str, str] | None = None, + client_ip: str | None = None, ) -> list[Resource]: try: headers: Final = ( @@ -4619,6 +4576,8 @@ class MCPServerManager: stdio_env=stdio_env, subject_token=subject_token, user_api_key_auth=user_api_key_auth, + raw_headers=raw_headers, + client_ip=client_ip, ) credential_fingerprint: Final = await client.discovery_auth_fingerprint() key: Final = self._discovery_key( @@ -4642,6 +4601,7 @@ class MCPServerManager: extra_headers: dict[str, str] | None = None, add_prefix: bool = True, raw_headers: dict[str, str] | None = None, + client_ip: str | None = None, ) -> list[ResourceTemplate]: try: headers: Final = ( @@ -4662,6 +4622,8 @@ class MCPServerManager: stdio_env=stdio_env, subject_token=subject_token, user_api_key_auth=user_api_key_auth, + raw_headers=raw_headers, + client_ip=client_ip, ) credential_fingerprint: Final = await client.discovery_auth_fingerprint() key: Final = self._discovery_key( @@ -4685,6 +4647,7 @@ class MCPServerManager: mcp_auth_header: str | dict[str, str] | None = None, extra_headers: dict[str, str] | None = None, raw_headers: dict[str, str] | None = None, + client_ip: str | None = None, ) -> ReadResourceResult: """Read resource contents from a specific MCP server.""" @@ -4705,6 +4668,9 @@ class MCPServerManager: extra_headers=extra_headers, stdio_env=stdio_env, subject_token=subject_token, + raw_headers=raw_headers, + client_ip=client_ip, + user_api_key_auth=user_api_key_auth, ) return await client.read_resource(url) @@ -4718,6 +4684,7 @@ class MCPServerManager: mcp_auth_header: str | dict[str, str] | None = None, extra_headers: dict[str, str] | None = None, raw_headers: dict[str, str] | None = None, + client_ip: str | None = None, ) -> GetPromptResult: """Fetch a specific prompt definition from a single MCP server.""" @@ -4738,6 +4705,9 @@ class MCPServerManager: extra_headers=extra_headers, stdio_env=stdio_env, subject_token=subject_token, + raw_headers=raw_headers, + client_ip=client_ip, + user_api_key_auth=user_api_key_auth, ) get_prompt_request_params: Final = GetPromptRequestParams( @@ -5818,6 +5788,8 @@ class MCPServerManager: stdio_env: dict[str, str] | None, subject_token: str | None, user_api_key_auth: UserAPIKeyAuth | None, + raw_headers: Mapping[str, str] | None = None, + client_ip: str | None = None, ) -> CallToolResult: """Call a token_exchange (OBO) tool; on an upstream 401/403 re-mint the token once and retry. @@ -5843,6 +5815,8 @@ class MCPServerManager: stdio_env=stdio_env, subject_token=subject_token, user_api_key_auth=user_api_key_auth, + raw_headers=raw_headers, + client_ip=client_ip, ) return await retry_client.call_tool(call_tool_params, host_progress_callback=host_progress_callback) @@ -5860,6 +5834,7 @@ class MCPServerManager: host_progress_callback: Callable | None = None, hook_extra_headers: dict[str, str] | None = None, user_api_key_auth: UserAPIKeyAuth | None = None, + client_ip: str | None = None, ) -> CallToolResult: """ Call a regular MCP tool using the MCP client. @@ -6004,6 +5979,8 @@ class MCPServerManager: stdio_env=stdio_env, subject_token=subject_token, user_api_key_auth=user_api_key_auth, + raw_headers=raw_headers, + client_ip=client_ip, ) call_tool_params: Final = MCPCallToolRequestParams( @@ -6027,6 +6004,8 @@ class MCPServerManager: stdio_env=stdio_env, subject_token=subject_token, user_api_key_auth=user_api_key_auth, + raw_headers=raw_headers, + client_ip=client_ip, ) tool_call_coro = _obo_call_tool_limited() @@ -6202,7 +6181,7 @@ class MCPServerManager: return oauth2_headers try: - from litellm.proxy._experimental.mcp_server.server import ( # noqa: PLC0415 + from litellm.proxy._experimental.mcp_server.operations import ( # noqa: PLC0415 _get_user_oauth_extra_headers_from_db, ) @@ -6308,6 +6287,7 @@ class MCPServerManager: host_progress_callback: Callable | None = None, litellm_logging_obj: "LiteLLMLoggingObj | None" = None, guardrail_context: Mapping[str, object] | None = None, + client_ip: str | None = None, ) -> CallToolResult: """ Call a tool with the given name and arguments @@ -6434,6 +6414,7 @@ class MCPServerManager: mcp_server_auth_headers=mcp_server_auth_headers, oauth2_headers=oauth2_headers, raw_headers=raw_headers, + client_ip=client_ip, proxy_logging_obj=proxy_logging_obj, host_progress_callback=host_progress_callback, hook_extra_headers=hook_result.get("extra_headers"), diff --git a/litellm/proxy/_experimental/mcp_server/operations.py b/litellm/proxy/_experimental/mcp_server/operations.py new file mode 100644 index 00000000000..fcee3483e15 --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/operations.py @@ -0,0 +1,3102 @@ +"""Shared MCP operation policy and dispatch.""" + +import asyncio +import traceback +import types +import uuid +from collections.abc import Mapping, Sequence +from datetime import datetime +from typing import Any, Final, NoReturn, TypeAlias, overload + +from fastapi import HTTPException +from mcp import ReadResourceResult, Resource +from mcp.types import ( + CallToolRequest, + CallToolRequestParams, + CallToolResult, + GetPromptRequest, + GetPromptRequestParams, + GetPromptResult, + ListPromptsRequest, + ListPromptsResult, + ListResourcesRequest, + ListResourcesResult, + ListResourceTemplatesRequest, + ListResourceTemplatesResult, + ListToolsRequest, + ListToolsResult, + PaginatedRequestParams, + Prompt, + ReadResourceRequest, + ReadResourceRequestParams, + ResourceTemplate, + TextContent, +) +from mcp.types import Tool as MCPTool +from pydantic import AnyUrl, ConfigDict, Field, TypeAdapter +from typing_extensions import ReadOnly, TypedDict, assert_never + +from litellm._logging import verbose_logger +from litellm.constants import ( + MAXIMUM_TRACEBACK_LINES_TO_LOG, +) +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + MCPRequestHandler, +) +from litellm.proxy._experimental.mcp_server.byok_credential_cache import ( + byok_credential_cache, + byok_credential_cache_key, + cache_byok_credential, + get_cached_byok_credential, +) +from litellm.proxy._experimental.mcp_server.contracts import ( + AuthorizedToolCall, + OperationContext, + ProgressCallback, +) +from litellm.proxy._experimental.mcp_server.db import OAuthCredentialPayload +from litellm.proxy._experimental.mcp_server.exceptions import ( + MCPToolResultError, + MCPUpstreamAuthError, +) +from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( + SERVER_OUTCOMES_META_KEY, + AggregateToolListing, + ServerListOk, + ServerOutcome, + classify_list_exception, + outcome_wire_value, +) +from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + MCPServerManager, + _caller_authorization_fans_out, + _client_forwarded_authorization_headers, + _resolve_openapi_tool_auth, + _should_strip_caller_authorization, + global_mcp_server_manager, +) +from litellm.proxy._experimental.mcp_server.oauth_utils import ( + _redact_mcp_resource_url, + get_byok_www_authenticate, +) +from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + _request_auth_header, + _request_extra_headers, + _request_resolved_auth_headers, +) +from litellm.proxy._experimental.mcp_server.tool_registry import ( + global_mcp_tool_registry, +) +from litellm.proxy._experimental.mcp_server.utils import ( + MCP_TOOL_PREFIX_SEPARATOR, + MCPMissingUserEnvVarsError, + add_server_prefix_to_name, + build_synthetic_mcp_request, + extract_mcp_tool_result_error_message, + get_server_prefix, + is_tool_name_prefixed, + iter_known_server_prefixes, + logging_safe_mcp_headers, + match_known_tool_name, + normalize_server_name, + split_server_prefix_from_name, + strip_known_server_prefix, +) +from litellm.proxy._types import ( + UserAPIKeyAuth, +) +from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import ( + publish_auth_cache_invalidation, +) +from litellm.proxy.litellm_pre_call_utils import ( + LiteLLMProxyRequestSetup, + get_chain_id_from_headers, +) +from litellm.types.mcp import ( + DEFAULT_CREDENTIAL_HEADER, + MCPAuth, + without_header, +) +from litellm.types.mcp_server.mcp_server_manager import MCPInfo, MCPServer +from litellm.types.utils import CallTypes, StandardLoggingMCPToolCall +from litellm.utils import Rules, client, function_setup + +__all__ = ( + "_MCP_CREDENTIAL_REQUEST_FIELDS", + "ListMCPToolsRestAPIResponseObject", + "MCPInfo", + "MCPServer", + "_McpDeniedDetail", + "_aggregate_server_key", + "_build_virtual_call_logging_obj", + "_check_byok_credential", + "_client_has_passthrough_authorization", + "_client_has_per_server_auth_header", + "_dispatch_virtual_mcp_tool", + "_fire_mcp_tool_call_logging", + "_get_allowed_mcp_servers", + "_get_allowed_mcp_servers_from_mcp_server_names", + "_get_byok_credential", + "_get_prompts_from_mcp_servers", + "_get_resource_templates_from_mcp_servers", + "_get_resources_from_mcp_servers", + "_get_standard_logging_mcp_tool_call", + "_get_tools_from_mcp_servers", + "_get_user_oauth_extra_headers_from_db", + "_handle_local_mcp_tool", + "_handle_managed_mcp_tool", + "_http_detail_message", + "_invalidate_byok_cred_cache", + "_list_mcp_prompts", + "_list_mcp_resource_templates", + "_list_mcp_resources", + "_list_mcp_tools", + "_list_tools_before_first_call", + "_mcp_session_id_from_headers", + "_merge_gateway_initialize_instructions", + "_prefetch_oauth_creds_for_user", + "_prepare_mcp_server_headers", + "_raise_if_initialize_grants_no_mcp_servers", + "_resolve_display_name_to_original", + "_run_post_mcp_call_guardrails", + "_server_answers_to", + "_tool_name_matches", + "apply_tool_overrides", + "call_mcp_tool", + "execute_mcp_tool", + "filter_tools_by_allowed_tools", + "filter_tools_by_key_team_permissions", + "fire_mcp_tool_call_failure_logging", + "mcp_get_prompt", + "mcp_read_resource", + "raise_denied_scoped_mcp_access", +) + + +async def _invalidate_byok_cred_cache(user_id: str, server_id: str) -> None: + """Drop a stored-or-deleted BYOK credential from this worker's cache and from every peer worker's.""" + cache_key: Final = byok_credential_cache_key(user_id, server_id) + byok_credential_cache.delete_cache(cache_key) + await publish_auth_cache_invalidation(cache_key=cache_key) + + +def _mcp_session_id_from_headers( + raw_headers: dict[str, str] | None, +) -> str | None: + """The ``mcp-session-id`` of a stateful MCP session, read case-insensitively + from the request headers. ``None`` for stateless calls (no such header).""" + if not raw_headers: + return None + for key, value in raw_headers.items(): + if isinstance(key, str) and key.lower() == "mcp-session-id": + return value or None + return None + + +class ListMCPToolsRestAPIResponseObject(MCPTool): + """ + Object returned by the /tools/list REST API route. + """ + + mcp_info: MCPInfo | None = Field(default=None, alias="mcp_info") + model_config = ConfigDict(arbitrary_types_allowed=True) + + +async def _build_virtual_call_logging_obj( + name: str, + arguments: dict[str, object], + user_api_key_auth: UserAPIKeyAuth, + raw_headers: Mapping[str, str] | None = None, + client_ip: str | None = None, +) -> LiteLLMLoggingObj | None: + """Run the pre-call pipeline (guardrails + logging setup) for a virtual + mcp_tool_call so the SSE path spend-logs like the REST path.""" + from litellm.proxy.common_request_processing import ( + ProxyBaseLLMRequestProcessing, + ) + from litellm.proxy.proxy_server import ( + general_settings, + proxy_config, + proxy_logging_obj, + ) + + request: Final = build_synthetic_mcp_request( + path="/mcp/tools/call", + raw_headers=raw_headers, + client_ip=client_ip, + ) + _, virtual_logging_obj = await ProxyBaseLLMRequestProcessing( + data={"name": name, "arguments": arguments} + ).common_processing_pre_call_logic( + request=request, + user_api_key_dict=user_api_key_auth, + proxy_config=proxy_config, + route_type=CallTypes.call_mcp_tool.value, + proxy_logging_obj=proxy_logging_obj, + general_settings=general_settings, + ) + return virtual_logging_obj + + +async def _dispatch_virtual_mcp_tool( + name: str, + arguments: dict[str, object] | None, + user_api_key_auth: UserAPIKeyAuth | None, + client_ip: str | None, + mcp_servers: list[str] | None = None, + mcp_auth_header: str | None = None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, + oauth2_headers: dict[str, str] | None = None, + raw_headers: dict[str, str] | None = None, + mcp_proxy_mode: bool = False, +) -> CallToolResult | None: + """Handle the mcp_tool_search / mcp_tool_call virtual tools. + + Returns a CallToolResult when ``name`` is a virtual tool, else ``None`` so + the caller falls through to normal tool routing. + """ + from litellm.llms.litellm_proxy.skills.skill_search import DEFAULT_SKILL_SEARCH_TOP_K + from litellm.proxy._experimental.mcp_server.tool_search import ( + AGENT_SEARCH_TOOL_NAME, + DEFAULT_AGENT_SEARCH_TOP_K, + MCP_PROXY_CALL_TOOL_NAME, + MCP_PROXY_TOOL_NAMES, + MCP_TOOL_SEARCH_TOOL_NAME, + SKILL_SEARCH_TOOL_NAME, + VIRTUAL_TOOL_NAMES, + coerce_top_k, + handle_agent_search, + handle_mcp_proxy_tool, + handle_mcp_tool_call, + handle_mcp_tool_search, + handle_skill_search, + ) + + if mcp_proxy_mode and name not in MCP_PROXY_TOOL_NAMES: + return CallToolResult( + content=[ # mutable-ok: MCP result content + TextContent(type="text", text=f"Tool {name} is unavailable on /mcp/proxy") + ], + is_error=True, + ) + + if mcp_proxy_mode and name in MCP_PROXY_TOOL_NAMES: + assert user_api_key_auth is not None + proxy_call_start: Final = datetime.now() # noqa: DTZ005 # logging pipeline uses naive datetimes + proxy_logging_obj: Final = ( + await _build_virtual_call_logging_obj( + name=name, + arguments=arguments or {}, # mutable-ok: logging pipeline payload + user_api_key_auth=user_api_key_auth, + raw_headers=raw_headers, + client_ip=client_ip, + ) + if name == MCP_PROXY_CALL_TOOL_NAME + else None + ) + try: + proxy_result: Final = await handle_mcp_proxy_tool( + name=name, + arguments=arguments or {}, # mutable-ok: proxy handler payload + user_api_key_dict=user_api_key_auth, + client_ip=client_ip, + mcp_servers=mcp_servers, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + litellm_logging_obj=proxy_logging_obj, + ) + except Exception as exc: + if proxy_logging_obj is not None: + from litellm.proxy.proxy_server import proxy_logging_obj as request_logging_obj + + failure_end: Final = datetime.now() # noqa: DTZ005 # matches the logging pipeline start time + failure_traceback: Final = traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG) + try: + proxy_logging_obj.failure_handler(exc, failure_traceback, proxy_call_start, failure_end) + await proxy_logging_obj.async_failure_handler(exc, failure_traceback, proxy_call_start, failure_end) + if not isinstance(exc, MCPUpstreamAuthError): + await request_logging_obj.post_call_failure_hook( + request_data={ # mutable-ok: failure hook mutates its request payload + "name": name, + "arguments": arguments, + "litellm_logging_obj": proxy_logging_obj, + }, + original_exception=exc, + user_api_key_dict=user_api_key_auth, + route="/mcp/call_tool", + traceback_str=failure_traceback, + ) + except Exception: # noqa: BLE001 # a failing failure hook must not mask the tool call's own error + verbose_logger.exception("Error logging failed MCP proxy tool call") + raise + if proxy_logging_obj is not None: + return await _fire_mcp_tool_call_logging( + logging_obj=proxy_logging_obj, + result=proxy_result, + start_time=proxy_call_start, + end_time=datetime.now(), # noqa: DTZ005 # matches the logging pipeline start time + user_api_key_auth=user_api_key_auth, + request_data=types.MappingProxyType({"name": name, "arguments": arguments}), + ) + return proxy_result + + if name not in VIRTUAL_TOOL_NAMES: + return None + + if not getattr( + getattr(user_api_key_auth, "object_permission", None), + "mcp_tool_search_enabled", + False, + ): + return CallToolResult( + content=[ + TextContent( + type="text", + text=f"Tool {name} requires mcp_tool_search_enabled on the key", + ) + ], + is_error=True, + ) + + args: Final = arguments or {} + if name == MCP_TOOL_SEARCH_TOOL_NAME: + return await handle_mcp_tool_search( + query=TypeAdapter(str).validate_python(args.get("query", "")), + top_k=coerce_top_k(args.get("top_k", 5)), + user_api_key_dict=user_api_key_auth, + client_ip=client_ip, + mcp_servers=mcp_servers, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + + assert user_api_key_auth is not None # guaranteed by the flag check above + if name == AGENT_SEARCH_TOOL_NAME: + return await handle_agent_search( + query=str(args.get("query", "")), + top_k=coerce_top_k(args.get("top_k", DEFAULT_AGENT_SEARCH_TOP_K), default=DEFAULT_AGENT_SEARCH_TOP_K), + user_api_key_dict=user_api_key_auth, + ) + if name == SKILL_SEARCH_TOOL_NAME: + return await handle_skill_search( + query=str(args.get("query", "")), + top_k=coerce_top_k(args.get("top_k", DEFAULT_SKILL_SEARCH_TOP_K), default=DEFAULT_SKILL_SEARCH_TOP_K), + user_api_key_dict=user_api_key_auth, + ) + virtual_logging_obj: Final = await _build_virtual_call_logging_obj( + name=name, + arguments=args, + user_api_key_auth=user_api_key_auth, + raw_headers=raw_headers, + client_ip=client_ip, + ) + tool_request: Final = CallToolRequestParams.model_validate( + types.MappingProxyType({"name": args.get("tool_name", ""), "arguments": args.get("arguments") or {}}) + ) + return await handle_mcp_tool_call( + tool_name=tool_request.name, + arguments=tool_request.arguments or {}, + user_api_key_dict=user_api_key_auth, + client_ip=client_ip, + mcp_servers=mcp_servers, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + litellm_logging_obj=virtual_logging_obj, + ) + + +async def _get_allowed_mcp_servers_from_mcp_server_names( + mcp_servers: Sequence[str] | None, + allowed_mcp_servers: list[MCPServer], +) -> list[MCPServer]: + """ + Get the filtered MCP servers from the MCP server names. + + Fails closed when ``mcp_servers`` is explicitly provided (path- or + header-derived) but none of the names resolve to a server alias or + access group the caller can access. The previous behavior returned + the full ``allowed_mcp_servers`` set, which silently widened scope + when a client targeted ``/mcp//`` and made URL/header + namespacing appear to work when it did not. + """ + + filtered_server: Final[dict[str, MCPServer]] = {} + # Filter servers based on mcp_servers parameter if provided + if mcp_servers is not None: + for server_or_group in mcp_servers: + server_name_matched = False + + for server in allowed_mcp_servers: + if server and _server_answers_to(server, server_or_group): + filtered_server[server.server_id] = server + server_name_matched = True + break + + if not server_name_matched: + try: + access_group_server_ids = await MCPRequestHandler._get_mcp_servers_from_access_groups( + [server_or_group] + ) + # Only include servers that the user has access to + for server_id in access_group_server_ids: + for server in allowed_mcp_servers: + if server_id == server.server_id: + filtered_server[server.server_id] = server + except Exception as e: + verbose_logger.debug("Could not resolve '%s' as access group: %s", server_or_group, e) + + if filtered_server: + return list(filtered_server.values()) + + if mcp_servers is not None: + # Caller asked for a specific scope but nothing resolved. Fail + # closed so URL/header namespacing cannot silently fall back to + # the caller's full allowed-server set. + verbose_logger.debug( + "MCP scope filter resolved to no servers for requested names %s; returning empty list (fail-closed).", + mcp_servers, + ) + return [] + + return allowed_mcp_servers + + +def _http_detail_message(detail: object) -> str: + return str(detail.get("error")) if isinstance(detail, dict) and detail.get("error") else str(detail) + + +def _server_answers_to(server: MCPServer, name: str) -> bool: + requested: Final = name.lower() + return any(requested == known.lower() for known in iter_known_server_prefixes(server) if known) + + +async def raise_denied_scoped_mcp_access( + requested_names: Sequence[str], + user_api_key_auth: UserAPIKeyAuth | None, + client_ip: str | None = None, +) -> None: + """A scoped request (``/mcp/`` path or ``x-mcp-servers`` header) resolved to zero + allowed servers, so the denial must be loud: a silent 200 with no tools reads as a healthy + server with no tools. Unknown, unauthorized, and access-group names all share one generic + error so scoping cannot probe which servers exist; the agent variant fires only when the + same request resolves once the agent binding is stripped, proving the binding caused the veto.""" + agent_id: Final = user_api_key_auth.agent_id if user_api_key_auth else None + if user_api_key_auth is not None and agent_id: + resolved_without_agent: Final = await _get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth.model_copy(update=types.MappingProxyType({"agent_id": None})), + mcp_servers=requested_names, + client_ip=client_ip, + ) + + def _resolved_to_server(name: str) -> bool: + return any(_server_answers_to(server, name) for server in resolved_without_agent) + + vetoed_server: Final = next((name for name in requested_names if _resolved_to_server(name)), None) + if vetoed_server is not None: + agent_denial: Final[_McpDeniedDetail] = { + "error": ( + f"MCP server '{vetoed_server}' is not available to this key: the key is bound to " + f"agent '{agent_id}', whose MCP grants do not include this server. Add the server " + f"to the agent's object_permission.mcp_servers (edit the agent in the Admin UI or " + f"PATCH /v1/agents/{agent_id}), or use a key that is not bound to the agent." + ) + } + raise HTTPException(status_code=403, detail=agent_denial) + vetoed_group: Final = next( + ( + name + for name in requested_names + if not _resolved_to_server(name) + and any(name in (server.access_groups or ()) for server in resolved_without_agent) + ), + None, + ) + if vetoed_group is not None: + group_denial: Final[_McpDeniedDetail] = { + "error": ( + f"MCP access group '{vetoed_group}' is not available to this key: the key is bound to " + f"agent '{agent_id}', whose MCP grants do not include it. Add the group to the " + f"agent's object_permission.mcp_access_groups (edit the agent in the Admin UI or " + f"PATCH /v1/agents/{agent_id}), or use a key that is not bound to the agent." + ) + } + raise HTTPException(status_code=403, detail=group_denial) + generic_denial: Final[_McpDeniedDetail] = { + "error": f"The key is not allowed to access the requested MCP servers: {', '.join(requested_names)}" + } + raise HTTPException(status_code=403, detail=generic_denial) + + +def _tool_name_matches(tool_name: str, filter_list: list[str], mcp_server: MCPServer) -> bool: + """ + Check if a tool name matches any name in the filter list. + + Reads the same owner the server-level permission checks use, so discovery hides + exactly what dispatch refuses. ``mcp_server`` is required: guessing the boundary + at the first separator mismatches every tool on a server whose prefix contains + the separator. + """ + bare_name: Final = strip_known_server_prefix(tool_name, mcp_server) + return match_known_tool_name(bare_name, mcp_server, filter_list) is not None + + +def filter_tools_by_allowed_tools( + tools: list[MCPTool], + mcp_server: MCPServer, +) -> list[MCPTool]: + """ + Filter tools by allowed/disallowed tools configuration. + + If allowed_tools is set, only tools in that list are returned. + If disallowed_tools is set, tools in that list are excluded. + Tool names are matched with and without server prefixes for flexibility. + + Args: + tools: List of tools to filter + mcp_server: Server configuration with allowed_tools/disallowed_tools + + Returns: + Filtered list of tools + """ + from litellm.proxy._experimental.mcp_server.utils import ( + server_applies_tool_allowlist, + ) + + tools_to_return = tools + + # Filter by allowed_tools (whitelist) + if server_applies_tool_allowlist(mcp_server): + if not mcp_server.allowed_tools: + return [] + tools_to_return = [ + tool for tool in tools if _tool_name_matches(tool.name, mcp_server.allowed_tools, mcp_server) + ] + + # Filter by disallowed_tools (blacklist) + if mcp_server.disallowed_tools: + tools_to_return = [ + tool + for tool in tools_to_return + if not _tool_name_matches(tool.name, mcp_server.disallowed_tools, mcp_server) + ] + + return tools_to_return + + +def apply_tool_overrides( + tools: list[MCPTool], + mcp_server: MCPServer, +) -> list[MCPTool]: + """Apply admin-configured display name/description overrides to tools. + + Overrides are keyed by the unprefixed tool name, same convention as + allowed_tools configuration. + """ + display_name_map: Final = mcp_server.tool_name_to_display_name or {} + description_map: Final = mcp_server.tool_name_to_description or {} + if not display_name_map and not description_map: + return tools + + for tool in tools: + unprefixed = strip_known_server_prefix(tool.name, mcp_server) + lookup_key = unprefixed or tool.name + if lookup_key in display_name_map: + tool.name = display_name_map[lookup_key] + if lookup_key in description_map: + tool.description = description_map[lookup_key] + return tools + + +async def _get_allowed_mcp_servers( + user_api_key_auth: UserAPIKeyAuth | None, + mcp_servers: Sequence[str] | None, + client_ip: str | None = None, +) -> list[MCPServer]: + """Return allowed MCP servers for a request after applying filters. + + Args: + user_api_key_auth: The authenticated user's API key info. + mcp_servers: Optional list of server names to filter to. + client_ip: Client IP for IP-based access control. If None, falls back to + auth context. Pass explicitly from request handlers for safety. + Note: If client_ip is None and auth context is not set, IP filtering is skipped. + This is intentional for internal callers but may indicate a bug if called + from a request handler without proper context setup. + """ + allowed_mcp_server_ids = await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth) + ( + allowed_mcp_server_ids, + _ip_blocked, + ) = global_mcp_server_manager.filter_server_ids_by_ip_with_info(allowed_mcp_server_ids, client_ip) + verbose_logger.debug( + "MCP IP filter: client_ip=%s, allowed_server_ids=%s", + client_ip, + allowed_mcp_server_ids, + ) + if _ip_blocked > 0: + verbose_logger.debug( + "MCP IP filtering: %d server(s) are not accessible from client IP %s " + "because they are restricted to internal networks. " + "No tools from those servers will be returned. " + "To expose a server externally, set 'available_on_public_internet: true' " + "in its configuration.", + _ip_blocked, + client_ip, + ) + allowed_mcp_servers: list[MCPServer] = [] + for allowed_mcp_server_id in allowed_mcp_server_ids: + mcp_server = global_mcp_server_manager.get_mcp_server_by_id(allowed_mcp_server_id) + if mcp_server is not None: + # Apply the request-time oauth2_flow backstop for legacy null rows. + mcp_server = MCPServerManager.resolve_oauth2_flow_for_request(mcp_server) + allowed_mcp_servers.append(mcp_server) + + if mcp_servers is not None: + allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names( + mcp_servers=mcp_servers, + allowed_mcp_servers=allowed_mcp_servers, + ) + + return allowed_mcp_servers + + +def _client_has_per_server_auth_header( + server: MCPServer, + mcp_server_auth_headers: dict[str, dict[str, str]] | None, +) -> bool: + """True if the request carries a per-server ``x-mcp-{alias}-authorization`` + header for this server. This is the multi-server binding: it names one + upstream, so it is unambiguously the caller's upstream token regardless of + auth mode (never the LiteLLM admission credential). + + Resolves through the same ``lookup_mcp_server_auth_in_headers`` egress uses, so + the connect gate and egress agree on which per-server header names match: a + dashboard client sends ``x-mcp-{sanitize_mcp_alias_for_header(alias)}-authorization``, + and matching only the raw alias here would 401 a token egress would forward. + """ + if not mcp_server_auth_headers: + return False + from litellm.proxy._experimental.mcp_server.utils import ( + lookup_mcp_server_auth_in_headers, + ) + + server_headers: Final = lookup_mcp_server_auth_in_headers( + mcp_server_auth_headers, + alias=server.alias, + server_name=server.server_name, + access_groups=server.access_groups, + ) + if isinstance(server_headers, str): + return bool(server_headers.strip()) + if isinstance(server_headers, dict): + return any(isinstance(hk, str) and hk.lower() == "authorization" for hk in server_headers) + return False + + +def _client_has_passthrough_authorization( + server: MCPServer, + oauth2_headers: dict[str, str] | None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None, +) -> bool: + """True if the incoming request already carries an ``Authorization`` + header the gateway will forward to this pass-through server. + + The client may supply the bearer as either the top-level + ``Authorization`` header (surfaced via ``oauth2_headers``) or a + per-server ``x-mcp-auth-`` style header (surfaced via + ``mcp_server_auth_headers``). Either form skips the pre-emptive 401. + """ + if oauth2_headers: + for k in oauth2_headers: + if k.lower() == "authorization": + return True + return _client_has_per_server_auth_header(server, mcp_server_auth_headers) + + +async def _get_user_oauth_extra_headers_from_db( + server: MCPServer, + user_api_key_auth: UserAPIKeyAuth | None, + prefetched_creds: 'Mapping[str, "OAuthCredentialPayload"] | None' = None, +) -> dict[str, str] | None: + """Stored OAuth2 token for (user, server) as an ``Authorization: Bearer`` header, or None. + + Thin wrapper over ``resolve_user_oauth_access_token`` (Redis cache, else DB + refresh); + ``prefetched_creds`` skips the per-server Redis/DB lookups for the batch path. + """ + if server.auth_type != MCPAuth.oauth2 or user_api_key_auth is None: + return None + from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 + resolve_user_oauth_access_token, + ) + + token: Final = await resolve_user_oauth_access_token( + getattr(user_api_key_auth, "user_id", None), server, prefetched_creds + ) + return {"Authorization": f"Bearer {token}"} if token else None + + +async def _prefetch_oauth_creds_for_user( + user_api_key_auth: UserAPIKeyAuth | None, +) -> dict[str, "OAuthCredentialPayload"]: + """Fetch all OAuth2 credentials for the user in one DB query. + + Returns a dict keyed by server_id to avoid N+1 queries in asyncio.gather loops. + """ + user_id: Final[str | None] = getattr(user_api_key_auth, "user_id", None) if user_api_key_auth else None + if not user_id: + return {} + try: + from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 + list_user_oauth_credentials, + ) + from litellm.proxy.utils import get_prisma_client_or_throw # noqa: PLC0415 + + prisma_client: Final = get_prisma_client_or_throw( + "Database not connected. Connect a database to use OAuth2 MCP tools." + ) + creds: Final = await list_user_oauth_credentials(prisma_client, user_id) + return {c["server_id"]: c for c in creds if "server_id" in c} + except Exception: + verbose_logger.warning("_prefetch_oauth_creds_for_user: failed to prefetch OAuth credentials") + return {} + + +def _prepare_mcp_server_headers( + server: MCPServer, + mcp_server_auth_headers: dict[str, dict[str, str]] | None, + mcp_auth_header: str | None, + oauth2_headers: dict[str, str] | None, + raw_headers: dict[str, str] | None, + user_api_key_auth: UserAPIKeyAuth | None = None, + scope_servers: list[MCPServer] | None = None, +) -> tuple[dict[str, str] | str | None, dict[str, str] | None]: + """Build auth and extra headers for a server. + + ``scope_servers`` is the full server list a fan-out handler iterates. Passing it lets the + client-forwarded token modes withhold the caller's request-wide ``Authorization`` when + another server in the scope would also receive it (``_caller_authorization_fans_out``); + explicitly-addressed operations leave it None. Per-server ``x-mcp-{alias}-authorization`` + headers are unaffected — they bind one token to one server and are the multi-server shape. + """ + server_auth_header: dict[str, str] | str | None = None + if mcp_server_auth_headers: + from litellm.proxy._experimental.mcp_server.utils import ( + lookup_mcp_server_auth_in_headers, + ) + + server_auth_header = lookup_mcp_server_auth_in_headers( + mcp_server_auth_headers, + alias=server.alias, + server_name=server.server_name, + access_groups=server.access_groups, + ) + + extra_headers: dict[str, str] | None = None + is_client_forwarded_mode: Final = server.is_client_forwarded_token + # In a multi-server listing scope the request-wide Authorization can only carry one token, + # so it is withheld from a client-forwarded server when another server in scope also consumes + # it (RFC 9700 cross-resource replay); such scopes must bind per-server via + # x-mcp-{alias}-authorization. The decision is computed once so BOTH the forwarding branch and + # the extra_headers copy loop below honor it — otherwise a server that lists Authorization in + # extra_headers would re-copy the withheld bearer from raw_headers and replay it anyway. + withhold_forwarded_authorization: Final = is_client_forwarded_mode and _caller_authorization_fans_out( + server, scope_servers + ) + if server.auth_type == MCPAuth.oauth2: + # For OAuth2 M2M servers, upstream Authorization must come from + # client_credentials token fetch, never from caller headers. + if server.has_client_credentials: + extra_headers = None + else: + # Copy to avoid mutating the original dict (important for parallel fetching) + extra_headers = oauth2_headers.copy() if oauth2_headers else None + # Migrated authorization_code: the v2 resolver injects the stored per-user + # token, so drop the caller-forwarded Authorization (apply-if-absent would + # otherwise let it shadow the resolved token). Delegate keeps it. Centralized + # via _should_strip_caller_authorization to match _call_regular_mcp_tool. + if extra_headers and _should_strip_caller_authorization( + mcp_server=server, + raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, + ): + extra_headers = without_header(extra_headers, DEFAULT_CREDENTIAL_HEADER) + elif is_client_forwarded_mode: + if not withhold_forwarded_authorization: + extra_headers = _client_forwarded_authorization_headers( + mcp_server=server, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, + ) + + if server.extra_headers and raw_headers: + if extra_headers is None: + extra_headers = {} + + normalized_raw_headers: Final = {str(k).lower(): v for k, v in raw_headers.items() if isinstance(k, str)} + + # Centralized strip decision shared with + # ``MCPServerManager._call_regular_mcp_tool`` so the two + # code paths cannot drift on this security-sensitive choice. + # See ``_should_strip_caller_authorization`` for the rules. + strip_caller_authorization: Final = _should_strip_caller_authorization( + mcp_server=server, + raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, + ) + + for header in server.extra_headers: + if not isinstance(header, str): + continue + if header.lower() == "authorization" and (strip_caller_authorization or withhold_forwarded_authorization): + continue + header_value = normalized_raw_headers.get(header.lower()) + if header_value is None: + continue + extra_headers[header] = header_value + + # Reset to None if no headers were actually added + if extra_headers is not None and len(extra_headers) == 0: + extra_headers = None + + if server_auth_header is None: + server_auth_header = mcp_auth_header + + return server_auth_header, extra_headers + + +def _merge_gateway_initialize_instructions( + allowed_mcp_servers: list[MCPServer], +) -> str | None: + """YAML/DB override, else upstream text (prefetch on init, or list_tools / health_check / call_tool cache).""" + if not allowed_mcp_servers: + return None + + texts: Final[list[tuple[str, str]]] = [] + for server in allowed_mcp_servers: + label = server.alias or server.server_name or server.name or server.server_id or "mcp" + if server.instructions and server.instructions.strip(): + texts.append((label, server.instructions.strip())) + continue + if server.spec_path: + continue + cached = global_mcp_server_manager._upstream_initialize_instructions_by_server_id.get(server.server_id) + if cached and cached.strip(): + texts.append((label, cached.strip())) + + if not texts: + return None + if len(texts) == 1: + return texts[0][1] + return "\n\n---\n\n".join(f"[{lbl}]\n{txt}" for lbl, txt in texts) + + +async def _raise_if_initialize_grants_no_mcp_servers( + allowed: Sequence[MCPServer], + user_api_key_auth: UserAPIKeyAuth | None, + mcp_servers: Sequence[str] | None, + client_ip: str | None, +) -> None: + if allowed or user_api_key_auth is None or not user_api_key_auth.api_key: + return + if mcp_servers: + await raise_denied_scoped_mcp_access( + requested_names=mcp_servers, + user_api_key_auth=user_api_key_auth, + client_ip=client_ip, + ) + no_servers_denial: Final[_McpDeniedDetail] = { + "error": ( + "The key has no MCP servers granted, or none of its granted servers is loaded and allowed for " + "this client IP. Grant servers or access groups to the key, its team, or its organization " + "(object_permission.mcp_servers), check the server's allowed IPs, and reconnect." + ) + } + raise HTTPException(status_code=403, detail=no_servers_denial) + + +def _aggregate_server_key(server: MCPServer) -> str: + """The client-visible key for a server in listing outcomes and spend metadata: the same + display prefix (alias, or the short prefix when that mode is enabled) the caller already + sees on the tool names. Canonical internal server names never key a caller-readable + surface; when the display naming deliberately hides them, the outcome keys must too.""" + return get_server_prefix(server) or "unknown" + + +async def _get_tools_from_mcp_servers( + user_api_key_auth: UserAPIKeyAuth | None, + mcp_auth_header: str | None, + mcp_servers: list[str] | None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, + oauth2_headers: dict[str, str] | None = None, + raw_headers: dict[str, str] | None = None, + log_list_tools_to_spendlogs: bool = False, + list_tools_log_source: str | None = None, + litellm_trace_id: str | None = None, + request_tags: list[str] | None = None, + client_ip: str | None = None, + mcp_proxy_mode: bool = False, +) -> AggregateToolListing: + """ + Helper method to fetch tools from MCP servers based on server filtering criteria. + + Args: + user_api_key_auth: User authentication info for access control + mcp_auth_header: Optional auth header for MCP server (deprecated) + mcp_servers: Optional list of server names/aliases to filter by + mcp_server_auth_headers: Optional dict of server-specific auth headers + oauth2_headers: Optional dict of oauth2 headers + + Returns: + AggregateToolListing: Combined tools from filtered servers plus each server's + classified listing outcome + """ + + list_tools_start_time: Final = datetime.now() + litellm_logging_obj: LiteLLMLoggingObj | None = None + list_tools_request_data: dict[str, object] = {} + + if log_list_tools_to_spendlogs: + # This is intentionally minimal: only async_success_handler / post_call_failure_hook + rules_obj: Final = Rules() + list_tools_call_id: Final = str(uuid.uuid4()) + # Derive trace_id from raw_headers when not explicitly passed (same as A2A / MCP call_tool) + effective_litellm_trace_id: Final = litellm_trace_id or get_chain_id_from_headers(raw_headers) + spend_logs_metadata: Final[dict[str, object]] = { + "mcp_operation": "list_tools", + } + if isinstance(list_tools_log_source, str): + spend_logs_metadata["source"] = list_tools_log_source + if isinstance(mcp_servers, list): + spend_logs_metadata["requested_mcp_servers"] = mcp_servers + + list_tools_request_data = { + "model": "MCP: list_tools", + "call_type": CallTypes.list_mcp_tools.value, + "litellm_call_id": list_tools_call_id, + "litellm_trace_id": effective_litellm_trace_id, + "metadata": { + "spend_logs_metadata": spend_logs_metadata, + "headers": logging_safe_mcp_headers(raw_headers), + **({"tags": request_tags} if request_tags else {}), + }, + # Provide a small input payload for standard logging + "input": [ + { + "role": "system", + "content": { + "mcp_operation": "list_tools", + "requested_mcp_servers": mcp_servers, + }, + } + ], + } + + # Attach user identifiers using the standard helper + if user_api_key_auth is not None: + LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( + data=list_tools_request_data, + user_api_key_dict=user_api_key_auth, + _metadata_variable_name="metadata", + ) + + user_identifier: Final = getattr(user_api_key_auth, "end_user_id", None) or getattr( + user_api_key_auth, "user_id", None + ) + if user_identifier: + list_tools_request_data["user"] = user_identifier + + try: + litellm_logging_obj, _ = function_setup( + original_function="list_mcp_tools", + is_async_call=False, + rules_obj=rules_obj, + start_time=list_tools_start_time, + **list_tools_request_data, + ) + if litellm_logging_obj: + litellm_logging_obj.call_type = CallTypes.list_mcp_tools.value + litellm_logging_obj.model = "MCP: list_tools" + except Exception as logging_error: + verbose_logger.debug("Failed to initialize logging for MCP list_tools: %s", logging_error) + litellm_logging_obj = None + + try: + allowed_mcp_servers: Final = await _get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_servers=mcp_servers, + client_ip=client_ip, + ) + if mcp_servers and not allowed_mcp_servers: + await raise_denied_scoped_mcp_access( + requested_names=mcp_servers, + user_api_key_auth=user_api_key_auth, + client_ip=client_ip, + ) + + # Pre-fetch OAuth credentials only when at least one server uses OAuth2, + # to avoid an unnecessary DB round-trip on requests with no OAuth2 MCP servers. + _has_oauth2_server = any(getattr(s, "auth_type", None) == MCPAuth.oauth2 for s in allowed_mcp_servers) + _prefetched_oauth_creds: Final = ( + await _prefetch_oauth_creds_for_user(user_api_key_auth) if _has_oauth2_server else {} + ) + + async def _fetch_and_filter_server_tools( + server: MCPServer, + ) -> "tuple[list[MCPTool], ServerOutcome]": + """Fetch and filter tools from a single server, classifying any failure into that + server's outcome so the aggregate can keep serving the healthy subset without a + broken server masquerading as an empty one.""" + if server is None: + return [], ServerListOk(tool_count=0) + + server_auth_header, extra_headers = _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=mcp_server_auth_headers, + mcp_auth_header=mcp_auth_header, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, + scope_servers=allowed_mcp_servers, + ) + + # Prefer server-stored per-user OAuth when configured, so a stale + # Authorization header from the MCP client cannot override Redis/DB + # (same issue as call_tool in mcp_server_manager: VS Code caches tokens). + from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( # noqa: PLC0415 + to_server_spec, + ) + + # A server migrated to the v2 resolver gets its token from the resolver at connect + # time; building it here would double-resolve and be shadowed by the v2 graft. The + # preemptive 401 already challenged a missing token, so one exists for the connect. + migrated_to_v2: Final = to_server_spec(server) is not None + if ( + not migrated_to_v2 + and server.auth_type == MCPAuth.oauth2 + and getattr(server, "needs_user_oauth_token", False) + and user_api_key_auth is not None + ): + db_headers: Final = await _get_user_oauth_extra_headers_from_db( + server, + user_api_key_auth, + prefetched_creds=_prefetched_oauth_creds, + ) + if db_headers: + extra_headers = db_headers + + # If still no OAuth2 token, fall back to pre-fetched creds (non-stale-client path) + elif not migrated_to_v2 and extra_headers is None and server.auth_type == MCPAuth.oauth2: + extra_headers = await _get_user_oauth_extra_headers_from_db( + server, + user_api_key_auth, + prefetched_creds=_prefetched_oauth_creds, + ) + + if server.is_byok and server.auth_type != MCPAuth.oauth2 and server_auth_header is None: + server_auth_header = await _get_byok_credential(server, user_api_key_auth) + + try: + tools: Final = await global_mcp_server_manager._get_tools_from_server( + server=server, + mcp_auth_header=server_auth_header, + extra_headers=extra_headers, + add_prefix=True, # Always add server prefix + raw_headers=raw_headers, + client_ip=client_ip, + user_api_key_auth=user_api_key_auth, + oauth2_headers=oauth2_headers, + ) + filtered_tools = filter_tools_by_allowed_tools(tools, server) + + filtered_tools = await filter_tools_by_key_team_permissions( + tools=filtered_tools, + server_id=server.server_id, + user_api_key_auth=user_api_key_auth, + ) + + if mcp_proxy_mode: + from litellm.proxy._experimental.mcp_server.tool_search import with_mcp_proxy_identity + + filtered_tools = [ # mutable-ok: MCP tool pipeline + with_mcp_proxy_identity(tool, server.server_id) for tool in filtered_tools + ] + else: + filtered_tools = apply_tool_overrides(filtered_tools, server) + + verbose_logger.debug( + "Successfully fetched %s tools from server %s, %s after filtering", + len(tools), + server.name, + len(filtered_tools), + ) + return filtered_tools, ServerListOk(tool_count=len(filtered_tools)) + except MCPUpstreamAuthError as e: + # Absorb so one unauthenticated server does not empty every other server's + # tools. Surfacing the upstream 401 to the client as a re-auth challenge is + # intentionally not done here: raising from this list handler cannot produce a + # 401 + WWW-Authenticate (the MCP session manager serializes it as a JSON-RPC + # error). Single-server routes surface it via the request-scope preemptive + # check in _raise_preemptive_401_for_unauthenticated_servers instead. + verbose_logger.debug("MCP list_tools: omitting %s; it needs upstream auth", server.name) + return [], classify_list_exception(e) + except Exception as e: + verbose_logger.exception("Error getting tools from server %s: %s", server.name, e) + return [], classify_list_exception(e) + + # Fetch tools from all servers in parallel + tasks: Final = [_fetch_and_filter_server_tools(server) for server in allowed_mcp_servers] + results: Final = await asyncio.gather(*tasks) + + # Flatten results into single list + all_tools: Final[list[MCPTool]] = [tool for tools, _ in results for tool in tools] + server_outcomes: Final[dict[str, ServerOutcome]] = { + _aggregate_server_key(server): outcome + for server, (_, outcome) in zip(allowed_mcp_servers, results) + if server is not None + } + + # If logging is enabled, enrich spend_logs_metadata with counts + if litellm_logging_obj: + per_server_tool_counts: Final[dict[str, int]] = { + _aggregate_server_key(server): len(server_tools) + for server, (server_tools, _) in zip(allowed_mcp_servers, results) + if server is not None + } + + metadata_dict: Final = litellm_logging_obj.model_call_details.get("metadata") + if isinstance(metadata_dict, dict): + spend_meta = metadata_dict.get("spend_logs_metadata") + if not isinstance(spend_meta, dict): + spend_meta = {} + metadata_dict["spend_logs_metadata"] = spend_meta + spend_meta["allowed_server_count"] = len(allowed_mcp_servers) + spend_meta["tool_count_total"] = len(all_tools) + spend_meta["per_server_tool_counts"] = per_server_tool_counts + spend_meta["per_server_list_outcomes"] = { + key: outcome_wire_value(outcome) for key, outcome in server_outcomes.items() + } + + end_time: Final = datetime.now() + try: + await litellm_logging_obj.async_success_handler( + result=[tool.model_dump(mode="json") if isinstance(tool, MCPTool) else tool for tool in all_tools], + start_time=list_tools_start_time, + end_time=end_time, + ) + except Exception as log_exc: + # list_tools responses must not be dropped due to non-blocking + # observability/serialization failures. + verbose_logger.warning( + "MCP list_tools success logging failed (continuing): %s", + log_exc, + ) + + verbose_logger.info("Successfully fetched %s tools total from all MCP servers", len(all_tools)) + + return AggregateToolListing(tools=all_tools, outcomes=server_outcomes) + except Exception as e: + # Only fire failure hook if logging was requested for this list-tools execution + if log_list_tools_to_spendlogs and user_api_key_auth is not None: + try: + from litellm.proxy.proxy_server import proxy_logging_obj + + if proxy_logging_obj: + traceback_str: Final = traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG) + await proxy_logging_obj.post_call_failure_hook( + request_data=list_tools_request_data or {}, + original_exception=e, + user_api_key_dict=user_api_key_auth, + route="/mcp/list_tools", + traceback_str=traceback_str, + ) + except Exception: + verbose_logger.debug("Failed to log MCP list_tools failure via post_call_failure_hook") + raise + + +async def _get_prompts_from_mcp_servers( + user_api_key_auth: UserAPIKeyAuth | None, + mcp_auth_header: str | None, + mcp_servers: list[str] | None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, + oauth2_headers: dict[str, str] | None = None, + raw_headers: dict[str, str] | None = None, + client_ip: str | None = None, +) -> list[Prompt]: + """ + Helper method to fetch prompt from MCP servers based on server filtering criteria. + + Args: + user_api_key_auth: User authentication info for access control + mcp_auth_header: Optional auth header for MCP server (deprecated) + mcp_servers: Optional list of server names/aliases to filter by + mcp_server_auth_headers: Optional dict of server-specific auth headers + oauth2_headers: Optional dict of oauth2 headers + + Returns: + List[Prompt]: Combined list of prompts from filtered servers + """ + + allowed_mcp_servers: Final = await _get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_servers=mcp_servers, + client_ip=client_ip, + ) + + # Get prompts from each allowed server + all_prompts: Final = [] + for server in allowed_mcp_servers: + if server is None: + continue + + server_auth_header, extra_headers = _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=mcp_server_auth_headers, + mcp_auth_header=mcp_auth_header, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, + scope_servers=allowed_mcp_servers, + ) + + try: + prompts = await global_mcp_server_manager.get_prompts_from_server( + server=server, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=server_auth_header, + extra_headers=extra_headers, + add_prefix=True, # Always add server prefix + raw_headers=raw_headers, + client_ip=client_ip, + ) + + all_prompts.extend(prompts) + + verbose_logger.debug("Successfully fetched %s prompts from server %s", len(prompts), server.name) + except Exception as e: + verbose_logger.exception("Error getting prompts from server %s: %s", server.name, e) + # Continue with other servers instead of failing completely + + verbose_logger.info("Successfully fetched %s prompts total from all MCP servers", len(all_prompts)) + + return all_prompts + + +async def _get_resources_from_mcp_servers( + user_api_key_auth: UserAPIKeyAuth | None, + mcp_auth_header: str | None, + mcp_servers: list[str] | None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, + oauth2_headers: dict[str, str] | None = None, + raw_headers: dict[str, str] | None = None, + client_ip: str | None = None, +) -> list[Resource]: + """Fetch resources from allowed MCP servers.""" + + allowed_mcp_servers: Final = await _get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_servers=mcp_servers, + client_ip=client_ip, + ) + + all_resources: Final[list[Resource]] = [] + for server in allowed_mcp_servers: + if server is None: + continue + + server_auth_header, extra_headers = _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=mcp_server_auth_headers, + mcp_auth_header=mcp_auth_header, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, + scope_servers=allowed_mcp_servers, + ) + + try: + resources = await global_mcp_server_manager.get_resources_from_server( + server=server, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=server_auth_header, + extra_headers=extra_headers, + add_prefix=True, # Always add server prefix + raw_headers=raw_headers, + client_ip=client_ip, + ) + all_resources.extend(resources) + + verbose_logger.debug("Successfully fetched %s resources from server %s", len(resources), server.name) + except Exception as e: + verbose_logger.exception("Error getting resources from server %s: %s", server.name, e) + + verbose_logger.info("Successfully fetched %s resources total from all MCP servers", len(all_resources)) + + return all_resources + + +async def _get_resource_templates_from_mcp_servers( + user_api_key_auth: UserAPIKeyAuth | None, + mcp_auth_header: str | None, + mcp_servers: list[str] | None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, + oauth2_headers: dict[str, str] | None = None, + raw_headers: dict[str, str] | None = None, + client_ip: str | None = None, +) -> list[ResourceTemplate]: + """Fetch resource templates from allowed MCP servers.""" + + allowed_mcp_servers: Final = await _get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_servers=mcp_servers, + client_ip=client_ip, + ) + + all_resource_templates: Final[list[ResourceTemplate]] = [] + for server in allowed_mcp_servers: + if server is None: + continue + + server_auth_header, extra_headers = _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=mcp_server_auth_headers, + mcp_auth_header=mcp_auth_header, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, + scope_servers=allowed_mcp_servers, + ) + + try: + resource_templates = await global_mcp_server_manager.get_resource_templates_from_server( + server=server, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=server_auth_header, + extra_headers=extra_headers, + add_prefix=True, # Always add server prefix + raw_headers=raw_headers, + client_ip=client_ip, + ) + all_resource_templates.extend(resource_templates) + verbose_logger.debug( + "Successfully fetched %s resource templates from server %s", + len(resource_templates), + server.name, + ) + except Exception as e: + verbose_logger.exception( + "Error getting resource templates from server %s: %s", + server.name, + str(e), + ) + + verbose_logger.info( + "Successfully fetched %s resource templates total from all MCP servers", + len(all_resource_templates), + ) + + return all_resource_templates + + +async def filter_tools_by_key_team_permissions( + tools: list[MCPTool], + server_id: str, + user_api_key_auth: UserAPIKeyAuth | None, +) -> list[MCPTool]: + """ + Filter tools based on key/team mcp_tool_permissions. + + Note: Tool names in the DB are stored without server prefixes, + but tool names from MCP servers are prefixed. We need to strip + the prefix before comparing. + """ + # Filter by key/team tool-level permissions + allowed_tool_names: Final = await MCPRequestHandler.get_allowed_tools_for_server( + server_id=server_id, + user_api_key_auth=user_api_key_auth, + ) + + # Tools arrive prefixed with the server's own prefix; strip exactly that + # prefix (resolved from the server) rather than the first separator, so a + # prefix containing the separator still reduces to the stored bare name. + server: Final = global_mcp_server_manager.get_mcp_server_by_id(server_id) + return [ + t + for t in tools + if MCPRequestHandler.tool_is_granted(strip_known_server_prefix(t.name, server), allowed_tool_names) + ] + + +async def _list_mcp_tools( + user_api_key_auth: UserAPIKeyAuth | None = None, + mcp_auth_header: str | None = None, + mcp_servers: list[str] | None = None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, + oauth2_headers: dict[str, str] | None = None, + raw_headers: dict[str, str] | None = None, + log_list_tools_to_spendlogs: bool = False, + list_tools_log_source: str | None = None, + client_ip: str | None = None, + mcp_proxy_mode: bool = False, +) -> AggregateToolListing: + """ + List all available MCP tools. + + Args: + user_api_key_auth: User authentication info for access control + mcp_auth_header: Optional auth header for MCP server (deprecated) + mcp_servers: Optional list of server names/aliases to filter by + mcp_server_auth_headers: Optional dict of server-specific auth headers {server_alias: auth_value} + client_ip: Client IP for IP-based server access control + + Returns: + AggregateToolListing: Combined tools from all accessible servers plus each server's + classified listing outcome + """ + + try: + listing: Final = await _get_tools_from_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + log_list_tools_to_spendlogs=log_list_tools_to_spendlogs, + list_tools_log_source=list_tools_log_source, + client_ip=client_ip, + mcp_proxy_mode=mcp_proxy_mode, + ) + verbose_logger.debug("Successfully fetched %s tools from managed MCP servers", len(listing.tools)) + return listing + except HTTPException: + raise + except Exception as e: + verbose_logger.exception("Error getting tools from managed MCP servers: %s", e) + # Continue with an empty listing instead of failing completely + return AggregateToolListing(tools=[], outcomes={}) + + +async def _list_mcp_prompts( + user_api_key_auth: UserAPIKeyAuth | None = None, + mcp_auth_header: str | None = None, + mcp_servers: list[str] | None = None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, + oauth2_headers: dict[str, str] | None = None, + raw_headers: dict[str, str] | None = None, + client_ip: str | None = None, +) -> list[Prompt]: + """ + List all available MCP prompts. + + Args: + user_api_key_auth: User authentication info for access control + mcp_auth_header: Optional auth header for MCP server (deprecated) + mcp_servers: Optional list of server names/aliases to filter by + mcp_server_auth_headers: Optional dict of server-specific auth headers {server_alias: auth_value} + + Returns: + List[Prompt]: Combined list of tools from all accessible servers + """ + # Get tools from managed MCP servers with error handling + managed_prompts = [] + try: + managed_prompts = await _get_prompts_from_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + client_ip=client_ip, + ) + verbose_logger.debug("Successfully fetched %s prompts from managed MCP servers", len(managed_prompts)) + except Exception as e: + verbose_logger.exception("Error getting tools from managed MCP servers: %s", e) + # Continue with empty managed tools list instead of failing completely + + return managed_prompts + + +async def _list_mcp_resources( + user_api_key_auth: UserAPIKeyAuth | None = None, + mcp_auth_header: str | None = None, + mcp_servers: list[str] | None = None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, + oauth2_headers: dict[str, str] | None = None, + raw_headers: dict[str, str] | None = None, + client_ip: str | None = None, +) -> list[Resource]: + """List all available MCP resources.""" + + managed_resources: list[Resource] = [] + try: + managed_resources = await _get_resources_from_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + client_ip=client_ip, + ) + verbose_logger.debug("Successfully fetched %s resources from managed MCP servers", len(managed_resources)) + except Exception as e: + verbose_logger.exception("Error getting resources from managed MCP servers: %s", e) + + return managed_resources + + +async def _list_mcp_resource_templates( + user_api_key_auth: UserAPIKeyAuth | None = None, + mcp_auth_header: str | None = None, + mcp_servers: list[str] | None = None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, + oauth2_headers: dict[str, str] | None = None, + raw_headers: dict[str, str] | None = None, + client_ip: str | None = None, +) -> list[ResourceTemplate]: + """List all available MCP resource templates.""" + + managed_resource_templates: list[ResourceTemplate] = [] + try: + managed_resource_templates = await _get_resource_templates_from_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + client_ip=client_ip, + ) + verbose_logger.debug( + "Successfully fetched %s resource templates from managed MCP servers", + len(managed_resource_templates), + ) + except Exception as e: + verbose_logger.exception( + "Error getting resource templates from managed MCP servers: %s", + str(e), + ) + + return managed_resource_templates + + +def _resolve_display_name_to_original( + name: str, + allowed_mcp_servers: list[MCPServer], +) -> str: + """Translate a display-name override back to the original prefixed tool name. + + When a client received a customised display name from tools/list (e.g. + "Get Pet") it will call tools/call with that same string. We need to + reverse-map it to the original prefixed name (e.g. + "petstore_mcp-getPetById") before any routing or permission logic runs. + """ + for server in allowed_mcp_servers: + display_map = server.tool_name_to_display_name or {} + for unprefixed_name, display_name in display_map.items(): + if display_name == name: + return add_server_prefix_to_name(unprefixed_name, get_server_prefix(server)) + return name + + +async def _get_byok_credential( + mcp_server: MCPServer, + user_api_key_auth: UserAPIKeyAuth | None, +) -> str | None: + """Retrieve the stored BYOK credential for a user+server pair, served from the worker cache within its TTL.""" + if not mcp_server.is_byok: + return None + user_id: Final = (user_api_key_auth.user_id if user_api_key_auth else None) or "" + if not user_id: + return None + + cached: Final = get_cached_byok_credential(user_id, mcp_server.server_id) + if cached is not None: + return cached.credential + + from litellm.proxy._experimental.mcp_server.db import get_user_credential + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + return None + credential: Final = await get_user_credential( + prisma_client=prisma_client, + user_id=user_id, + server_id=mcp_server.server_id, + ) + cache_byok_credential(user_id, mcp_server.server_id, credential) + return credential + + +async def _check_byok_credential( + mcp_server: MCPServer, + user_api_key_auth: UserAPIKeyAuth | None, +) -> None: + """ + If the MCP server is BYOK-enabled, verify that the requesting user has a + stored credential. When no credential is found, raise an HTTP 401 with a + WWW-Authenticate header that points the MCP client to our OAuth metadata + endpoint so it can drive the authorization flow. + """ + if not mcp_server.is_byok: + return + + user_id: Final = (user_api_key_auth.user_id if user_api_key_auth else None) or "" + if not user_id: + raise HTTPException( + status_code=401, + detail={ + "error": "byok_auth_required", + "server_id": mcp_server.server_id, + "server_name": mcp_server.server_name or mcp_server.name, + "message": "User identity is required for BYOK servers", + }, + headers={"WWW-Authenticate": get_byok_www_authenticate()}, + ) + + cached: Final = get_cached_byok_credential(user_id, mcp_server.server_id) + if cached is not None: + if cached.credential is None: + raise HTTPException( + status_code=401, + detail={ + "error": "byok_auth_required", + "server_id": mcp_server.server_id, + "server_name": mcp_server.server_name or mcp_server.name, + "message": ( + "No stored credential found for this BYOK server. " + "Complete the OAuth authorization flow to provide your API key." + ), + }, + headers={"WWW-Authenticate": get_byok_www_authenticate()}, + ) + return + + from litellm.proxy._experimental.mcp_server.db import get_user_credential + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + # Fail closed on DB unavailability: returning here previously + # bypassed the ownership check and let any proxy-authenticated + # caller invoke BYOK tools during outage windows. + raise HTTPException( + status_code=503, + detail={ + "error": "byok_auth_unavailable", + "server_id": mcp_server.server_id, + "server_name": mcp_server.server_name or mcp_server.name, + "message": "BYOK credential check requires a database connection.", + }, + ) + + credential: Final = await get_user_credential( + prisma_client=prisma_client, + user_id=user_id, + server_id=mcp_server.server_id, + ) + cache_byok_credential(user_id, mcp_server.server_id, credential) + if credential is None: + raise HTTPException( + status_code=401, + detail={ + "error": "byok_auth_required", + "server_id": mcp_server.server_id, + "server_name": mcp_server.server_name or mcp_server.name, + "message": ( + "No stored credential found for this BYOK server. " + "Complete the OAuth authorization flow to provide your API key." + ), + }, + headers={"WWW-Authenticate": get_byok_www_authenticate()}, + ) + + +async def _list_tools_before_first_call( + server: MCPServer | None, + tool_name: str, + allowed_mcp_servers: list[MCPServer], + user_api_key_auth: UserAPIKeyAuth | None, + mcp_auth_header: str | None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None, + oauth2_headers: dict[str, str] | None, + raw_headers: dict[str, str] | None, + client_ip: str | None = None, +) -> None: + """List ``server`` with the caller's own credentials when it does not yet expose ``tool_name`` here. + + The startup fill skips a server whose upstream wants the caller's token, and mcp 2 no + longer lists before an uncached tools/call, so a worker that has not served tools/list + for this caller would otherwise answer 404 for a tool the caller can see. Gating on the + requested tool, not on any prior listing, keeps callers with different upstream catalogs + from masking each other. + """ + if server is None or global_mcp_server_manager.server_exposes_tool(server, tool_name): + return + if all(allowed.server_id != server.server_id for allowed in allowed_mcp_servers): + return + try: + await _get_tools_from_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=[server.server_id], + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + client_ip=client_ip, + ) + except Exception as e: # noqa: BLE001 # best effort: resolution below answers as it did before + verbose_logger.debug("MCP tools/call: listing %s before its first call failed: %s", server.name, e) + + +async def execute_mcp_tool( + name: str, + arguments: dict[str, object], + allowed_mcp_servers: list[MCPServer], + start_time: datetime, + user_api_key_auth: UserAPIKeyAuth | None = None, + mcp_auth_header: str | None = None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, + oauth2_headers: dict[str, str] | None = None, + raw_headers: dict[str, str] | None = None, + host_progress_callback: ProgressCallback | None = None, + guardrail_context: Mapping[str, object] | None = None, + client_ip: str | None = None, + **kwargs: object, # kwargs-ok: preserves the existing REST and decorated logging call contract +) -> CallToolResult: + context: Final = prepare_context( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + client_ip=client_ip, + ) + operation: Final = AuthorizedToolCall( + name=name, + arguments=arguments, + allowed_mcp_servers=tuple(allowed_mcp_servers), + start_time=start_time, + host_progress_callback=host_progress_callback, + guardrail_context=guardrail_context, + logging_data=types.MappingProxyType(kwargs), + ) + return await GatewayOperations().execute(operation, context) + + +async def _execute_mcp_tool( + name: str, + arguments: dict[str, object], + allowed_mcp_servers: list[MCPServer], + start_time: datetime, + user_api_key_auth: UserAPIKeyAuth | None = None, + mcp_auth_header: str | None = None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, + oauth2_headers: dict[str, str] | None = None, + raw_headers: dict[str, str] | None = None, + host_progress_callback: ProgressCallback | None = None, + guardrail_context: Mapping[str, object] | None = None, + client_ip: str | None = None, + **kwargs: Any, +) -> CallToolResult: + """ + Execute MCP tool. + + This function assumes permission checks have already been performed. + + Args: + name: Tool name (may include server prefix) + arguments: Tool arguments + allowed_mcp_servers: Pre-validated list of servers the user can access + start_time: Start time for logging + user_api_key_auth: Optional user API key auth for logging + mcp_auth_header: Optional MCP auth header + mcp_server_auth_headers: Optional server-specific auth headers + oauth2_headers: Optional OAuth2 headers + raw_headers: Optional raw HTTP headers + **kwargs: Additional arguments (e.g., litellm_logging_obj) + + Returns: + CallToolResult: Tool execution result + """ + # Track resolved MCP server for both permission checks and dispatch + mcp_server: MCPServer | None = None + requested_server_id: Final[str | None] = kwargs.get("requested_server_id") + + # If the client called with a display-name override (e.g. "Get Pet"), + # translate it back to the original prefixed name before any routing. + name = _resolve_display_name_to_original(name, allowed_mcp_servers) + + # Remove prefix from tool name for logging and processing + original_tool_name, server_name = split_server_prefix_from_name(name) + + requested_server: MCPServer | None = None + if requested_server_id: + requested_server = next( + (s for s in allowed_mcp_servers if s.server_id == requested_server_id), + None, + ) + + name_is_prefixed = False + if requested_server is not None and MCP_TOOL_PREFIX_SEPARATOR in name: + all_registry_prefixes: Final[set[str]] = set() + for registry_server in global_mcp_server_manager.get_registry().values(): + for known_prefix in iter_known_server_prefixes(registry_server): + all_registry_prefixes.add(normalize_server_name(known_prefix)) + name_is_prefixed = is_tool_name_prefixed(name, known_server_prefixes=all_registry_prefixes) + + first_call_target: Final = ( + requested_server + if requested_server is not None and not name_is_prefixed + else global_mcp_server_manager.server_owning_tool_name_prefix(name) + ) + first_call_tool_name: Final = ( + name + if first_call_target is None or (requested_server is not None and not name_is_prefixed) + else strip_known_server_prefix(name, first_call_target) + ) + await _list_tools_before_first_call( + server=first_call_target, + tool_name=first_call_tool_name, + allowed_mcp_servers=allowed_mcp_servers, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + client_ip=client_ip, + ) + + if requested_server is not None and not name_is_prefixed: + # REST callers may pass server_id with the upstream tool name (no + # LiteLLM prefix). The first segment is not a registered server + # prefix, so the whole string is the upstream tool name and may + # legitimately contain the separator (e.g. "text-to-speech"). + # server_id is authoritative for routing and auth. + mcp_server = requested_server + server_name = requested_server.name + original_tool_name = name + else: + # Resolve from tool name (MCP JSON-RPC or prefixed REST tool names). + mcp_server = global_mcp_server_manager._get_mcp_server_from_tool_name(name) + if mcp_server is None and requested_server is not None: + for known_prefix in iter_known_server_prefixes(requested_server): + candidate = global_mcp_server_manager._get_mcp_server_from_tool_name( + add_server_prefix_to_name(name, known_prefix) + ) + if candidate is not None: + mcp_server = candidate + break + if mcp_server is not None: + server_name = mcp_server.name + original_tool_name = strip_known_server_prefix(name, mcp_server) + + if requested_server is not None: + if mcp_server is not None and mcp_server.server_id != requested_server.server_id: + raise HTTPException( + status_code=403, + detail={ + "error": "tool_server_mismatch", + "message": ( + f"Tool '{name}' belongs to MCP server " + f"'{mcp_server.name}' but request specified " + f"server_id for '{requested_server.name}'." + ), + }, + ) + if mcp_server is None: + mcp_server = requested_server + server_name = requested_server.name + original_tool_name = strip_known_server_prefix(name, requested_server) + + # Only enforce server-level permissions when we can resolve a server + if server_name: + if not MCPRequestHandler.is_tool_allowed( + allowed_mcp_servers=[server.name for server in allowed_mcp_servers], + server_name=server_name, + ): + raise HTTPException( + status_code=403, + detail="User not allowed to call this tool.", + ) + + standard_logging_mcp_tool_call: Final[StandardLoggingMCPToolCall] = _get_standard_logging_mcp_tool_call( + name=original_tool_name, # Use original name for logging + arguments=arguments, + server_name=server_name, + session_id=_mcp_session_id_from_headers(raw_headers), + ) + litellm_logging_obj: Final[LiteLLMLoggingObj | None] = kwargs.get("litellm_logging_obj", None) + if litellm_logging_obj: + litellm_logging_obj.model_call_details["mcp_tool_call_metadata"] = standard_logging_mcp_tool_call + litellm_logging_obj.model = f"MCP: {name}" + litellm_logging_obj.model_call_details["model"] = f"MCP: {name}" + # Resolve the MCP server early so BYOK checks and credential injection + # apply to ALL dispatch paths (local tool registry AND managed MCP server). + if mcp_server is None: + mcp_server = global_mcp_server_manager._get_mcp_server_from_tool_name(name) + + if mcp_server: + standard_logging_mcp_tool_call["mcp_server_cost_info"] = (mcp_server.mcp_info or {}).get("mcp_server_cost_info") + if litellm_logging_obj: + litellm_logging_obj.model_call_details["mcp_tool_call_metadata"] = standard_logging_mcp_tool_call + + # BYOK: retrieve the stored per-user credential. A single DB call + # both checks existence and fetches the value, avoiding a double query. + if mcp_server.is_byok and not mcp_auth_header: + byok_cred: Final = await _get_byok_credential(mcp_server, user_api_key_auth) + if byok_cred is None: + raise HTTPException( + status_code=401, + detail={ + "error": "byok_auth_required", + "server_id": mcp_server.server_id, + "server_name": mcp_server.server_name or mcp_server.name, + "message": ( + "No stored credential found for this BYOK server. " + "Complete the OAuth authorization flow to provide your API key." + ), + }, + headers={"WWW-Authenticate": get_byok_www_authenticate()}, + ) + mcp_auth_header = byok_cred + elif mcp_server.is_byok: + # External auth header supplied; still enforce user-identity check. + await _check_byok_credential(mcp_server, user_api_key_auth) + + # Check if tool exists in local registry first (for OpenAPI-based tools) + # These tools are registered with their prefixed names + ######################################################### + local_tool: Final = global_mcp_tool_registry.get_tool(name) + if local_tool: + # OpenAPI-backed tools used to bypass `pre_call_tool_check` — + # only the managed path ran allowed/banned-tool checks, key/team + # tool permissions, and parameter validation. Run the same checks + # before dispatching to the local registry. Refuse the call if + # we cannot resolve a server: tools registered via + # openapi_to_mcp_generator are always tied to a server, so a + # missing mcp_server here means the tool->server mapping has + # not finished initializing or the registry entry is orphaned. + # Skipping the check would re-open the same authorization gap. + if mcp_server is None: + raise HTTPException( + status_code=503, + detail=( + f"MCP server for tool '{name}' is not available; " + "refusing to dispatch without authorization checks. " + "Retry once the server is registered." + ), + ) + + # `pre_call_tool_check` calls into `proxy_logging_obj` for the + # pre-call guardrail hooks, so source it from the canonical + # `proxy_server` module the same way `_handle_managed_mcp_tool` + # does. `kwargs.get("proxy_logging_obj")` is None on the MCP + # entry path and would crash with AttributeError after the + # security checks pass. + from litellm.proxy.proxy_server import proxy_logging_obj + + hook_result = await global_mcp_server_manager.pre_call_tool_check( + name=original_tool_name, + arguments=arguments or {}, + server_name=server_name or mcp_server.name, + user_api_key_auth=user_api_key_auth, + proxy_logging_obj=proxy_logging_obj, + server=mcp_server, + raw_headers=raw_headers, + litellm_logging_obj=litellm_logging_obj, + guardrail_context=guardrail_context, + ) + # `pre_call_tool_check` may return guardrail-modified + # arguments; honor them on the local path too. + if isinstance(hook_result, dict) and "arguments" in hook_result: + arguments = hook_result["arguments"] + + verbose_logger.debug("Executing local registry tool: %s", name) + # The credential rides ContextVars because the tool function has its + # headers baked into the closure at registration time. + auth_header_value, openapi_forwarded_headers, upstream_credential = _resolve_openapi_tool_auth( + mcp_server=mcp_server, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, + ) + ( + resolved_auth_headers, + forwarded_headers, + ) = await global_mcp_server_manager.resolve_openapi_upstream_auth( + mcp_server=mcp_server, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + mcp_auth_header=upstream_credential, + user_api_key_auth=user_api_key_auth, + forwarded_headers=openapi_forwarded_headers, + ) + + _auth_token: Final = _request_auth_header.set(auth_header_value) + _extra_token: Final = _request_extra_headers.set(forwarded_headers) + _resolved_token: Final = _request_resolved_auth_headers.set(resolved_auth_headers) + try: + response = await _handle_local_mcp_tool(name, arguments) + finally: + _request_auth_header.reset(_auth_token) + _request_extra_headers.reset(_extra_token) + _request_resolved_auth_headers.reset(_resolved_token) + + # Try managed MCP server tool (the name is bare; the prefix boundary was + # already resolved above against this server's registered prefixes) + # Primary and recommended way to use external MCP servers + ######################################################### + elif mcp_server: + response = await _handle_managed_mcp_tool( + server_name=server_name, + name=original_tool_name, + arguments=arguments, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + client_ip=client_ip, + litellm_logging_obj=litellm_logging_obj, + guardrail_context=guardrail_context, + host_progress_callback=host_progress_callback, + ) + + # Fall back to local tool registry with original name (legacy support) + ######################################################### + # Deprecated: Local MCP Server Tool + ######################################################### + else: + # Gate only what can actually dispatch. When the unprefixed name is + # not in the registry either, `_handle_local_mcp_tool` below reports + # 404 and nothing runs, so demanding a server here would turn every + # unknown tool name into a misleading 503. + if global_mcp_tool_registry.get_tool(original_tool_name) is not None: + # `mcp_server` is None here because the tool name is not in the + # tool -> server mapping, but the name still carries a prefix + # that the server-level check above compared against the + # caller's `allowed_mcp_servers` by exact `name`. So the named + # server is in that list and can carry the tool-level checks, + # even with the mapping cold. Resolve it from + # `allowed_mcp_servers` rather than the registry: the registry + # would happily return a server the caller holds no grant for, + # and matching anything other than `name` would accept a server + # the check never validated. + prefix_server: Final = next( + (candidate for candidate in allowed_mcp_servers if candidate.name == server_name), + None, + ) + if prefix_server is None: + # A non-empty prefix that passed the server-level check + # always matches here, so this arm only fires when the + # prefix was empty, which is exactly the case that check + # skips. Fail closed rather than dispatch with no server to + # evaluate a tool ceiling against. + raise HTTPException( + status_code=503, + detail=( + f"MCP server for tool '{original_tool_name}' is not available; " + "refusing to dispatch without authorization checks. " + "Retry once the server is registered." + ), + ) + + from litellm.proxy.proxy_server import proxy_logging_obj + + hook_result = await global_mcp_server_manager.pre_call_tool_check( + name=original_tool_name, + arguments=arguments, + server_name=server_name, + user_api_key_auth=user_api_key_auth, + proxy_logging_obj=proxy_logging_obj, + server=prefix_server, + raw_headers=raw_headers, + litellm_logging_obj=litellm_logging_obj, + guardrail_context=guardrail_context, + ) + if "arguments" in hook_result: + arguments = hook_result["arguments"] # pyright: ignore[reportAny] # hook returns untyped args + + response = await _handle_local_mcp_tool(original_tool_name, arguments) + + return await _run_post_mcp_call_guardrails( + result=response, + litellm_logging_obj=litellm_logging_obj, + user_api_key_auth=user_api_key_auth, + request_data=kwargs, + ) + + +async def _run_post_mcp_call_guardrails( + result: CallToolResult, + litellm_logging_obj: LiteLLMLoggingObj | None, + user_api_key_auth: UserAPIKeyAuth | None, + request_data: Mapping[str, object], +) -> CallToolResult: + """Run ``post_mcp_call`` guardrails over an executed tool result. + + Lives on ``execute_mcp_tool``'s return path rather than inside + ``_fire_mcp_tool_call_logging`` so enforcement never depends on logging + being configured, and so every dispatch route gets it: the MCP protocol + handler, the REST endpoint, and tool search all funnel through here. + A guardrail that rejects the result raises, matching ``pre_mcp_call``. + """ + from litellm.proxy.proxy_server import proxy_logging_obj + + if proxy_logging_obj is None: + return result + return await proxy_logging_obj.post_mcp_call_hook( + response=result, + request_data=( + litellm_logging_obj.model_call_details if litellm_logging_obj is not None else dict(request_data) + ), + user_api_key_dict=user_api_key_auth, + ) + + +async def _fire_mcp_tool_call_logging( + logging_obj: LiteLLMLoggingObj, + result: CallToolResult, + start_time: datetime, + end_time: datetime, + user_api_key_auth: UserAPIKeyAuth | None = None, + request_data: Mapping[str, object] | None = None, +) -> CallToolResult: + """Fire post-call logging for an executed MCP tool call, returning the result to send. + + The returned result is what the caller must forward to the client: a + ``post_mcp_call`` guardrail may rewrite the tool output (e.g. mask + sensitive values) or reject it, in which case its exception propagates. + Guardrails run before the success/failure logging so the masked text, not + the raw one, is what gets logged. + + A result with ``is_error=True`` is logged as a failure (``status="failure"`` + payload, so OTel marks the span ERROR) while the HTTP wire behavior stays + 200 + ``isError: true`` per the MCP spec. The error check runs after + ``async_post_mcp_tool_call_hook`` because guardrails may flip the result + to ``is_error=True`` in that hook. Raised exceptions never reach here (the + ``@client`` wrapper and ``call_mcp_tool``'s except path log those), so + this cannot double-log a failure. + + ``request_data`` may carry credential-bearing fields (the REST path puts + ``raw_headers``, ``mcp_auth_header``, ``mcp_server_auth_headers``, and + ``oauth2_headers`` at the top level of its data dict), so those are + stripped before the dict is handed to ``post_call_failure_hook`` + callbacks. + """ + from litellm.proxy.proxy_server import proxy_logging_obj + + logging_obj.post_call(original_response=result) + await logging_obj.async_post_mcp_tool_call_hook( + kwargs=logging_obj.model_call_details, + response_obj=result, + start_time=start_time, + end_time=end_time, + ) + logging_obj.call_type = CallTypes.call_mcp_tool.value + error_message: Final = extract_mcp_tool_result_error_message(result) + if error_message is None: + await logging_obj.async_success_handler(result=result, start_time=start_time, end_time=end_time) + return result + + logging_obj.has_run_logging(event_type="sync_success") + logging_obj.has_run_logging(event_type="async_success") + tool_error: Final = MCPToolResultError(error_message) + logging_obj.failure_handler(tool_error, "", start_time, end_time) + await logging_obj.async_failure_handler(tool_error, "", start_time, end_time) + + if user_api_key_auth is None: + return result + + if proxy_logging_obj: + sanitized_request_data: Final = { + key: value for key, value in (request_data or {}).items() if key not in _MCP_CREDENTIAL_REQUEST_FIELDS + } + await proxy_logging_obj.post_call_failure_hook( + request_data=sanitized_request_data, + original_exception=tool_error, + user_api_key_dict=user_api_key_auth, + route="/mcp/call_tool", + ) + return result + + +async def fire_mcp_tool_call_failure_logging( + logging_obj: LiteLLMLoggingObj | None, + exception: Exception, + start_time: datetime, + user_api_key_auth: UserAPIKeyAuth | None, + request_data: Mapping[str, object], +) -> None: + """Failure logging shared by the ``/mcp`` path and the REST endpoint. Call from + inside the ``except`` block so the traceback is still available. + + The failure handlers run first because ``_ProxyDBLogger.async_post_call_failure_hook`` + builds the failure spend-log row from the ``standard_logging_object`` they produce; + both gate on ``should_run_logging``, so the ``@client`` wrapper does not log twice. + A relayed upstream 401 (``MCPUpstreamAuthError``) is an expected caller-must-reauth + signal and skips ``post_call_failure_hook``, which fires the ``llm_exceptions`` alert. + """ + from litellm.proxy.proxy_server import proxy_logging_obj + + traceback_str: Final = traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG) + if logging_obj is not None: + end_time: Final = datetime.now() # noqa: DTZ005 # naive to match `start_time`, which it is subtracted from + logging_obj.failure_handler(exception, traceback_str, start_time, end_time) + await logging_obj.async_failure_handler(exception, traceback_str, start_time, end_time) + + if isinstance(exception, MCPUpstreamAuthError) or not proxy_logging_obj or user_api_key_auth is None: + return + sanitized_request_data: Final = { + key: value for key, value in request_data.items() if key not in _MCP_CREDENTIAL_REQUEST_FIELDS + } + await proxy_logging_obj.post_call_failure_hook( + request_data=sanitized_request_data, + original_exception=exception, + user_api_key_dict=user_api_key_auth, + route="/mcp/call_tool", + traceback_str=traceback_str, + ) + + +@client +async def call_mcp_tool( + name: str, + arguments: dict[str, object] | None = None, + user_api_key_auth: UserAPIKeyAuth | None = None, + mcp_auth_header: str | None = None, + mcp_servers: list[str] | None = None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, + oauth2_headers: dict[str, str] | None = None, + raw_headers: dict[str, str] | None = None, + client_ip: str | None = None, + **kwargs: Any, +) -> CallToolResult: + """ + Call a specific tool with the provided arguments (handles prefixed tool names). + """ + start_time: Final = datetime.now() + litellm_logging_obj: Final[LiteLLMLoggingObj | None] = kwargs.get("litellm_logging_obj", None) + + try: + if arguments is None: + raise HTTPException(status_code=400, detail="Request arguments are required") + + ## CHECK IF USER IS ALLOWED TO CALL THIS TOOL + allowed_mcp_server_ids: Final = await global_mcp_server_manager.get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth, + ) + + allowed_mcp_servers: list[MCPServer] = [] + for allowed_mcp_server_id in allowed_mcp_server_ids: + allowed_server = global_mcp_server_manager.get_mcp_server_by_id(allowed_mcp_server_id) + if allowed_server is not None: + # Same request-time oauth2_flow backstop the listing path applies, + # so a null-flow M2M-shape row is treated as M2M on tool calls too. + allowed_server = MCPServerManager.resolve_oauth2_flow_for_request(allowed_server) + allowed_mcp_servers.append(allowed_server) + + allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names( + mcp_servers=mcp_servers, + allowed_mcp_servers=allowed_mcp_servers, + ) + if mcp_servers and not allowed_mcp_servers: + await raise_denied_scoped_mcp_access( + requested_names=mcp_servers, + user_api_key_auth=user_api_key_auth, + client_ip=client_ip, + ) + if not allowed_mcp_servers: + raise HTTPException( + status_code=403, + detail="User not allowed to call this tool.", + ) + + # Delegate to execute_mcp_tool for execution + response = await execute_mcp_tool( + name=name, + arguments=arguments, + allowed_mcp_servers=allowed_mcp_servers, + start_time=start_time, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + client_ip=client_ip, + **kwargs, + ) + except Exception as e: + await fire_mcp_tool_call_failure_logging(litellm_logging_obj, e, start_time, user_api_key_auth, kwargs) + raise + + if litellm_logging_obj: + response = await _fire_mcp_tool_call_logging( + logging_obj=litellm_logging_obj, + result=response, + start_time=start_time, + end_time=datetime.now(), + user_api_key_auth=user_api_key_auth, + request_data=kwargs, + ) + return response + + +async def mcp_get_prompt( + name: str, + arguments: dict[str, str] | None = None, + user_api_key_auth: UserAPIKeyAuth | None = None, + mcp_auth_header: str | None = None, + mcp_servers: list[str] | None = None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, + oauth2_headers: dict[str, str] | None = None, + raw_headers: dict[str, str] | None = None, + client_ip: str | None = None, +) -> GetPromptResult: + """ + Fetch a specific MCP prompt, handling both prefixed and unprefixed names. + """ + allowed_mcp_servers: Final = await _get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_servers=mcp_servers, + client_ip=client_ip, + ) + + if not allowed_mcp_servers: + raise HTTPException( + status_code=403, + detail="User not allowed to get this prompt.", + ) + + # Extract server name from prefixed prompt name + original_prompt_name, server_name = split_server_prefix_from_name(name) + + server: Final = next((s for s in allowed_mcp_servers if s.name == server_name), None) + if server is None: + raise HTTPException( + status_code=403, + detail="User not allowed to get this prompt.", + ) + + server_auth_header, extra_headers = _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=mcp_server_auth_headers, + mcp_auth_header=mcp_auth_header, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, + ) + + return await global_mcp_server_manager.get_prompt_from_server( + server=server, + user_api_key_auth=user_api_key_auth, + prompt_name=original_prompt_name, + arguments=arguments, + mcp_auth_header=server_auth_header, + extra_headers=extra_headers, + raw_headers=raw_headers, + client_ip=client_ip, + ) + + +async def mcp_read_resource( + url: AnyUrl, + user_api_key_auth: UserAPIKeyAuth | None = None, + mcp_auth_header: str | None = None, + mcp_servers: list[str] | None = None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, + oauth2_headers: dict[str, str] | None = None, + raw_headers: dict[str, str] | None = None, + client_ip: str | None = None, +) -> ReadResourceResult: + """Read resource contents from upstream MCP servers.""" + + allowed_mcp_servers: Final = await _get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth, + mcp_servers=mcp_servers, + client_ip=client_ip, + ) + + if not allowed_mcp_servers: + raise HTTPException( + status_code=403, + detail="User not allowed to read this resource.", + ) + + if len(allowed_mcp_servers) != 1: + raise HTTPException( + status_code=400, + detail=("Multiple MCP servers configured; read_resource currently supports exactly one allowed server."), + ) + + server: Final = allowed_mcp_servers[0] + + server_auth_header, extra_headers = _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=mcp_server_auth_headers, + mcp_auth_header=mcp_auth_header, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, + ) + + return await global_mcp_server_manager.read_resource_from_server( + server=server, + user_api_key_auth=user_api_key_auth, + url=url, + mcp_auth_header=server_auth_header, + extra_headers=extra_headers, + raw_headers=raw_headers, + client_ip=client_ip, + ) + + +def _get_standard_logging_mcp_tool_call( + name: str, + arguments: dict[str, object], + server_name: str | None, + session_id: str | None = None, +) -> StandardLoggingMCPToolCall: + mcp_server: Final = global_mcp_server_manager._get_mcp_server_from_tool_name( + add_server_prefix_to_name(name, server_name) if server_name else name + ) + namespaced_tool_name: Final = f"{server_name}/{name}" if server_name else name + if mcp_server: + mcp_info: Final = mcp_server.mcp_info or {} + return StandardLoggingMCPToolCall( + name=name, + arguments=arguments, + mcp_server_name=mcp_info.get("server_name"), + mcp_server_logo_url=mcp_info.get("logo_url"), + namespaced_tool_name=namespaced_tool_name, + mcp_session_id=session_id, + mcp_auth_mode=mcp_server.auth_type, + mcp_server_resource=_redact_mcp_resource_url(mcp_server.url), + ) + else: + return StandardLoggingMCPToolCall( + name=name, + arguments=arguments, + namespaced_tool_name=namespaced_tool_name, + mcp_session_id=session_id, + ) + + +async def _handle_managed_mcp_tool( + server_name: str, + name: str, + arguments: dict[str, object], + user_api_key_auth: UserAPIKeyAuth | None = None, + mcp_auth_header: str | None = None, + mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, + oauth2_headers: dict[str, str] | None = None, + raw_headers: dict[str, str] | None = None, + litellm_logging_obj: LiteLLMLoggingObj | None = None, + host_progress_callback: ProgressCallback | None = None, + guardrail_context: Mapping[str, object] | None = None, + client_ip: str | None = None, +) -> CallToolResult: + """Handle tool execution for managed server tools""" + # Import here to avoid circular import + from litellm.proxy.proxy_server import proxy_logging_obj + + call_tool_result: Final = await global_mcp_server_manager.call_tool( + server_name=server_name, + name=name, + arguments=arguments, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + client_ip=client_ip, + proxy_logging_obj=proxy_logging_obj, + host_progress_callback=host_progress_callback, + litellm_logging_obj=litellm_logging_obj, + guardrail_context=guardrail_context, + ) + verbose_logger.debug("CALL TOOL RESULT: %s", call_tool_result) + return call_tool_result + + +async def _handle_local_mcp_tool(name: str, arguments: dict[str, object]) -> CallToolResult: + """Execute a local-registry tool and report whether it succeeded. + + Returns the result rather than bare content because the verdict is part of it: the content + alone cannot say whether the handler failed, so callers used to stamp is_error=False on every + outcome and an upstream rejection was served as tool output. + + A failure is reported as ``is_error=True`` here rather than raised, because the REST surface + turns an unrecognized exception into a 500 and an upstream 403 or 429 is not a gateway crash. + ``MCPUpstreamAuthError`` is the exception: it propagates so the caller is told to + re-authenticate, which both renderers already know how to say. + + Note: Local tools don't use prefixes, so we use the original name + """ + import inspect + + tool: Final = global_mcp_tool_registry.get_tool(name) + if not tool: + raise HTTPException(status_code=404, detail=f"Tool '{name}' not found") + + try: + if inspect.iscoroutinefunction(tool.handler): + result = await tool.handler(**arguments) + else: + result = tool.handler(**arguments) + except MCPUpstreamAuthError: + raise + except Exception as e: + verbose_logger.exception("Error executing local tool %s: %s", name, e) + return CallToolResult( + content=[TextContent(text=f"Error: {e}", type="text")], # mutable-ok: MCP result content + is_error=True, + ) + return CallToolResult( + content=[TextContent(text=str(result), type="text")], # mutable-ok: MCP result content + is_error=False, + ) + + +_MCP_CREDENTIAL_REQUEST_FIELDS: Final = frozenset( + { + "raw_headers", + "mcp_auth_header", + "mcp_server_auth_headers", + "oauth2_headers", + "user_api_key_auth", + } +) + + +class _McpDeniedDetail(TypedDict): + error: ReadOnly[str] + + +async def _execute_handle_list_tools( + context: OperationContext, params: PaginatedRequestParams, host_progress_callback: ProgressCallback | None = None +) -> ListToolsResult: + try: + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + _client_ip, + ) = context.legacy_auth() + verbose_logger.debug("MCP list_tools - User API Key Auth from context: %s", user_api_key_auth) + verbose_logger.debug("MCP list_tools - MCP servers from context: %s", mcp_servers) + verbose_logger.debug( + "MCP list_tools - MCP server auth headers: %s", + list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None, + ) + from mcp.types import Tool + + from litellm.proxy._experimental.mcp_server.tool_search import ( + get_mcp_proxy_tool_definitions, + get_virtual_tool_definitions, + ) + + if context.mcp_proxy_mode: + return ListToolsResult(tools=[Tool.model_validate(d) for d in get_mcp_proxy_tool_definitions()]) + if getattr( + getattr(user_api_key_auth, "object_permission", None), + "mcp_tool_search_enabled", + False, + ): + return ListToolsResult(tools=[Tool.model_validate(d) for d in get_virtual_tool_definitions()]) + + # Get mcp_servers from context variable + verbose_logger.debug("MCP list_tools - Calling _list_mcp_tools") + listing: Final = await _list_mcp_tools( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + log_list_tools_to_spendlogs=True, + list_tools_log_source="mcp_protocol", + client_ip=_client_ip, + ) + verbose_logger.info("MCP list_tools - Successfully returned %s tools", len(listing.tools)) + if not listing.outcomes: + return ListToolsResult(tools=listing.tools) + outcome_meta: Final = { + SERVER_OUTCOMES_META_KEY: {key: outcome_wire_value(outcome) for key, outcome in listing.outcomes.items()} + } + return ListToolsResult.model_validate({"tools": listing.tools, "_meta": outcome_meta}) + except HTTPException as e: + from mcp.shared.exceptions import MCPError + from mcp.types import INVALID_REQUEST + + raise MCPError(code=INVALID_REQUEST, message=_http_detail_message(e.detail)) from e + except Exception as e: + verbose_logger.exception("Error in list_tools endpoint: %s", e) + # Return empty list instead of failing completely + # This prevents the HTTP stream from failing and allows the client to get a response + return ListToolsResult(tools=[]) # mutable-ok: MCP result payload + + +async def _execute_mcp_server_tool_call( + context: OperationContext, params: CallToolRequestParams, host_progress_callback: ProgressCallback | None = None +) -> CallToolResult: + from mcp.types import CallToolResult + + from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + from litellm.proxy.proxy_server import proxy_config + + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + _client_ip, + ) = context.legacy_auth() + verbose_logger.debug( + "MCP mcp_server_tool_call - user_api_key_auth=%s, user_role=%s", + user_api_key_auth, + getattr(user_api_key_auth, "user_role", "N/A"), + ) + + verbose_logger.debug("MCP mcp_server_tool_call - User API Key Auth from context: %s", user_api_key_auth) + + try: + # Inside this try so virtual-tool errors convert to isError + # CallToolResult instead of raising out of the protocol handler. + virtual_tool_result: Final = await _dispatch_virtual_mcp_tool( + name=params.name, + arguments=params.arguments, + user_api_key_auth=user_api_key_auth, + client_ip=_client_ip, + mcp_servers=mcp_servers, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + mcp_proxy_mode=context.mcp_proxy_mode, + ) + if virtual_tool_result is not None: + return virtual_tool_result + + # Create a body date for logging + body_data: Final = {"name": params.name, "arguments": params.arguments} # mutable-ok: logging payload + # Set trace/session id from raw_headers so spend logs and logging_obj stay consistent (same as A2A) + chain_id: Final = get_chain_id_from_headers(raw_headers) + if chain_id: + body_data["litellm_trace_id"] = chain_id + body_data["litellm_session_id"] = chain_id + + request: Final = build_synthetic_mcp_request( + path="/mcp/tools/call", + raw_headers=raw_headers, + client_ip=_client_ip, + ) + if user_api_key_auth is not None: + data = await add_litellm_data_to_request( + data=body_data, + request=request, + # Bill a team-derived call to the team that granted it. A keyless admitted + # subject carries no team_id, so spend skipped team updates entirely and + # charged the user's PRIMARY org — the granting team's budget never + # accumulated (so it could never begin to block) and, cross-org, the wrong + # organization was charged. This is the ACCOUNTING half; the enforcement + # half (an already-over-budget team stops granting) lives in the source gate. + # Authorization is unaffected: it ran before this, and the union is resolved + # from the untouched auth object passed to call_mcp_tool below. + user_api_key_dict=await MCPRequestHandler.billing_auth_for_tool_call( + user_api_key_auth, tool_name=params.name + ), + proxy_config=proxy_config, + ) + else: + data = body_data + + response: Final = await call_mcp_tool( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + client_ip=_client_ip, + host_progress_callback=host_progress_callback, + **data, # for logging + ) + except MCPMissingUserEnvVarsError as e: + verbose_logger.info( + "MCP mcp_server_tool_call missing per-user env vars: server_id=%s missing=%s", + e.server_id, + e.missing, + ) + return CallToolResult( + content=[TextContent(text=str(e), type="text")], + is_error=True, + ) + except BlockedPiiEntityError as e: + verbose_logger.error("BlockedPiiEntityError in MCP tool call: %s", e) + return CallToolResult( + content=[ + TextContent( + text=f"Error: Blocked PII entity detected - {e}", + type="text", + ) + ], + is_error=True, + ) + except GuardrailRaisedException as e: + verbose_logger.error("GuardrailRaisedException in MCP tool call: %s", e) + return CallToolResult( + content=[TextContent(text=f"Error: Guardrail violation - {e}", type="text")], + is_error=True, + ) + except HTTPException as e: + verbose_logger.error("HTTPException in MCP tool call: %s", e) + return CallToolResult( + content=[TextContent(text=f"Error: {_http_detail_message(e.detail)}", type="text")], + is_error=True, + ) + except MCPUpstreamAuthError as e: + # The MCP session manager serializes handler exceptions as JSON-RPC errors, so a + # mid-session tool call cannot emit a raw 401 + WWW-Authenticate the way the REST + # call path and the connect-time preemptive check do. Return an explicit isError + # naming the upstream status (at info level, not a traceback) so the client still + # learns it must re-authenticate upstream and expected pass-through 401s don't spam. + verbose_logger.info("Upstream auth failure calling MCP tool: HTTP %s", e.status_code) + return CallToolResult( + content=[ + TextContent( + text=f"Error: upstream authentication required (HTTP {e.status_code})", + type="text", + ) + ], + is_error=True, + ) + except Exception as e: + verbose_logger.exception("MCP mcp_server_tool_call - error: %s", e) + return CallToolResult( + content=[TextContent(text=f"Error: {e}", type="text")], + is_error=True, + ) + + return response + + +async def _execute_list_prompts( + context: OperationContext, params: PaginatedRequestParams, host_progress_callback: ProgressCallback | None = None +) -> ListPromptsResult: + if context.mcp_proxy_mode: + _reject_mcp_proxy_operation() + try: + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + _client_ip, + ) = context.legacy_auth() + verbose_logger.debug("MCP list_prompts - User API Key Auth from context: %s", user_api_key_auth) + verbose_logger.debug("MCP list_prompts - MCP servers from context: %s", mcp_servers) + verbose_logger.debug( + "MCP list_prompts - MCP server auth headers: %s", + list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None, + ) + # Get mcp_servers from context variable + verbose_logger.debug("MCP list_prompts - Calling _list_prompts") + prompts: Final = await _list_mcp_prompts( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + client_ip=_client_ip, + ) + verbose_logger.info("MCP list_prompts - Successfully returned %s prompts", len(prompts)) + return ListPromptsResult(prompts=prompts) + except Exception as e: + verbose_logger.exception("Error in list_prompts endpoint: %s", e) + # Return empty list instead of failing completely + # This prevents the HTTP stream from failing and allows the client to get a response + return ListPromptsResult(prompts=[]) # mutable-ok: MCP result payload + + +async def _execute_get_prompt( + context: OperationContext, params: GetPromptRequestParams, host_progress_callback: ProgressCallback | None = None +) -> GetPromptResult: + if context.mcp_proxy_mode: + _reject_mcp_proxy_operation() + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + _client_ip, + ) = context.legacy_auth() + + verbose_logger.debug("MCP mcp_server_tool_call - User API Key Auth from context: %s", user_api_key_auth) + return await mcp_get_prompt( + name=params.name, + arguments=params.arguments, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + client_ip=_client_ip, + ) + + +async def _execute_list_resources( + context: OperationContext, params: PaginatedRequestParams, host_progress_callback: ProgressCallback | None = None +) -> ListResourcesResult: + if context.mcp_proxy_mode: + _reject_mcp_proxy_operation() + try: + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + _client_ip, + ) = context.legacy_auth() + verbose_logger.debug("MCP list_resources - User API Key Auth from context: %s", user_api_key_auth) + verbose_logger.debug("MCP list_resources - MCP servers from context: %s", mcp_servers) + verbose_logger.debug( + "MCP list_resources - MCP server auth headers: %s", + list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None, + ) + + resources: Final = await _list_mcp_resources( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + client_ip=_client_ip, + ) + verbose_logger.info("MCP list_resources - Successfully returned %s resources", len(resources)) + return ListResourcesResult(resources=resources) + except Exception as e: + verbose_logger.exception("Error in list_resources endpoint: %s", e) + return ListResourcesResult(resources=[]) # mutable-ok: MCP result payload + + +async def _execute_list_resource_templates( + context: OperationContext, params: PaginatedRequestParams, host_progress_callback: ProgressCallback | None = None +) -> ListResourceTemplatesResult: + if context.mcp_proxy_mode: + _reject_mcp_proxy_operation() + try: + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + _client_ip, + ) = context.legacy_auth() + verbose_logger.debug("MCP list_resource_templates - User API Key Auth from context: %s", user_api_key_auth) + verbose_logger.debug("MCP list_resource_templates - MCP servers from context: %s", mcp_servers) + verbose_logger.debug( + "MCP list_resource_templates - MCP server auth headers: %s", + list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None, + ) + + resource_templates: Final = await _list_mcp_resource_templates( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + client_ip=_client_ip, + ) + verbose_logger.info( + "MCP list_resource_templates - Successfully returned %s resource templates", len(resource_templates) + ) + return ListResourceTemplatesResult(resource_templates=resource_templates) + except Exception as e: + verbose_logger.exception("Error in list_resource_templates endpoint: %s", e) + return ListResourceTemplatesResult(resource_templates=[]) # mutable-ok: MCP result payload + + +async def _execute_read_resource( + context: OperationContext, params: ReadResourceRequestParams, host_progress_callback: ProgressCallback | None = None +) -> ReadResourceResult: + if context.mcp_proxy_mode: + _reject_mcp_proxy_operation() + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + _client_ip, + ) = context.legacy_auth() + + read_resource_result: Final = await mcp_read_resource( + url=params.uri, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + client_ip=_client_ip, + ) + + return read_resource_result + + +def _reject_mcp_proxy_operation() -> NoReturn: + from mcp.shared.exceptions import MCPError + from mcp.types import METHOD_NOT_FOUND + + raise MCPError(code=METHOD_NOT_FOUND, message="Operation unavailable on /mcp/proxy") + + +def prepare_context( + user_api_key_auth: UserAPIKeyAuth | None = None, + mcp_auth_header: str | None = None, + mcp_servers: Sequence[str] | None = None, + mcp_server_auth_headers: Mapping[str, Mapping[str, str]] | None = None, + oauth2_headers: Mapping[str, str] | None = None, + raw_headers: Mapping[str, str] | None = None, + client_ip: str | None = None, + mcp_proxy_mode: bool = False, +) -> OperationContext: + return OperationContext( + _caller=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=tuple(mcp_servers) if mcp_servers is not None else None, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + client_ip=client_ip, + mcp_proxy_mode=mcp_proxy_mode, + ) + + +GatewayOperation: TypeAlias = ( + AuthorizedToolCall + | ListToolsRequest + | CallToolRequest + | ListPromptsRequest + | GetPromptRequest + | ListResourcesRequest + | ListResourceTemplatesRequest + | ReadResourceRequest +) +GatewayResult: TypeAlias = ( + ListToolsResult + | CallToolResult + | ListPromptsResult + | GetPromptResult + | ListResourcesResult + | ListResourceTemplatesResult + | ReadResourceResult +) + + +class GatewayOperations: + def __init__(self, host_progress_callback: ProgressCallback | None = None) -> None: + self._host_progress_callback = host_progress_callback + + @overload + async def execute(self, operation: AuthorizedToolCall, context: OperationContext) -> CallToolResult: ... + + @overload + async def execute(self, operation: ListToolsRequest, context: OperationContext) -> ListToolsResult: ... + + @overload + async def execute(self, operation: CallToolRequest, context: OperationContext) -> CallToolResult: ... + + @overload + async def execute(self, operation: ListPromptsRequest, context: OperationContext) -> ListPromptsResult: ... + + @overload + async def execute(self, operation: GetPromptRequest, context: OperationContext) -> GetPromptResult: ... + + @overload + async def execute(self, operation: ListResourcesRequest, context: OperationContext) -> ListResourcesResult: ... + + @overload + async def execute( + self, operation: ListResourceTemplatesRequest, context: OperationContext + ) -> ListResourceTemplatesResult: ... + + @overload + async def execute(self, operation: ReadResourceRequest, context: OperationContext) -> ReadResourceResult: ... + + async def execute(self, operation: GatewayOperation, context: OperationContext) -> GatewayResult: + match operation: + case AuthorizedToolCall(): + auth, token, _servers, server_headers, oauth_headers, headers, _client_ip = context.legacy_auth() + return await _execute_mcp_tool( + name=operation.name, + arguments=dict(operation.arguments), # mutable-ok: existing tool hooks own mutable argument data + allowed_mcp_servers=list( + operation.allowed_mcp_servers + ), # mutable-ok: legacy dispatch list contract + start_time=operation.start_time, + user_api_key_auth=auth, + mcp_auth_header=token, + mcp_server_auth_headers=server_headers, + oauth2_headers=oauth_headers, + raw_headers=headers, + client_ip=_client_ip, + host_progress_callback=operation.host_progress_callback, + guardrail_context=operation.guardrail_context, + **operation.logging_data, + ) + case ListToolsRequest(params=params): + return await _execute_handle_list_tools( + context, params or PaginatedRequestParams(), self._host_progress_callback + ) + case CallToolRequest(params=params): + return await _execute_mcp_server_tool_call(context, params, self._host_progress_callback) + case ListPromptsRequest(params=params): + return await _execute_list_prompts( + context, params or PaginatedRequestParams(), self._host_progress_callback + ) + case GetPromptRequest(params=params): + return await _execute_get_prompt(context, params, self._host_progress_callback) + case ListResourcesRequest(params=params): + return await _execute_list_resources( + context, params or PaginatedRequestParams(), self._host_progress_callback + ) + case ListResourceTemplatesRequest(params=params): + return await _execute_list_resource_templates( + context, params or PaginatedRequestParams(), self._host_progress_callback + ) + case ReadResourceRequest(params=params): + return await _execute_read_resource(context, params, self._host_progress_callback) + case _: + return assert_never(operation) diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 15f97a15b73..c2f7bf7d531 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -203,17 +203,19 @@ if MCP_AVAILABLE: from litellm.proxy._experimental.mcp_server.oauth_utils import ( get_request_base_url, ) - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._experimental.mcp_server.operations import ( ListMCPToolsRestAPIResponseObject, MCPInfo, MCPServer, - _aggregate_server_key, # pyright: ignore[reportPrivateUsage] # same per-server key as the tools/list _meta outcomes - _apply_toolset_scope, + _aggregate_server_key, _fire_mcp_tool_call_logging, execute_mcp_tool, filter_tools_by_allowed_tools, filter_tools_by_key_team_permissions, fire_mcp_tool_call_failure_logging, + ) + from litellm.proxy._experimental.mcp_server.server import ( + _apply_toolset_scope, reject_disallowed_mcp_client, ) @@ -670,6 +672,7 @@ if MCP_AVAILABLE: user_api_key_auth: UserAPIKeyAuth | None = None, extra_headers: dict[str, str] | None = None, apply_tool_filters: bool = True, + client_ip: str | None = None, ): """Helper function to get tools for a single server. @@ -684,6 +687,7 @@ if MCP_AVAILABLE: extra_headers=extra_headers, add_prefix=False, raw_headers=raw_headers, + client_ip=client_ip, user_api_key_auth=user_api_key_auth, ) @@ -797,6 +801,7 @@ if MCP_AVAILABLE: user_api_key_dict, extra_headers=user_oauth_extra_headers, apply_tool_filters=apply_tool_filters, + client_ip=rest_client_ip, ) except MCPUpstreamAuthError: # Surface the upstream 401/403 to the caller so it can emit the @@ -1016,6 +1021,7 @@ if MCP_AVAILABLE: user_api_key_dict, extra_headers=user_oauth_extra_headers, apply_tool_filters=apply_tool_filters, + client_ip=_rest_client_ip, ) except Exception as e: verbose_logger.warning( @@ -1193,6 +1199,7 @@ if MCP_AVAILABLE: mcp_server_auth_headers=data.get("mcp_server_auth_headers"), oauth2_headers=user_oauth_extra_headers or data.get("oauth2_headers"), raw_headers=data.get("raw_headers"), + client_ip=IPAddressUtils.get_mcp_client_ip(request), litellm_logging_obj=data.get("litellm_logging_obj"), guardrail_context=MCPRequestContext.resolve_guardrail_context(data), requested_server_id=canonical_server_id, diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 397a82cfa45..433b693fcae 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -11,28 +11,22 @@ import hashlib import json import os import time -import traceback import types -import uuid from collections import Counter -from collections.abc import AsyncIterator, Callable, Iterable, Mapping, Sequence -from datetime import datetime -from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol +from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterable, Mapping, Sequence +from typing import TYPE_CHECKING, Final, NoReturn, Protocol import httpx from fastapi import FastAPI, HTTPException -from pydantic import AnyUrl, ConfigDict, Field, TypeAdapter, ValidationError +from pydantic import ConfigDict, TypeAdapter, ValidationError from starlette.requests import Request as StarletteRequest from starlette.responses import JSONResponse from starlette.types import Message, Receive, Scope, Send -from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_logger from litellm.constants import ( - MAXIMUM_TRACEBACK_LINES_TO_LOG, MCP_GATEWAY_SESSION_ID_PREFIX_LENGTH, ) -from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, @@ -41,12 +35,6 @@ from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, _is_mcp_admitted_user_subject, ) -from litellm.proxy._experimental.mcp_server.byok_credential_cache import ( - byok_credential_cache, - byok_credential_cache_key, - cache_byok_credential, - get_cached_byok_credential, -) from litellm.proxy._experimental.mcp_server.client_allowlist import ( MCPClientAllowlist, check_mcp_client_allowed, @@ -56,7 +44,6 @@ from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( get_request_base_url, ) from litellm.proxy._experimental.mcp_server.exceptions import ( - MCPToolResultError, MCPUpstreamAuthError, ) from litellm.proxy._experimental.mcp_server.mcp_context import ( @@ -74,7 +61,6 @@ from litellm.proxy._experimental.mcp_server.mcp_debug import ( ) from litellm.proxy._experimental.mcp_server.oauth_utils import ( _redact_mcp_resource_url, - get_byok_www_authenticate, get_passthrough_www_authenticate, get_route_relative_request_path, well_known_root_suffix, @@ -84,14 +70,6 @@ from litellm.proxy._experimental.mcp_server.utils import ( LITELLM_MCP_SERVER_DESCRIPTION, LITELLM_MCP_SERVER_NAME, LITELLM_MCP_SERVER_VERSION, - MCPMissingUserEnvVarsError, - add_server_prefix_to_name, - build_synthetic_mcp_request, - extract_mcp_tool_result_error_message, - get_server_prefix, - iter_known_server_prefixes, - logging_safe_mcp_headers, - match_known_tool_name, ) from litellm.proxy._types import ( ProxyException, @@ -99,13 +77,6 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.ip_address_utils import IPAddressUtils -from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import ( - publish_auth_cache_invalidation, -) -from litellm.proxy.litellm_pre_call_utils import ( - LiteLLMProxyRequestSetup, - get_chain_id_from_headers, -) from litellm.types.mcp import ( MCPAuth, MCPGatewaySession, @@ -114,14 +85,11 @@ from litellm.types.mcp import ( MCPGatewaySessionsTerminateResponse, MCPSpecVersion, ) -from litellm.types.mcp_server.mcp_server_manager import MCPInfo, MCPServer -from litellm.types.utils import CallTypes, StandardLoggingMCPToolCall -from litellm.utils import Rules, client, function_setup +from litellm.types.mcp_server.mcp_server_manager import MCPServer if TYPE_CHECKING: from mcp.server.session import ServerSession as _McpServerSession - from litellm.proxy._experimental.mcp_server.db import OAuthCredentialPayload _STATEFUL_SESSION_IDLE_TIMEOUT_SECONDS: Final = 30 * 60 # Upper bound on concurrent stateful sessions a single caller may hold. Each @@ -159,13 +127,6 @@ def unsupported_protocol_version(scope: Scope) -> str | None: return None -async def _invalidate_byok_cred_cache(user_id: str, server_id: str) -> None: - """Drop a stored-or-deleted BYOK credential from this worker's cache and from every peer worker's.""" - cache_key: Final = byok_credential_cache_key(user_id, server_id) - byok_credential_cache.delete_cache(cache_key) - await publish_auth_cache_invalidation(cache_key=cache_key) - - # Check if MCP is available # "mcp" requires python 3.10 or higher, but several litellm users use python 3.8 # We're making this conditional import to avoid breaking users who use python 3.8. @@ -210,19 +171,6 @@ _SESSION_MANAGERS_INITIALIZED = False _INITIALIZATION_LOCK: Final = asyncio.Lock() -def _mcp_session_id_from_headers( - raw_headers: dict[str, str] | None, -) -> str | None: - """The ``mcp-session-id`` of a stateful MCP session, read case-insensitively - from the request headers. ``None`` for stateless calls (no such header).""" - if not raw_headers: - return None - for key, value in raw_headers.items(): - if isinstance(key, str) and key.lower() == "mcp-session-id": - return value or None - return None - - def _jsonrpc_text_has_top_level_method(text: str) -> bool: """Whether a (possibly truncated) JSON-RPC envelope has a ``method`` key at the root object's top level. @@ -466,6 +414,59 @@ def _proxy_exception_to_http_exception(exc: ProxyException) -> HTTPException: if MCP_AVAILABLE: + __all__ = ( + "_MCP_CREDENTIAL_REQUEST_FIELDS", + "BlobResourceContents", + "ListMCPToolsRestAPIResponseObject", + "ResourceTemplate", + "TextResourceContents", + "_McpDeniedDetail", + "_aggregate_server_key", + "_build_virtual_call_logging_obj", + "_check_byok_credential", + "_client_has_passthrough_authorization", + "_client_has_per_server_auth_header", + "_dispatch_virtual_mcp_tool", + "_fire_mcp_tool_call_logging", + "_get_allowed_mcp_servers", + "_get_allowed_mcp_servers_from_mcp_server_names", + "_get_byok_credential", + "_get_prompts_from_mcp_servers", + "_get_resource_templates_from_mcp_servers", + "_get_resources_from_mcp_servers", + "_get_standard_logging_mcp_tool_call", + "_get_tools_from_mcp_servers", + "_get_user_oauth_extra_headers_from_db", + "_handle_local_mcp_tool", + "_handle_managed_mcp_tool", + "_http_detail_message", + "_invalidate_byok_cred_cache", + "_list_mcp_prompts", + "_list_mcp_resource_templates", + "_list_mcp_resources", + "_list_mcp_tools", + "_list_tools_before_first_call", + "_mcp_session_id_from_headers", + "_merge_gateway_initialize_instructions", + "_prefetch_oauth_creds_for_user", + "_prepare_mcp_server_headers", + "_raise_if_initialize_grants_no_mcp_servers", + "_redact_mcp_resource_url", + "_resolve_display_name_to_original", + "_run_post_mcp_call_guardrails", + "_server_answers_to", + "_tool_name_matches", + "apply_tool_overrides", + "call_mcp_tool", + "execute_mcp_tool", + "filter_tools_by_allowed_tools", + "filter_tools_by_key_team_permissions", + "fire_mcp_tool_call_failure_logging", + "global_mcp_server_manager", + "mcp_get_prompt", + "mcp_read_resource", + "raise_denied_scoped_mcp_access", + ) from mcp.server import Server # Import auth context variables and middleware @@ -476,6 +477,23 @@ if MCP_AVAILABLE: from mcp.server.context import ServerRequestContext from mcp.server.lowlevel.server import NotificationOptions from mcp.server.models import InitializationOptions + from mcp.shared.exceptions import MCPError + from mcp.types import ( + CallToolRequest, + GetPromptRequest, + ListPromptsRequest, + ListResourcesRequest, + ListResourceTemplatesRequest, + ListToolsRequest, + ReadResourceRequest, + ) + + from litellm.proxy._experimental.mcp_server import operations + from litellm.proxy._experimental.mcp_server.contracts import OperationContext + from litellm.proxy._experimental.mcp_server.operations import ( + _invalidate_byok_cred_cache, + _mcp_session_id_from_headers, + ) try: from mcp.server.streamable_http_manager import StreamableHTTPSessionManager @@ -493,62 +511,27 @@ if MCP_AVAILABLE: ListResourceTemplatesResult, ListToolsResult, PaginatedRequestParams, - Prompt, ReadResourceRequestParams, - TextContent, ) - from mcp.types import Tool as MCPTool from mcp_types.version import HANDSHAKE_PROTOCOL_VERSIONS from litellm.proxy._experimental.mcp_server.auth.litellm_auth_handler import ( MCPAuthenticatedUser, ) - from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ( - SERVER_OUTCOMES_META_KEY, - AggregateToolListing, - ServerListOk, - ServerOutcome, - classify_list_exception, - outcome_wire_value, - ) from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( MCPServerManager, - _caller_authorization_fans_out, - _client_forwarded_authorization_headers, - _resolve_openapi_tool_auth, - _should_strip_caller_authorization, global_mcp_server_manager, ) - from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( - _request_auth_header, - _request_extra_headers, - _request_resolved_auth_headers, - ) - from litellm.proxy._experimental.mcp_server.sse_transport import SseServerTransport - from litellm.proxy._experimental.mcp_server.tool_registry import ( - global_mcp_tool_registry, - ) - from litellm.proxy._experimental.mcp_server.utils import ( - MCP_TOOL_PREFIX_SEPARATOR, - is_tool_name_prefixed, - normalize_server_name, - split_server_prefix_from_name, - strip_known_server_prefix, - ) - from litellm.types.mcp import DEFAULT_CREDENTIAL_HEADER, without_header ###################################################### ############ MCP Tools List REST API Response Object # # Defined here because we don't want to add `mcp` as a # required dependency for `litellm` pip package ###################################################### - class ListMCPToolsRestAPIResponseObject(MCPTool): - """ - Object returned by the /tools/list REST API route. - """ - - mcp_info: MCPInfo | None = Field(default=None, alias="mcp_info") - model_config = ConfigDict(arbitrary_types_allowed=True) + from litellm.proxy._experimental.mcp_server.operations import ( + ListMCPToolsRestAPIResponseObject, + ) + from litellm.proxy._experimental.mcp_server.sse_transport import SseServerTransport def _gateway_create_initialization_options( self, @@ -818,94 +801,45 @@ if MCP_AVAILABLE: ############### MCP Server Routes ####################### ######################################################## - async def handle_list_tools(ctx: ServerRequestContext, params: PaginatedRequestParams) -> ListToolsResult: - """ - List all available tools, with each server's listing outcome attached to the result's - ``_meta`` (SERVER_OUTCOMES_META_KEY) so a broken upstream is distinguishable from a healthy - server with no tools. Returning a ListToolsResult (rather than a bare list) makes the MCP SDK - pass the result through unwrapped, which is what lets the ``_meta`` survive to the client. - Also captures the active session for propagation to callbacks. - """ - req_ctx: Final = ctx - _ctx_reset_token: Final = active_mcp_request_ctx_var.set(ctx) - _session_reset_token: Final = active_mcp_session_var.set(ctx.session) - _trace_token = None - _transport_token = None - _destinations_token = None - - try: - _trace_token = _otel_set_mcp_trace_carrier(_mcp_meta_trace_carrier(req_ctx)) - _transport_token = _otel_set_mcp_transport_span(_otel_transport_span_from_message(req_ctx)) - _destinations_token = _otel_set_mcp_request_destinations(req_ctx) - # Get user authentication from context variable + @contextlib.asynccontextmanager + async def _legacy_operation_context(ctx: ServerRequestContext, *, trace: bool) -> AsyncGenerator[OperationContext]: + with contextlib.ExitStack() as cleanup: + cleanup.callback(active_mcp_request_ctx_var.reset, active_mcp_request_ctx_var.set(ctx)) + cleanup.callback(active_mcp_session_var.reset, active_mcp_session_var.set(ctx.session)) + if trace: + cleanup.callback( + _otel_reset_mcp_trace_carrier, _otel_set_mcp_trace_carrier(_mcp_meta_trace_carrier(ctx)) + ) + cleanup.callback( + _otel_reset_mcp_transport_span, _otel_set_mcp_transport_span(_otel_transport_span_from_message(ctx)) + ) + cleanup.callback(_otel_reset_mcp_request_destinations, _otel_set_mcp_request_destinations(ctx)) ( - user_api_key_auth, - mcp_auth_header, - mcp_servers, - mcp_server_auth_headers, - oauth2_headers, - raw_headers, - _client_ip, + auth, + token, + servers, + server_headers, + oauth_headers, + headers, + client_ip, ) = await get_or_extract_auth_context() - verbose_logger.debug("MCP list_tools - User API Key Auth from context: %s", user_api_key_auth) - verbose_logger.debug("MCP list_tools - MCP servers from context: %s", mcp_servers) - verbose_logger.debug( - "MCP list_tools - MCP server auth headers: %s", - list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None, - ) - from mcp.types import Tool - - from litellm.proxy._experimental.mcp_server.tool_search import ( - get_mcp_proxy_tool_definitions, - get_virtual_tool_definitions, + yield operations.prepare_context( + auth, token, servers, server_headers, oauth_headers, headers, client_ip, _mcp_proxy_mode.get() ) - if _mcp_proxy_mode.get(): - return ListToolsResult(tools=[Tool.model_validate(d) for d in get_mcp_proxy_tool_definitions()]) - if getattr( - getattr(user_api_key_auth, "object_permission", None), - "mcp_tool_search_enabled", - False, - ): - return ListToolsResult(tools=[Tool.model_validate(d) for d in get_virtual_tool_definitions()]) - - # Get mcp_servers from context variable - verbose_logger.debug("MCP list_tools - Calling _list_mcp_tools") - listing: Final = await _list_mcp_tools( - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_servers=mcp_servers, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - log_list_tools_to_spendlogs=True, - list_tools_log_source="mcp_protocol", - ) - verbose_logger.info("MCP list_tools - Successfully returned %s tools", len(listing.tools)) - if not listing.outcomes: - return ListToolsResult(tools=listing.tools) - outcome_meta: Final = { - SERVER_OUTCOMES_META_KEY: { - key: outcome_wire_value(outcome) for key, outcome in listing.outcomes.items() - } - } - return ListToolsResult.model_validate({"tools": listing.tools, "_meta": outcome_meta}) - except HTTPException as e: - from mcp.shared.exceptions import MCPError - from mcp.types import INVALID_REQUEST - - raise MCPError(code=INVALID_REQUEST, message=_http_detail_message(e.detail)) from e - except Exception as e: - verbose_logger.exception("Error in list_tools endpoint: %s", e) - # Return empty list instead of failing completely - # This prevents the HTTP stream from failing and allows the client to get a response - return ListToolsResult(tools=[]) # mutable-ok: MCP result payload - finally: - _otel_reset_mcp_request_destinations(_destinations_token) - _otel_reset_mcp_transport_span(_transport_token) - _otel_reset_mcp_trace_carrier(_trace_token) - active_mcp_session_var.reset(_session_reset_token) - active_mcp_request_ctx_var.reset(_ctx_reset_token) + async def handle_list_tools(ctx: ServerRequestContext, params: PaginatedRequestParams) -> ListToolsResult: + try: + async with _legacy_operation_context(ctx, trace=True) as context: + return await operations.GatewayOperations(_capture_host_progress_callback(ctx)).execute( + ListToolsRequest(params=params), context + ) + except MCPError: + raise + except HTTPException as exc: + raise MCPError(code=INVALID_REQUEST, message=operations._http_detail_message(exc.detail)) from exc + except Exception as exc: # noqa: BLE001 # preserve native listing fallback for ingress failures + verbose_logger.exception("Error in list_tools endpoint: %s", exc) + return ListToolsResult(tools=[]) def _capture_host_progress_callback(ctx: ServerRequestContext) -> Callable | None: """Return a progress-forwarding callback bound to the host MCP session. @@ -942,581 +876,71 @@ if MCP_AVAILABLE: raise MCPError(code=METHOD_NOT_FOUND, message="Operation unavailable on /mcp/proxy") - async def _build_virtual_call_logging_obj( - name: str, - arguments: dict[str, object], - user_api_key_auth: UserAPIKeyAuth, - raw_headers: Mapping[str, str] | None = None, - client_ip: str | None = None, - ) -> LiteLLMLoggingObj | None: - """Run the pre-call pipeline (guardrails + logging setup) for a virtual - mcp_tool_call so the SSE path spend-logs like the REST path.""" - from litellm.proxy.common_request_processing import ( - ProxyBaseLLMRequestProcessing, - ) - from litellm.proxy.proxy_server import ( - general_settings, - proxy_config, - proxy_logging_obj, - ) - - request: Final = build_synthetic_mcp_request( - path="/mcp/tools/call", - raw_headers=raw_headers, - client_ip=client_ip, - ) - _, virtual_logging_obj = await ProxyBaseLLMRequestProcessing( - data={"name": name, "arguments": arguments} - ).common_processing_pre_call_logic( - request=request, - user_api_key_dict=user_api_key_auth, - proxy_config=proxy_config, - route_type=CallTypes.call_mcp_tool.value, - proxy_logging_obj=proxy_logging_obj, - general_settings=general_settings, - ) - return virtual_logging_obj - - async def _dispatch_virtual_mcp_tool( - name: str, - arguments: dict[str, object] | None, - user_api_key_auth: UserAPIKeyAuth | None, - client_ip: str | None, - mcp_servers: list[str] | None = None, - mcp_auth_header: str | None = None, - mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, - oauth2_headers: dict[str, str] | None = None, - raw_headers: dict[str, str] | None = None, - ) -> CallToolResult | None: - """Handle the mcp_tool_search / mcp_tool_call virtual tools. - - Returns a CallToolResult when ``name`` is a virtual tool, else ``None`` so - the caller falls through to normal tool routing. - """ - from litellm.llms.litellm_proxy.skills.skill_search import DEFAULT_SKILL_SEARCH_TOP_K - from litellm.proxy._experimental.mcp_server.tool_search import ( - AGENT_SEARCH_TOOL_NAME, - DEFAULT_AGENT_SEARCH_TOP_K, - MCP_PROXY_CALL_TOOL_NAME, - MCP_PROXY_TOOL_NAMES, - MCP_TOOL_SEARCH_TOOL_NAME, - SKILL_SEARCH_TOOL_NAME, - VIRTUAL_TOOL_NAMES, - coerce_top_k, - handle_agent_search, - handle_mcp_proxy_tool, - handle_mcp_tool_call, - handle_mcp_tool_search, - handle_skill_search, - ) - - if _mcp_proxy_mode.get() and name not in MCP_PROXY_TOOL_NAMES: - return CallToolResult( - content=[ # mutable-ok: MCP result content - TextContent(type="text", text=f"Tool {name} is unavailable on /mcp/proxy") - ], - is_error=True, - ) - - if _mcp_proxy_mode.get() and name in MCP_PROXY_TOOL_NAMES: - assert user_api_key_auth is not None - proxy_call_start: Final = datetime.now() # noqa: DTZ005 # logging pipeline uses naive datetimes - proxy_logging_obj: Final = ( - await _build_virtual_call_logging_obj( - name=name, - arguments=arguments or {}, # mutable-ok: logging pipeline payload - user_api_key_auth=user_api_key_auth, - raw_headers=raw_headers, - client_ip=client_ip, - ) - if name == MCP_PROXY_CALL_TOOL_NAME - else None - ) - try: - proxy_result: Final = await handle_mcp_proxy_tool( - name=name, - arguments=arguments or {}, # mutable-ok: proxy handler payload - user_api_key_dict=user_api_key_auth, - client_ip=client_ip, - mcp_servers=mcp_servers, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - litellm_logging_obj=proxy_logging_obj, - ) - except Exception as exc: - if proxy_logging_obj is not None: - from litellm.proxy.proxy_server import proxy_logging_obj as request_logging_obj - - failure_end: Final = datetime.now() # noqa: DTZ005 # matches the logging pipeline start time - failure_traceback: Final = traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG) - try: - proxy_logging_obj.failure_handler(exc, failure_traceback, proxy_call_start, failure_end) - await proxy_logging_obj.async_failure_handler( - exc, failure_traceback, proxy_call_start, failure_end - ) - if not isinstance(exc, MCPUpstreamAuthError): - await request_logging_obj.post_call_failure_hook( - request_data={ # mutable-ok: failure hook mutates its request payload - "name": name, - "arguments": arguments, - "litellm_logging_obj": proxy_logging_obj, - }, - original_exception=exc, - user_api_key_dict=user_api_key_auth, - route="/mcp/call_tool", - traceback_str=failure_traceback, - ) - except Exception: # noqa: BLE001 # a failing failure hook must not mask the tool call's own error - verbose_logger.exception("Error logging failed MCP proxy tool call") - raise - if proxy_logging_obj is not None: - return await _fire_mcp_tool_call_logging( - logging_obj=proxy_logging_obj, - result=proxy_result, - start_time=proxy_call_start, - end_time=datetime.now(), # noqa: DTZ005 # matches the logging pipeline start time - user_api_key_auth=user_api_key_auth, - request_data=types.MappingProxyType({"name": name, "arguments": arguments}), - ) - return proxy_result - - if name not in VIRTUAL_TOOL_NAMES: - return None - - if not getattr( - getattr(user_api_key_auth, "object_permission", None), - "mcp_tool_search_enabled", - False, - ): - return CallToolResult( - content=[ - TextContent( - type="text", - text=f"Tool {name} requires mcp_tool_search_enabled on the key", - ) - ], - is_error=True, - ) - - args: Final = arguments or {} - if name == MCP_TOOL_SEARCH_TOOL_NAME: - return await handle_mcp_tool_search( - query=args.get("query", ""), - top_k=coerce_top_k(args.get("top_k", 5)), - user_api_key_dict=user_api_key_auth, - client_ip=client_ip, - mcp_servers=mcp_servers, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - ) - - assert user_api_key_auth is not None # guaranteed by the flag check above - if name == AGENT_SEARCH_TOOL_NAME: - return await handle_agent_search( - query=str(args.get("query", "")), - top_k=coerce_top_k(args.get("top_k", DEFAULT_AGENT_SEARCH_TOP_K), default=DEFAULT_AGENT_SEARCH_TOP_K), - user_api_key_dict=user_api_key_auth, - ) - if name == SKILL_SEARCH_TOOL_NAME: - return await handle_skill_search( - query=str(args.get("query", "")), - top_k=coerce_top_k(args.get("top_k", DEFAULT_SKILL_SEARCH_TOP_K), default=DEFAULT_SKILL_SEARCH_TOP_K), - user_api_key_dict=user_api_key_auth, - ) - virtual_logging_obj: Final = await _build_virtual_call_logging_obj( - name=name, - arguments=args, - user_api_key_auth=user_api_key_auth, - raw_headers=raw_headers, - client_ip=client_ip, - ) - return await handle_mcp_tool_call( - tool_name=args.get("tool_name", ""), - arguments=args.get("arguments") or {}, - user_api_key_dict=user_api_key_auth, - client_ip=client_ip, - mcp_servers=mcp_servers, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - litellm_logging_obj=virtual_logging_obj, - ) + from litellm.proxy._experimental.mcp_server.operations import ( + _build_virtual_call_logging_obj, + _dispatch_virtual_mcp_tool, + ) async def mcp_server_tool_call(ctx: ServerRequestContext, params: CallToolRequestParams) -> CallToolResult: - """ - Call a specific tool with the provided arguments - Args: - ctx: SDK request context carrying the client session and HTTP request - params (CallToolRequestParams): Tool name and arguments - Returns: - CallToolResult: Tool execution results - """ - from mcp.types import CallToolResult - - from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException - from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request - from litellm.proxy.proxy_server import proxy_config - - req_ctx: Final = ctx - _ctx_reset_token: Final = active_mcp_request_ctx_var.set(ctx) - _session_reset_token: Final = active_mcp_session_var.set(ctx.session) - _trace_token = None - _transport_token = None - _destinations_token = None - - try: - _trace_token = _otel_set_mcp_trace_carrier(_mcp_meta_trace_carrier(req_ctx)) - _transport_token = _otel_set_mcp_transport_span(_otel_transport_span_from_message(req_ctx)) - _destinations_token = _otel_set_mcp_request_destinations(req_ctx) - # Validate arguments - ( - user_api_key_auth, - mcp_auth_header, - mcp_servers, - mcp_server_auth_headers, - oauth2_headers, - raw_headers, - _client_ip, - ) = await get_or_extract_auth_context() - verbose_logger.debug( - "MCP mcp_server_tool_call - user_api_key_auth=%s, user_role=%s", - user_api_key_auth, - getattr(user_api_key_auth, "user_role", "N/A"), + async with _legacy_operation_context(ctx, trace=True) as context: + return await operations.GatewayOperations(_capture_host_progress_callback(ctx)).execute( + CallToolRequest(params=params), context ) - verbose_logger.debug("MCP mcp_server_tool_call - User API Key Auth from context: %s", user_api_key_auth) - - try: - # Inside this try so virtual-tool errors convert to isError - # CallToolResult instead of raising out of the protocol handler. - virtual_tool_result: Final = await _dispatch_virtual_mcp_tool( - name=params.name, - arguments=params.arguments, - user_api_key_auth=user_api_key_auth, - client_ip=_client_ip, - mcp_servers=mcp_servers, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - ) - if virtual_tool_result is not None: - return virtual_tool_result - - host_progress_callback: Final = _capture_host_progress_callback(ctx) - # Create a body date for logging - body_data: Final = {"name": params.name, "arguments": params.arguments} # mutable-ok: logging payload - # Set trace/session id from raw_headers so spend logs and logging_obj stay consistent (same as A2A) - chain_id: Final = get_chain_id_from_headers(raw_headers) - if chain_id: - body_data["litellm_trace_id"] = chain_id - body_data["litellm_session_id"] = chain_id - - request: Final = build_synthetic_mcp_request( - path="/mcp/tools/call", - raw_headers=raw_headers, - client_ip=_client_ip, - ) - if user_api_key_auth is not None: - data = await add_litellm_data_to_request( - data=body_data, - request=request, - # Bill a team-derived call to the team that granted it. A keyless admitted - # subject carries no team_id, so spend skipped team updates entirely and - # charged the user's PRIMARY org — the granting team's budget never - # accumulated (so it could never begin to block) and, cross-org, the wrong - # organization was charged. This is the ACCOUNTING half; the enforcement - # half (an already-over-budget team stops granting) lives in the source gate. - # Authorization is unaffected: it ran before this, and the union is resolved - # from the untouched auth object passed to call_mcp_tool below. - user_api_key_dict=await MCPRequestHandler.billing_auth_for_tool_call( - user_api_key_auth, tool_name=params.name - ), - proxy_config=proxy_config, - ) - else: - data = body_data - - response: Final = await call_mcp_tool( - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_servers=mcp_servers, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - client_ip=_client_ip, - host_progress_callback=host_progress_callback, - **data, # for logging - ) - except MCPMissingUserEnvVarsError as e: - verbose_logger.info( - "MCP mcp_server_tool_call missing per-user env vars: server_id=%s missing=%s", - e.server_id, - e.missing, - ) - return CallToolResult( - content=[TextContent(text=str(e), type="text")], - is_error=True, - ) - except BlockedPiiEntityError as e: - verbose_logger.error("BlockedPiiEntityError in MCP tool call: %s", e) - return CallToolResult( - content=[ - TextContent( - text=f"Error: Blocked PII entity detected - {e}", - type="text", - ) - ], - is_error=True, - ) - except GuardrailRaisedException as e: - verbose_logger.error("GuardrailRaisedException in MCP tool call: %s", e) - return CallToolResult( - content=[TextContent(text=f"Error: Guardrail violation - {e}", type="text")], - is_error=True, - ) - except HTTPException as e: - verbose_logger.error("HTTPException in MCP tool call: %s", e) - return CallToolResult( - content=[TextContent(text=f"Error: {_http_detail_message(e.detail)}", type="text")], - is_error=True, - ) - except MCPUpstreamAuthError as e: - # The MCP session manager serializes handler exceptions as JSON-RPC errors, so a - # mid-session tool call cannot emit a raw 401 + WWW-Authenticate the way the REST - # call path and the connect-time preemptive check do. Return an explicit isError - # naming the upstream status (at info level, not a traceback) so the client still - # learns it must re-authenticate upstream and expected pass-through 401s don't spam. - verbose_logger.info("Upstream auth failure calling MCP tool: HTTP %s", e.status_code) - return CallToolResult( - content=[ - TextContent( - text=f"Error: upstream authentication required (HTTP {e.status_code})", - type="text", - ) - ], - is_error=True, - ) - except Exception as e: - verbose_logger.exception("MCP mcp_server_tool_call - error: %s", e) - return CallToolResult( - content=[TextContent(text=f"Error: {e}", type="text")], - is_error=True, - ) - - return response - finally: - _otel_reset_mcp_request_destinations(_destinations_token) - _otel_reset_mcp_transport_span(_transport_token) - _otel_reset_mcp_trace_carrier(_trace_token) - active_mcp_session_var.reset(_session_reset_token) - active_mcp_request_ctx_var.reset(_ctx_reset_token) - async def list_prompts(ctx: ServerRequestContext, params: PaginatedRequestParams) -> ListPromptsResult: - """ - List all available prompts - """ if _mcp_proxy_mode.get(): _reject_mcp_proxy_operation() - _ctx_reset_token: Final = active_mcp_request_ctx_var.set(ctx) - _session_reset_token: Final = active_mcp_session_var.set(ctx.session) - try: - # Get user authentication from context variable - ( - user_api_key_auth, - mcp_auth_header, - mcp_servers, - mcp_server_auth_headers, - oauth2_headers, - raw_headers, - _client_ip, - ) = await get_or_extract_auth_context() - verbose_logger.debug("MCP list_prompts - User API Key Auth from context: %s", user_api_key_auth) - verbose_logger.debug("MCP list_prompts - MCP servers from context: %s", mcp_servers) - verbose_logger.debug( - "MCP list_prompts - MCP server auth headers: %s", - list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None, - ) - # Get mcp_servers from context variable - verbose_logger.debug("MCP list_prompts - Calling _list_prompts") - prompts: Final = await _list_mcp_prompts( - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_servers=mcp_servers, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - ) - verbose_logger.info("MCP list_prompts - Successfully returned %s prompts", len(prompts)) - return ListPromptsResult(prompts=prompts) - except Exception as e: - verbose_logger.exception("Error in list_prompts endpoint: %s", e) - # Return empty list instead of failing completely - # This prevents the HTTP stream from failing and allows the client to get a response - return ListPromptsResult(prompts=[]) # mutable-ok: MCP result payload - finally: - active_mcp_session_var.reset(_session_reset_token) - active_mcp_request_ctx_var.reset(_ctx_reset_token) + async with _legacy_operation_context(ctx, trace=False) as context: + return await operations.GatewayOperations(_capture_host_progress_callback(ctx)).execute( + ListPromptsRequest(params=params), context + ) + except Exception as exc: # noqa: BLE001 # preserve native listing fallback for ingress failures + verbose_logger.exception("Error in list_prompts endpoint: %s", exc) + return ListPromptsResult(prompts=[]) async def get_prompt(ctx: ServerRequestContext, params: GetPromptRequestParams) -> GetPromptResult: - """ - Get a specific prompt with the provided arguments - """ if _mcp_proxy_mode.get(): _reject_mcp_proxy_operation() - _ctx_reset_token: Final = active_mcp_request_ctx_var.set(ctx) - _session_reset_token: Final = active_mcp_session_var.set(ctx.session) - - try: - ( - user_api_key_auth, - mcp_auth_header, - mcp_servers, - mcp_server_auth_headers, - oauth2_headers, - raw_headers, - _client_ip, - ) = await get_or_extract_auth_context() - - verbose_logger.debug("MCP mcp_server_tool_call - User API Key Auth from context: %s", user_api_key_auth) - return await mcp_get_prompt( - name=params.name, - arguments=params.arguments, - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_servers=mcp_servers, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, + async with _legacy_operation_context(ctx, trace=False) as context: + return await operations.GatewayOperations(_capture_host_progress_callback(ctx)).execute( + GetPromptRequest(params=params), context ) - finally: - active_mcp_session_var.reset(_session_reset_token) - active_mcp_request_ctx_var.reset(_ctx_reset_token) async def list_resources(ctx: ServerRequestContext, params: PaginatedRequestParams) -> ListResourcesResult: - """List all available resources.""" if _mcp_proxy_mode.get(): _reject_mcp_proxy_operation() - _ctx_reset_token: Final = active_mcp_request_ctx_var.set(ctx) - _session_reset_token: Final = active_mcp_session_var.set(ctx.session) - try: - ( - user_api_key_auth, - mcp_auth_header, - mcp_servers, - mcp_server_auth_headers, - oauth2_headers, - raw_headers, - _client_ip, - ) = await get_or_extract_auth_context() - verbose_logger.debug("MCP list_resources - User API Key Auth from context: %s", user_api_key_auth) - verbose_logger.debug("MCP list_resources - MCP servers from context: %s", mcp_servers) - verbose_logger.debug( - "MCP list_resources - MCP server auth headers: %s", - list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None, - ) - - resources: Final = await _list_mcp_resources( - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_servers=mcp_servers, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - ) - verbose_logger.info("MCP list_resources - Successfully returned %s resources", len(resources)) - return ListResourcesResult(resources=resources) - except Exception as e: - verbose_logger.exception("Error in list_resources endpoint: %s", e) - return ListResourcesResult(resources=[]) # mutable-ok: MCP result payload - finally: - active_mcp_session_var.reset(_session_reset_token) - active_mcp_request_ctx_var.reset(_ctx_reset_token) + async with _legacy_operation_context(ctx, trace=False) as context: + return await operations.GatewayOperations(_capture_host_progress_callback(ctx)).execute( + ListResourcesRequest(params=params), context + ) + except Exception as exc: # noqa: BLE001 # preserve native listing fallback for ingress failures + verbose_logger.exception("Error in list_resources endpoint: %s", exc) + return ListResourcesResult(resources=[]) async def list_resource_templates( ctx: ServerRequestContext, params: PaginatedRequestParams ) -> ListResourceTemplatesResult: - """List all available resource templates.""" if _mcp_proxy_mode.get(): _reject_mcp_proxy_operation() - _ctx_reset_token: Final = active_mcp_request_ctx_var.set(ctx) - _session_reset_token: Final = active_mcp_session_var.set(ctx.session) - try: - ( - user_api_key_auth, - mcp_auth_header, - mcp_servers, - mcp_server_auth_headers, - oauth2_headers, - raw_headers, - _client_ip, - ) = await get_or_extract_auth_context() - verbose_logger.debug("MCP list_resource_templates - User API Key Auth from context: %s", user_api_key_auth) - verbose_logger.debug("MCP list_resource_templates - MCP servers from context: %s", mcp_servers) - verbose_logger.debug( - "MCP list_resource_templates - MCP server auth headers: %s", - list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None, - ) - - resource_templates: Final = await _list_mcp_resource_templates( - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_servers=mcp_servers, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - ) - verbose_logger.info( - "MCP list_resource_templates - Successfully returned %s resource templates", len(resource_templates) - ) - return ListResourceTemplatesResult(resource_templates=resource_templates) - except Exception as e: - verbose_logger.exception("Error in list_resource_templates endpoint: %s", e) - return ListResourceTemplatesResult(resource_templates=[]) # mutable-ok: MCP result payload - finally: - active_mcp_session_var.reset(_session_reset_token) - active_mcp_request_ctx_var.reset(_ctx_reset_token) + async with _legacy_operation_context(ctx, trace=False) as context: + return await operations.GatewayOperations(_capture_host_progress_callback(ctx)).execute( + ListResourceTemplatesRequest(params=params), context + ) + except Exception as exc: # noqa: BLE001 # preserve native listing fallback for ingress failures + verbose_logger.exception("Error in list_resource_templates endpoint: %s", exc) + return ListResourceTemplatesResult(resource_templates=[]) async def read_resource(ctx: ServerRequestContext, params: ReadResourceRequestParams) -> ReadResourceResult: if _mcp_proxy_mode.get(): _reject_mcp_proxy_operation() - _ctx_reset_token: Final = active_mcp_request_ctx_var.set(ctx) - _session_reset_token: Final = active_mcp_session_var.set(ctx.session) - - try: - ( - user_api_key_auth, - mcp_auth_header, - mcp_servers, - mcp_server_auth_headers, - oauth2_headers, - raw_headers, - _client_ip, - ) = await get_or_extract_auth_context() - - read_resource_result: Final = await mcp_read_resource( - url=params.uri, - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_servers=mcp_servers, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, + async with _legacy_operation_context(ctx, trace=False) as context: + return await operations.GatewayOperations(_capture_host_progress_callback(ctx)).execute( + ReadResourceRequest(params=params), context ) - return read_resource_result - finally: - active_mcp_session_var.reset(_session_reset_token) - active_mcp_request_ctx_var.reset(_ctx_reset_token) - server.add_request_handler("tools/list", PaginatedRequestParams, handle_list_tools) server.add_request_handler("tools/call", CallToolRequestParams, mcp_server_tool_call) server.add_request_handler("prompts/list", PaginatedRequestParams, list_prompts) @@ -1533,527 +957,24 @@ if MCP_AVAILABLE: ############ Helper Functions ########################## ######################################################## - async def _get_allowed_mcp_servers_from_mcp_server_names( - mcp_servers: Sequence[str] | None, - allowed_mcp_servers: list[MCPServer], - ) -> list[MCPServer]: - """ - Get the filtered MCP servers from the MCP server names. - - Fails closed when ``mcp_servers`` is explicitly provided (path- or - header-derived) but none of the names resolve to a server alias or - access group the caller can access. The previous behavior returned - the full ``allowed_mcp_servers`` set, which silently widened scope - when a client targeted ``/mcp//`` and made URL/header - namespacing appear to work when it did not. - """ - - filtered_server: Final[dict[str, MCPServer]] = {} - # Filter servers based on mcp_servers parameter if provided - if mcp_servers is not None: - for server_or_group in mcp_servers: - server_name_matched = False - - for server in allowed_mcp_servers: - if server and _server_answers_to(server, server_or_group): - filtered_server[server.server_id] = server - server_name_matched = True - break - - if not server_name_matched: - try: - access_group_server_ids = await MCPRequestHandler._get_mcp_servers_from_access_groups( - [server_or_group] - ) - # Only include servers that the user has access to - for server_id in access_group_server_ids: - for server in allowed_mcp_servers: - if server_id == server.server_id: - filtered_server[server.server_id] = server - except Exception as e: - verbose_logger.debug("Could not resolve '%s' as access group: %s", server_or_group, e) - - if filtered_server: - return list(filtered_server.values()) - - if mcp_servers is not None: - # Caller asked for a specific scope but nothing resolved. Fail - # closed so URL/header namespacing cannot silently fall back to - # the caller's full allowed-server set. - verbose_logger.debug( - "MCP scope filter resolved to no servers for requested names %s; returning empty list (fail-closed).", - mcp_servers, - ) - return [] - - return allowed_mcp_servers - - def _http_detail_message(detail: object) -> str: - return str(detail.get("error")) if isinstance(detail, dict) and detail.get("error") else str(detail) - - def _server_answers_to(server: MCPServer, name: str) -> bool: - requested: Final = name.lower() - return any(requested == known.lower() for known in iter_known_server_prefixes(server) if known) - - class _McpDeniedDetail(TypedDict): - error: ReadOnly[str] - - async def raise_denied_scoped_mcp_access( - requested_names: Sequence[str], - user_api_key_auth: UserAPIKeyAuth | None, - client_ip: str | None = None, - ) -> None: - """A scoped request (``/mcp/`` path or ``x-mcp-servers`` header) resolved to zero - allowed servers, so the denial must be loud: a silent 200 with no tools reads as a healthy - server with no tools. Unknown, unauthorized, and access-group names all share one generic - error so scoping cannot probe which servers exist; the agent variant fires only when the - same request resolves once the agent binding is stripped, proving the binding caused the veto.""" - agent_id: Final = user_api_key_auth.agent_id if user_api_key_auth else None - if user_api_key_auth is not None and agent_id: - resolved_without_agent: Final = await _get_allowed_mcp_servers( - user_api_key_auth=user_api_key_auth.model_copy(update=types.MappingProxyType({"agent_id": None})), - mcp_servers=requested_names, - client_ip=client_ip, - ) - - def _resolved_to_server(name: str) -> bool: - return any(_server_answers_to(server, name) for server in resolved_without_agent) - - vetoed_server: Final = next((name for name in requested_names if _resolved_to_server(name)), None) - if vetoed_server is not None: - agent_denial: Final[_McpDeniedDetail] = { - "error": ( - f"MCP server '{vetoed_server}' is not available to this key: the key is bound to " - f"agent '{agent_id}', whose MCP grants do not include this server. Add the server " - f"to the agent's object_permission.mcp_servers (edit the agent in the Admin UI or " - f"PATCH /v1/agents/{agent_id}), or use a key that is not bound to the agent." - ) - } - raise HTTPException(status_code=403, detail=agent_denial) - vetoed_group: Final = next( - ( - name - for name in requested_names - if not _resolved_to_server(name) - and any(name in (server.access_groups or ()) for server in resolved_without_agent) - ), - None, - ) - if vetoed_group is not None: - group_denial: Final[_McpDeniedDetail] = { - "error": ( - f"MCP access group '{vetoed_group}' is not available to this key: the key is bound to " - f"agent '{agent_id}', whose MCP grants do not include it. Add the group to the " - f"agent's object_permission.mcp_access_groups (edit the agent in the Admin UI or " - f"PATCH /v1/agents/{agent_id}), or use a key that is not bound to the agent." - ) - } - raise HTTPException(status_code=403, detail=group_denial) - generic_denial: Final[_McpDeniedDetail] = { - "error": f"The key is not allowed to access the requested MCP servers: {', '.join(requested_names)}" - } - raise HTTPException(status_code=403, detail=generic_denial) - - def _tool_name_matches(tool_name: str, filter_list: list[str], mcp_server: MCPServer) -> bool: - """ - Check if a tool name matches any name in the filter list. - - Reads the same owner the server-level permission checks use, so discovery hides - exactly what dispatch refuses. ``mcp_server`` is required: guessing the boundary - at the first separator mismatches every tool on a server whose prefix contains - the separator. - """ - bare_name: Final = strip_known_server_prefix(tool_name, mcp_server) - return match_known_tool_name(bare_name, mcp_server, filter_list) is not None - - def filter_tools_by_allowed_tools( - tools: list[MCPTool], - mcp_server: MCPServer, - ) -> list[MCPTool]: - """ - Filter tools by allowed/disallowed tools configuration. - - If allowed_tools is set, only tools in that list are returned. - If disallowed_tools is set, tools in that list are excluded. - Tool names are matched with and without server prefixes for flexibility. - - Args: - tools: List of tools to filter - mcp_server: Server configuration with allowed_tools/disallowed_tools - - Returns: - Filtered list of tools - """ - from litellm.proxy._experimental.mcp_server.utils import ( - server_applies_tool_allowlist, - ) - - tools_to_return = tools - - # Filter by allowed_tools (whitelist) - if server_applies_tool_allowlist(mcp_server): - if not mcp_server.allowed_tools: - return [] - tools_to_return = [ - tool for tool in tools if _tool_name_matches(tool.name, mcp_server.allowed_tools, mcp_server) - ] - - # Filter by disallowed_tools (blacklist) - if mcp_server.disallowed_tools: - tools_to_return = [ - tool - for tool in tools_to_return - if not _tool_name_matches(tool.name, mcp_server.disallowed_tools, mcp_server) - ] - - return tools_to_return - - def apply_tool_overrides( - tools: list[MCPTool], - mcp_server: MCPServer, - ) -> list[MCPTool]: - """Apply admin-configured display name/description overrides to tools. - - Overrides are keyed by the unprefixed tool name, same convention as - allowed_tools configuration. - """ - display_name_map: Final = mcp_server.tool_name_to_display_name or {} - description_map: Final = mcp_server.tool_name_to_description or {} - if not display_name_map and not description_map: - return tools - - for tool in tools: - unprefixed = strip_known_server_prefix(tool.name, mcp_server) - lookup_key = unprefixed or tool.name - if lookup_key in display_name_map: - tool.name = display_name_map[lookup_key] - if lookup_key in description_map: - tool.description = description_map[lookup_key] - return tools - - def _get_client_ip_from_context() -> str | None: - """ - Extract client_ip from auth context. - Returns None if context not set (caller should handle this as "no IP filtering"). - """ - try: - auth_user: Final = auth_context_var.get() - if auth_user and isinstance(auth_user, MCPAuthenticatedUser): - return auth_user.client_ip - except Exception: - pass - return None - - async def _get_allowed_mcp_servers( - user_api_key_auth: UserAPIKeyAuth | None, - mcp_servers: Sequence[str] | None, - client_ip: str | None = None, - ) -> list[MCPServer]: - """Return allowed MCP servers for a request after applying filters. - - Args: - user_api_key_auth: The authenticated user's API key info. - mcp_servers: Optional list of server names to filter to. - client_ip: Client IP for IP-based access control. If None, falls back to - auth context. Pass explicitly from request handlers for safety. - Note: If client_ip is None and auth context is not set, IP filtering is skipped. - This is intentional for internal callers but may indicate a bug if called - from a request handler without proper context setup. - """ - # Use explicit client_ip if provided, otherwise try auth context - if client_ip is None: - client_ip = _get_client_ip_from_context() - if client_ip is None: - verbose_logger.debug( - "MCP _get_allowed_mcp_servers called without client_ip and no auth context. " - "IP filtering will be skipped. This is expected for internal calls." - ) - - allowed_mcp_server_ids = await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth) - ( - allowed_mcp_server_ids, - _ip_blocked, - ) = global_mcp_server_manager.filter_server_ids_by_ip_with_info(allowed_mcp_server_ids, client_ip) - verbose_logger.debug( - "MCP IP filter: client_ip=%s, allowed_server_ids=%s", - client_ip, - allowed_mcp_server_ids, - ) - if _ip_blocked > 0: - verbose_logger.debug( - "MCP IP filtering: %d server(s) are not accessible from client IP %s " - "because they are restricted to internal networks. " - "No tools from those servers will be returned. " - "To expose a server externally, set 'available_on_public_internet: true' " - "in its configuration.", - _ip_blocked, - client_ip, - ) - allowed_mcp_servers: list[MCPServer] = [] - for allowed_mcp_server_id in allowed_mcp_server_ids: - mcp_server = global_mcp_server_manager.get_mcp_server_by_id(allowed_mcp_server_id) - if mcp_server is not None: - # Apply the request-time oauth2_flow backstop for legacy null rows. - mcp_server = MCPServerManager.resolve_oauth2_flow_for_request(mcp_server) - allowed_mcp_servers.append(mcp_server) - - if mcp_servers is not None: - allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names( - mcp_servers=mcp_servers, - allowed_mcp_servers=allowed_mcp_servers, - ) - - return allowed_mcp_servers - - def _client_has_per_server_auth_header( - server: MCPServer, - mcp_server_auth_headers: dict[str, dict[str, str]] | None, - ) -> bool: - """True if the request carries a per-server ``x-mcp-{alias}-authorization`` - header for this server. This is the multi-server binding: it names one - upstream, so it is unambiguously the caller's upstream token regardless of - auth mode (never the LiteLLM admission credential). - - Resolves through the same ``lookup_mcp_server_auth_in_headers`` egress uses, so - the connect gate and egress agree on which per-server header names match: a - dashboard client sends ``x-mcp-{sanitize_mcp_alias_for_header(alias)}-authorization``, - and matching only the raw alias here would 401 a token egress would forward. - """ - if not mcp_server_auth_headers: - return False - from litellm.proxy._experimental.mcp_server.utils import ( - lookup_mcp_server_auth_in_headers, - ) - - server_headers: Final = lookup_mcp_server_auth_in_headers( - mcp_server_auth_headers, - alias=server.alias, - server_name=server.server_name, - access_groups=server.access_groups, - ) - if isinstance(server_headers, str): - return bool(server_headers.strip()) - if isinstance(server_headers, dict): - return any(isinstance(hk, str) and hk.lower() == "authorization" for hk in server_headers) - return False - - def _client_has_passthrough_authorization( - server: MCPServer, - oauth2_headers: dict[str, str] | None, - mcp_server_auth_headers: dict[str, dict[str, str]] | None, - ) -> bool: - """True if the incoming request already carries an ``Authorization`` - header the gateway will forward to this pass-through server. - - The client may supply the bearer as either the top-level - ``Authorization`` header (surfaced via ``oauth2_headers``) or a - per-server ``x-mcp-auth-`` style header (surfaced via - ``mcp_server_auth_headers``). Either form skips the pre-emptive 401. - """ - if oauth2_headers: - for k in oauth2_headers: - if k.lower() == "authorization": - return True - return _client_has_per_server_auth_header(server, mcp_server_auth_headers) - - async def _get_user_oauth_extra_headers_from_db( - server: MCPServer, - user_api_key_auth: UserAPIKeyAuth | None, - prefetched_creds: 'Mapping[str, "OAuthCredentialPayload"] | None' = None, - ) -> dict[str, str] | None: - """Stored OAuth2 token for (user, server) as an ``Authorization: Bearer`` header, or None. - - Thin wrapper over ``resolve_user_oauth_access_token`` (Redis cache, else DB + refresh); - ``prefetched_creds`` skips the per-server Redis/DB lookups for the batch path. - """ - if server.auth_type != MCPAuth.oauth2 or user_api_key_auth is None: - return None - from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 - resolve_user_oauth_access_token, - ) - - token: Final = await resolve_user_oauth_access_token( - getattr(user_api_key_auth, "user_id", None), server, prefetched_creds - ) - return {"Authorization": f"Bearer {token}"} if token else None - - async def _prefetch_oauth_creds_for_user( - user_api_key_auth: UserAPIKeyAuth | None, - ) -> dict[str, "OAuthCredentialPayload"]: - """Fetch all OAuth2 credentials for the user in one DB query. - - Returns a dict keyed by server_id to avoid N+1 queries in asyncio.gather loops. - """ - user_id: Final[str | None] = getattr(user_api_key_auth, "user_id", None) if user_api_key_auth else None - if not user_id: - return {} - try: - from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 - list_user_oauth_credentials, - ) - from litellm.proxy.utils import get_prisma_client_or_throw # noqa: PLC0415 - - prisma_client: Final = get_prisma_client_or_throw( - "Database not connected. Connect a database to use OAuth2 MCP tools." - ) - creds: Final = await list_user_oauth_credentials(prisma_client, user_id) - return {c["server_id"]: c for c in creds if "server_id" in c} - except Exception as e: - verbose_logger.warning("_prefetch_oauth_creds_for_user: failed to prefetch for user=%s: %s", user_id, e) - return {} - - def _prepare_mcp_server_headers( - server: MCPServer, - mcp_server_auth_headers: dict[str, dict[str, str]] | None, - mcp_auth_header: str | None, - oauth2_headers: dict[str, str] | None, - raw_headers: dict[str, str] | None, - user_api_key_auth: UserAPIKeyAuth | None = None, - scope_servers: list[MCPServer] | None = None, - ) -> tuple[dict[str, str] | str | None, dict[str, str] | None]: - """Build auth and extra headers for a server. - - ``scope_servers`` is the full server list a fan-out handler iterates. Passing it lets the - client-forwarded token modes withhold the caller's request-wide ``Authorization`` when - another server in the scope would also receive it (``_caller_authorization_fans_out``); - explicitly-addressed operations leave it None. Per-server ``x-mcp-{alias}-authorization`` - headers are unaffected — they bind one token to one server and are the multi-server shape. - """ - server_auth_header: dict[str, str] | str | None = None - if mcp_server_auth_headers: - from litellm.proxy._experimental.mcp_server.utils import ( - lookup_mcp_server_auth_in_headers, - ) - - server_auth_header = lookup_mcp_server_auth_in_headers( - mcp_server_auth_headers, - alias=server.alias, - server_name=server.server_name, - access_groups=server.access_groups, - ) - - extra_headers: dict[str, str] | None = None - is_client_forwarded_mode: Final = server.is_client_forwarded_token - # In a multi-server listing scope the request-wide Authorization can only carry one token, - # so it is withheld from a client-forwarded server when another server in scope also consumes - # it (RFC 9700 cross-resource replay); such scopes must bind per-server via - # x-mcp-{alias}-authorization. The decision is computed once so BOTH the forwarding branch and - # the extra_headers copy loop below honor it — otherwise a server that lists Authorization in - # extra_headers would re-copy the withheld bearer from raw_headers and replay it anyway. - withhold_forwarded_authorization: Final = is_client_forwarded_mode and _caller_authorization_fans_out( - server, scope_servers - ) - if server.auth_type == MCPAuth.oauth2: - # For OAuth2 M2M servers, upstream Authorization must come from - # client_credentials token fetch, never from caller headers. - if server.has_client_credentials: - extra_headers = None - else: - # Copy to avoid mutating the original dict (important for parallel fetching) - extra_headers = oauth2_headers.copy() if oauth2_headers else None - # Migrated authorization_code: the v2 resolver injects the stored per-user - # token, so drop the caller-forwarded Authorization (apply-if-absent would - # otherwise let it shadow the resolved token). Delegate keeps it. Centralized - # via _should_strip_caller_authorization to match _call_regular_mcp_tool. - if extra_headers and _should_strip_caller_authorization( - mcp_server=server, - raw_headers=raw_headers, - user_api_key_auth=user_api_key_auth, - ): - extra_headers = without_header(extra_headers, DEFAULT_CREDENTIAL_HEADER) - elif is_client_forwarded_mode: - if not withhold_forwarded_authorization: - extra_headers = _client_forwarded_authorization_headers( - mcp_server=server, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - user_api_key_auth=user_api_key_auth, - ) - - if server.extra_headers and raw_headers: - if extra_headers is None: - extra_headers = {} - - normalized_raw_headers: Final = {str(k).lower(): v for k, v in raw_headers.items() if isinstance(k, str)} - - # Centralized strip decision shared with - # ``MCPServerManager._call_regular_mcp_tool`` so the two - # code paths cannot drift on this security-sensitive choice. - # See ``_should_strip_caller_authorization`` for the rules. - strip_caller_authorization: Final = _should_strip_caller_authorization( - mcp_server=server, - raw_headers=raw_headers, - user_api_key_auth=user_api_key_auth, - ) - - for header in server.extra_headers: - if not isinstance(header, str): - continue - if header.lower() == "authorization" and ( - strip_caller_authorization or withhold_forwarded_authorization - ): - continue - header_value = normalized_raw_headers.get(header.lower()) - if header_value is None: - continue - extra_headers[header] = header_value - - # Reset to None if no headers were actually added - if extra_headers is not None and len(extra_headers) == 0: - extra_headers = None - - if server_auth_header is None: - server_auth_header = mcp_auth_header - - return server_auth_header, extra_headers - - def _merge_gateway_initialize_instructions( - allowed_mcp_servers: list[MCPServer], - ) -> str | None: - """YAML/DB override, else upstream text (prefetch on init, or list_tools / health_check / call_tool cache).""" - if not allowed_mcp_servers: - return None - - texts: Final[list[tuple[str, str]]] = [] - for server in allowed_mcp_servers: - label = server.alias or server.server_name or server.name or server.server_id or "mcp" - if server.instructions and server.instructions.strip(): - texts.append((label, server.instructions.strip())) - continue - if server.spec_path: - continue - cached = global_mcp_server_manager._upstream_initialize_instructions_by_server_id.get(server.server_id) - if cached and cached.strip(): - texts.append((label, cached.strip())) - - if not texts: - return None - if len(texts) == 1: - return texts[0][1] - return "\n\n---\n\n".join(f"[{lbl}]\n{txt}" for lbl, txt in texts) - - async def _raise_if_initialize_grants_no_mcp_servers( - allowed: Sequence[MCPServer], - user_api_key_auth: UserAPIKeyAuth | None, - mcp_servers: Sequence[str] | None, - client_ip: str | None, - ) -> None: - if allowed or user_api_key_auth is None or not user_api_key_auth.api_key: - return - if mcp_servers: - await raise_denied_scoped_mcp_access( - requested_names=mcp_servers, - user_api_key_auth=user_api_key_auth, - client_ip=client_ip, - ) - no_servers_denial: Final[_McpDeniedDetail] = { - "error": ( - "The key has no MCP servers granted, or none of its granted servers is loaded and allowed for " - "this client IP. Grant servers or access groups to the key, its team, or its organization " - "(object_permission.mcp_servers), check the server's allowed IPs, and reconnect." - ) - } - raise HTTPException(status_code=403, detail=no_servers_denial) + from litellm.proxy._experimental.mcp_server.operations import ( + _client_has_passthrough_authorization, + _client_has_per_server_auth_header, + _get_allowed_mcp_servers, + _get_allowed_mcp_servers_from_mcp_server_names, + _get_user_oauth_extra_headers_from_db, + _http_detail_message, + _McpDeniedDetail, + _merge_gateway_initialize_instructions, + _prefetch_oauth_creds_for_user, + _prepare_mcp_server_headers, + _raise_if_initialize_grants_no_mcp_servers, + _server_answers_to, + _tool_name_matches, + apply_tool_overrides, + filter_tools_by_allowed_tools, + raise_denied_scoped_mcp_access, + ) @contextlib.asynccontextmanager async def _gateway_initialize_instructions_request_scope( @@ -2063,26 +984,28 @@ if MCP_AVAILABLE: scoped_server_endpoint: bool = False, is_initialize: bool = False, ) -> AsyncIterator[None]: - allowed: Final = await _get_allowed_mcp_servers( + allowed: Final = await operations._get_allowed_mcp_servers( user_api_key_auth=user_api_key_auth, mcp_servers=mcp_servers, client_ip=client_ip, ) if is_initialize: - await _raise_if_initialize_grants_no_mcp_servers(allowed, user_api_key_auth, mcp_servers, client_ip) + await operations._raise_if_initialize_grants_no_mcp_servers( + allowed, user_api_key_auth, mcp_servers, client_ip + ) if allowed: # return_exceptions=True: a per-server probe failure (incl. CancelledError # bubbled from anyio task group teardown on connection refused) must not # cancel sibling probes or 500 the gateway initialize request. await asyncio.gather( *[ - global_mcp_server_manager._ensure_upstream_initialize_instructions_cached(s) + operations.global_mcp_server_manager._ensure_upstream_initialize_instructions_cached(s) for s in allowed if s is not None ], return_exceptions=True, ) - merged: Final = _merge_gateway_initialize_instructions(allowed_mcp_servers=allowed) + merged: Final = operations._merge_gateway_initialize_instructions(allowed_mcp_servers=allowed) scoped_server_name = None if scoped_server_endpoint and len(allowed) == 1: scoped_server: Final = allowed[0] @@ -2097,1599 +1020,34 @@ if MCP_AVAILABLE: _mcp_gateway_initialize_instructions.reset(instructions_token) _mcp_gateway_server_name.reset(server_name_token) - def _aggregate_server_key(server: MCPServer) -> str: - """The client-visible key for a server in listing outcomes and spend metadata: the same - display prefix (alias, or the short prefix when that mode is enabled) the caller already - sees on the tool names. Canonical internal server names never key a caller-readable - surface; when the display naming deliberately hides them, the outcome keys must too.""" - return get_server_prefix(server) or "unknown" - - async def _get_tools_from_mcp_servers( - user_api_key_auth: UserAPIKeyAuth | None, - mcp_auth_header: str | None, - mcp_servers: list[str] | None, - mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, - oauth2_headers: dict[str, str] | None = None, - raw_headers: dict[str, str] | None = None, - log_list_tools_to_spendlogs: bool = False, - list_tools_log_source: str | None = None, - litellm_trace_id: str | None = None, - request_tags: list[str] | None = None, - client_ip: str | None = None, - mcp_proxy_mode: bool = False, - ) -> AggregateToolListing: - """ - Helper method to fetch tools from MCP servers based on server filtering criteria. - - Args: - user_api_key_auth: User authentication info for access control - mcp_auth_header: Optional auth header for MCP server (deprecated) - mcp_servers: Optional list of server names/aliases to filter by - mcp_server_auth_headers: Optional dict of server-specific auth headers - oauth2_headers: Optional dict of oauth2 headers - - Returns: - AggregateToolListing: Combined tools from filtered servers plus each server's - classified listing outcome - """ - if not MCP_AVAILABLE: - return AggregateToolListing(tools=[], outcomes={}) - - list_tools_start_time: Final = datetime.now() - litellm_logging_obj: LiteLLMLoggingObj | None = None - list_tools_request_data: dict[str, object] = {} - - if log_list_tools_to_spendlogs: - # This is intentionally minimal: only async_success_handler / post_call_failure_hook - rules_obj: Final = Rules() - list_tools_call_id: Final = str(uuid.uuid4()) - # Derive trace_id from raw_headers when not explicitly passed (same as A2A / MCP call_tool) - effective_litellm_trace_id: Final = litellm_trace_id or get_chain_id_from_headers(raw_headers) - spend_logs_metadata: Final[dict[str, object]] = { - "mcp_operation": "list_tools", - } - if isinstance(list_tools_log_source, str): - spend_logs_metadata["source"] = list_tools_log_source - if isinstance(mcp_servers, list): - spend_logs_metadata["requested_mcp_servers"] = mcp_servers - - list_tools_request_data = { - "model": "MCP: list_tools", - "call_type": CallTypes.list_mcp_tools.value, - "litellm_call_id": list_tools_call_id, - "litellm_trace_id": effective_litellm_trace_id, - "metadata": { - "spend_logs_metadata": spend_logs_metadata, - "headers": logging_safe_mcp_headers(raw_headers), - **({"tags": request_tags} if request_tags else {}), - }, - # Provide a small input payload for standard logging - "input": [ - { - "role": "system", - "content": { - "mcp_operation": "list_tools", - "requested_mcp_servers": mcp_servers, - }, - } - ], - } - - # Attach user identifiers using the standard helper - if user_api_key_auth is not None: - LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( - data=list_tools_request_data, - user_api_key_dict=user_api_key_auth, - _metadata_variable_name="metadata", - ) - - user_identifier: Final = getattr(user_api_key_auth, "end_user_id", None) or getattr( - user_api_key_auth, "user_id", None - ) - if user_identifier: - list_tools_request_data["user"] = user_identifier - - try: - litellm_logging_obj, _ = function_setup( - original_function="list_mcp_tools", - rules_obj=rules_obj, - start_time=list_tools_start_time, - **list_tools_request_data, - ) - if litellm_logging_obj: - litellm_logging_obj.call_type = CallTypes.list_mcp_tools.value - litellm_logging_obj.model = "MCP: list_tools" - except Exception as logging_error: - verbose_logger.debug("Failed to initialize logging for MCP list_tools: %s", logging_error) - litellm_logging_obj = None - - try: - allowed_mcp_servers: Final = await _get_allowed_mcp_servers( - user_api_key_auth=user_api_key_auth, - mcp_servers=mcp_servers, - client_ip=client_ip, - ) - if mcp_servers and not allowed_mcp_servers: - await raise_denied_scoped_mcp_access( - requested_names=mcp_servers, - user_api_key_auth=user_api_key_auth, - client_ip=client_ip, - ) - - # Pre-fetch OAuth credentials only when at least one server uses OAuth2, - # to avoid an unnecessary DB round-trip on requests with no OAuth2 MCP servers. - _has_oauth2_server = any(getattr(s, "auth_type", None) == MCPAuth.oauth2 for s in allowed_mcp_servers) - _prefetched_oauth_creds: Final = ( - await _prefetch_oauth_creds_for_user(user_api_key_auth) if _has_oauth2_server else {} - ) - - async def _fetch_and_filter_server_tools( - server: MCPServer, - ) -> "tuple[list[MCPTool], ServerOutcome]": - """Fetch and filter tools from a single server, classifying any failure into that - server's outcome so the aggregate can keep serving the healthy subset without a - broken server masquerading as an empty one.""" - if server is None: - return [], ServerListOk(tool_count=0) - - server_auth_header, extra_headers = _prepare_mcp_server_headers( - server=server, - mcp_server_auth_headers=mcp_server_auth_headers, - mcp_auth_header=mcp_auth_header, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - user_api_key_auth=user_api_key_auth, - scope_servers=allowed_mcp_servers, - ) - - # Prefer server-stored per-user OAuth when configured, so a stale - # Authorization header from the MCP client cannot override Redis/DB - # (same issue as call_tool in mcp_server_manager: VS Code caches tokens). - from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import ( # noqa: PLC0415 - to_server_spec, - ) - - # A server migrated to the v2 resolver gets its token from the resolver at connect - # time; building it here would double-resolve and be shadowed by the v2 graft. The - # preemptive 401 already challenged a missing token, so one exists for the connect. - migrated_to_v2: Final = to_server_spec(server) is not None - if ( - not migrated_to_v2 - and server.auth_type == MCPAuth.oauth2 - and getattr(server, "needs_user_oauth_token", False) - and user_api_key_auth is not None - ): - db_headers: Final = await _get_user_oauth_extra_headers_from_db( - server, - user_api_key_auth, - prefetched_creds=_prefetched_oauth_creds, - ) - if db_headers: - extra_headers = db_headers - - # If still no OAuth2 token, fall back to pre-fetched creds (non-stale-client path) - elif not migrated_to_v2 and extra_headers is None and server.auth_type == MCPAuth.oauth2: - extra_headers = await _get_user_oauth_extra_headers_from_db( - server, - user_api_key_auth, - prefetched_creds=_prefetched_oauth_creds, - ) - - if server.is_byok and server.auth_type != MCPAuth.oauth2 and server_auth_header is None: - server_auth_header = await _get_byok_credential(server, user_api_key_auth) - - try: - tools: Final = await global_mcp_server_manager._get_tools_from_server( - server=server, - mcp_auth_header=server_auth_header, - extra_headers=extra_headers, - add_prefix=True, # Always add server prefix - raw_headers=raw_headers, - user_api_key_auth=user_api_key_auth, - oauth2_headers=oauth2_headers, - ) - filtered_tools = filter_tools_by_allowed_tools(tools, server) - - filtered_tools = await filter_tools_by_key_team_permissions( - tools=filtered_tools, - server_id=server.server_id, - user_api_key_auth=user_api_key_auth, - ) - - if mcp_proxy_mode: - from litellm.proxy._experimental.mcp_server.tool_search import with_mcp_proxy_identity - - filtered_tools = [ # mutable-ok: MCP tool pipeline - with_mcp_proxy_identity(tool, server.server_id) for tool in filtered_tools - ] - else: - filtered_tools = apply_tool_overrides(filtered_tools, server) - - verbose_logger.debug( - "Successfully fetched %s tools from server %s, %s after filtering", - len(tools), - server.name, - len(filtered_tools), - ) - return filtered_tools, ServerListOk(tool_count=len(filtered_tools)) - except MCPUpstreamAuthError as e: - # Absorb so one unauthenticated server does not empty every other server's - # tools. Surfacing the upstream 401 to the client as a re-auth challenge is - # intentionally not done here: raising from this list handler cannot produce a - # 401 + WWW-Authenticate (the MCP session manager serializes it as a JSON-RPC - # error). Single-server routes surface it via the request-scope preemptive - # check in _raise_preemptive_401_for_unauthenticated_servers instead. - verbose_logger.debug("MCP list_tools: omitting %s; it needs upstream auth", server.name) - return [], classify_list_exception(e) - except Exception as e: - verbose_logger.exception("Error getting tools from server %s: %s", server.name, e) - return [], classify_list_exception(e) - - # Fetch tools from all servers in parallel - tasks: Final = [_fetch_and_filter_server_tools(server) for server in allowed_mcp_servers] - results: Final = await asyncio.gather(*tasks) - - # Flatten results into single list - all_tools: Final[list[MCPTool]] = [tool for tools, _ in results for tool in tools] - server_outcomes: Final[dict[str, ServerOutcome]] = { - _aggregate_server_key(server): outcome - for server, (_, outcome) in zip(allowed_mcp_servers, results) - if server is not None - } - - # If logging is enabled, enrich spend_logs_metadata with counts - if litellm_logging_obj: - per_server_tool_counts: Final[dict[str, int]] = { - _aggregate_server_key(server): len(server_tools) - for server, (server_tools, _) in zip(allowed_mcp_servers, results) - if server is not None - } - - metadata_dict: Final = litellm_logging_obj.model_call_details.get("metadata") - if isinstance(metadata_dict, dict): - spend_meta = metadata_dict.get("spend_logs_metadata") - if not isinstance(spend_meta, dict): - spend_meta = {} - metadata_dict["spend_logs_metadata"] = spend_meta - spend_meta["allowed_server_count"] = len(allowed_mcp_servers) - spend_meta["tool_count_total"] = len(all_tools) - spend_meta["per_server_tool_counts"] = per_server_tool_counts - spend_meta["per_server_list_outcomes"] = { - key: outcome_wire_value(outcome) for key, outcome in server_outcomes.items() - } - - end_time: Final = datetime.now() - try: - await litellm_logging_obj.async_success_handler( - result=[ - tool.model_dump(mode="json") if isinstance(tool, MCPTool) else tool for tool in all_tools - ], - start_time=list_tools_start_time, - end_time=end_time, - ) - except Exception as log_exc: - # list_tools responses must not be dropped due to non-blocking - # observability/serialization failures. - verbose_logger.warning( - "MCP list_tools success logging failed (continuing): %s", - log_exc, - ) - - verbose_logger.info("Successfully fetched %s tools total from all MCP servers", len(all_tools)) - - return AggregateToolListing(tools=all_tools, outcomes=server_outcomes) - except Exception as e: - # Only fire failure hook if logging was requested for this list-tools execution - if log_list_tools_to_spendlogs and user_api_key_auth is not None: - try: - from litellm.proxy.proxy_server import proxy_logging_obj - - if proxy_logging_obj: - traceback_str: Final = traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG) - await proxy_logging_obj.post_call_failure_hook( - request_data=list_tools_request_data or {}, - original_exception=e, - user_api_key_dict=user_api_key_auth, - route="/mcp/list_tools", - traceback_str=traceback_str, - ) - except Exception: - verbose_logger.debug("Failed to log MCP list_tools failure via post_call_failure_hook") - raise - - async def _get_prompts_from_mcp_servers( - user_api_key_auth: UserAPIKeyAuth | None, - mcp_auth_header: str | None, - mcp_servers: list[str] | None, - mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, - oauth2_headers: dict[str, str] | None = None, - raw_headers: dict[str, str] | None = None, - ) -> list[Prompt]: - """ - Helper method to fetch prompt from MCP servers based on server filtering criteria. - - Args: - user_api_key_auth: User authentication info for access control - mcp_auth_header: Optional auth header for MCP server (deprecated) - mcp_servers: Optional list of server names/aliases to filter by - mcp_server_auth_headers: Optional dict of server-specific auth headers - oauth2_headers: Optional dict of oauth2 headers - - Returns: - List[Prompt]: Combined list of prompts from filtered servers - """ - if not MCP_AVAILABLE: - return [] - - allowed_mcp_servers: Final = await _get_allowed_mcp_servers( - user_api_key_auth=user_api_key_auth, - mcp_servers=mcp_servers, - ) - - # Get prompts from each allowed server - all_prompts: Final = [] - for server in allowed_mcp_servers: - if server is None: - continue - - server_auth_header, extra_headers = _prepare_mcp_server_headers( - server=server, - mcp_server_auth_headers=mcp_server_auth_headers, - mcp_auth_header=mcp_auth_header, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - user_api_key_auth=user_api_key_auth, - scope_servers=allowed_mcp_servers, - ) - - try: - prompts = await global_mcp_server_manager.get_prompts_from_server( - server=server, - user_api_key_auth=user_api_key_auth, - mcp_auth_header=server_auth_header, - extra_headers=extra_headers, - add_prefix=True, # Always add server prefix - raw_headers=raw_headers, - ) - - all_prompts.extend(prompts) - - verbose_logger.debug("Successfully fetched %s prompts from server %s", len(prompts), server.name) - except Exception as e: - verbose_logger.exception("Error getting prompts from server %s: %s", server.name, e) - # Continue with other servers instead of failing completely - - verbose_logger.info("Successfully fetched %s prompts total from all MCP servers", len(all_prompts)) - - return all_prompts - - async def _get_resources_from_mcp_servers( - user_api_key_auth: UserAPIKeyAuth | None, - mcp_auth_header: str | None, - mcp_servers: list[str] | None, - mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, - oauth2_headers: dict[str, str] | None = None, - raw_headers: dict[str, str] | None = None, - ) -> list[Resource]: - """Fetch resources from allowed MCP servers.""" - - if not MCP_AVAILABLE: - return [] - - allowed_mcp_servers: Final = await _get_allowed_mcp_servers( - user_api_key_auth=user_api_key_auth, - mcp_servers=mcp_servers, - ) - - all_resources: Final[list[Resource]] = [] - for server in allowed_mcp_servers: - if server is None: - continue - - server_auth_header, extra_headers = _prepare_mcp_server_headers( - server=server, - mcp_server_auth_headers=mcp_server_auth_headers, - mcp_auth_header=mcp_auth_header, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - user_api_key_auth=user_api_key_auth, - scope_servers=allowed_mcp_servers, - ) - - try: - resources = await global_mcp_server_manager.get_resources_from_server( - server=server, - user_api_key_auth=user_api_key_auth, - mcp_auth_header=server_auth_header, - extra_headers=extra_headers, - add_prefix=True, # Always add server prefix - raw_headers=raw_headers, - ) - all_resources.extend(resources) - - verbose_logger.debug("Successfully fetched %s resources from server %s", len(resources), server.name) - except Exception as e: - verbose_logger.exception("Error getting resources from server %s: %s", server.name, e) - - verbose_logger.info("Successfully fetched %s resources total from all MCP servers", len(all_resources)) - - return all_resources - - async def _get_resource_templates_from_mcp_servers( - user_api_key_auth: UserAPIKeyAuth | None, - mcp_auth_header: str | None, - mcp_servers: list[str] | None, - mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, - oauth2_headers: dict[str, str] | None = None, - raw_headers: dict[str, str] | None = None, - ) -> list[ResourceTemplate]: - """Fetch resource templates from allowed MCP servers.""" - - if not MCP_AVAILABLE: - return [] - - allowed_mcp_servers: Final = await _get_allowed_mcp_servers( - user_api_key_auth=user_api_key_auth, - mcp_servers=mcp_servers, - ) - - all_resource_templates: Final[list[ResourceTemplate]] = [] - for server in allowed_mcp_servers: - if server is None: - continue - - server_auth_header, extra_headers = _prepare_mcp_server_headers( - server=server, - mcp_server_auth_headers=mcp_server_auth_headers, - mcp_auth_header=mcp_auth_header, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - user_api_key_auth=user_api_key_auth, - scope_servers=allowed_mcp_servers, - ) - - try: - resource_templates = await global_mcp_server_manager.get_resource_templates_from_server( - server=server, - user_api_key_auth=user_api_key_auth, - mcp_auth_header=server_auth_header, - extra_headers=extra_headers, - add_prefix=True, # Always add server prefix - raw_headers=raw_headers, - ) - all_resource_templates.extend(resource_templates) - verbose_logger.debug( - "Successfully fetched %s resource templates from server %s", - len(resource_templates), - server.name, - ) - except Exception as e: - verbose_logger.exception( - "Error getting resource templates from server %s: %s", - server.name, - str(e), - ) - - verbose_logger.info( - "Successfully fetched %s resource templates total from all MCP servers", - len(all_resource_templates), - ) - - return all_resource_templates - - async def filter_tools_by_key_team_permissions( - tools: list[MCPTool], - server_id: str, - user_api_key_auth: UserAPIKeyAuth | None, - ) -> list[MCPTool]: - """ - Filter tools based on key/team mcp_tool_permissions. - - Note: Tool names in the DB are stored without server prefixes, - but tool names from MCP servers are prefixed. We need to strip - the prefix before comparing. - """ - # Filter by key/team tool-level permissions - allowed_tool_names: Final = await MCPRequestHandler.get_allowed_tools_for_server( - server_id=server_id, - user_api_key_auth=user_api_key_auth, - ) - - # Tools arrive prefixed with the server's own prefix; strip exactly that - # prefix (resolved from the server) rather than the first separator, so a - # prefix containing the separator still reduces to the stored bare name. - server: Final = global_mcp_server_manager.get_mcp_server_by_id(server_id) - return [ - t - for t in tools - if MCPRequestHandler.tool_is_granted(strip_known_server_prefix(t.name, server), allowed_tool_names) - ] - - async def _list_mcp_tools( - user_api_key_auth: UserAPIKeyAuth | None = None, - mcp_auth_header: str | None = None, - mcp_servers: list[str] | None = None, - mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, - oauth2_headers: dict[str, str] | None = None, - raw_headers: dict[str, str] | None = None, - log_list_tools_to_spendlogs: bool = False, - list_tools_log_source: str | None = None, - client_ip: str | None = None, - mcp_proxy_mode: bool = False, - ) -> AggregateToolListing: - """ - List all available MCP tools. - - Args: - user_api_key_auth: User authentication info for access control - mcp_auth_header: Optional auth header for MCP server (deprecated) - mcp_servers: Optional list of server names/aliases to filter by - mcp_server_auth_headers: Optional dict of server-specific auth headers {server_alias: auth_value} - client_ip: Client IP for IP-based server access control - - Returns: - AggregateToolListing: Combined tools from all accessible servers plus each server's - classified listing outcome - """ - if not MCP_AVAILABLE: - return AggregateToolListing(tools=[], outcomes={}) - - try: - listing: Final = await _get_tools_from_mcp_servers( - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_servers=mcp_servers, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - log_list_tools_to_spendlogs=log_list_tools_to_spendlogs, - list_tools_log_source=list_tools_log_source, - client_ip=client_ip, - mcp_proxy_mode=mcp_proxy_mode, - ) - verbose_logger.debug("Successfully fetched %s tools from managed MCP servers", len(listing.tools)) - return listing - except HTTPException: - raise - except Exception as e: - verbose_logger.exception("Error getting tools from managed MCP servers: %s", e) - # Continue with an empty listing instead of failing completely - return AggregateToolListing(tools=[], outcomes={}) - - async def _list_mcp_prompts( - user_api_key_auth: UserAPIKeyAuth | None = None, - mcp_auth_header: str | None = None, - mcp_servers: list[str] | None = None, - mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, - oauth2_headers: dict[str, str] | None = None, - raw_headers: dict[str, str] | None = None, - ) -> list[Prompt]: - """ - List all available MCP prompts. - - Args: - user_api_key_auth: User authentication info for access control - mcp_auth_header: Optional auth header for MCP server (deprecated) - mcp_servers: Optional list of server names/aliases to filter by - mcp_server_auth_headers: Optional dict of server-specific auth headers {server_alias: auth_value} - - Returns: - List[Prompt]: Combined list of tools from all accessible servers - """ - if not MCP_AVAILABLE: - return [] - # Get tools from managed MCP servers with error handling - managed_prompts = [] - try: - managed_prompts = await _get_prompts_from_mcp_servers( - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_servers=mcp_servers, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - ) - verbose_logger.debug("Successfully fetched %s prompts from managed MCP servers", len(managed_prompts)) - except Exception as e: - verbose_logger.exception("Error getting tools from managed MCP servers: %s", e) - # Continue with empty managed tools list instead of failing completely - - return managed_prompts - - async def _list_mcp_resources( - user_api_key_auth: UserAPIKeyAuth | None = None, - mcp_auth_header: str | None = None, - mcp_servers: list[str] | None = None, - mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, - oauth2_headers: dict[str, str] | None = None, - raw_headers: dict[str, str] | None = None, - ) -> list[Resource]: - """List all available MCP resources.""" - - if not MCP_AVAILABLE: - return [] - - managed_resources: list[Resource] = [] - try: - managed_resources = await _get_resources_from_mcp_servers( - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_servers=mcp_servers, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - ) - verbose_logger.debug("Successfully fetched %s resources from managed MCP servers", len(managed_resources)) - except Exception as e: - verbose_logger.exception("Error getting resources from managed MCP servers: %s", e) - - return managed_resources - - async def _list_mcp_resource_templates( - user_api_key_auth: UserAPIKeyAuth | None = None, - mcp_auth_header: str | None = None, - mcp_servers: list[str] | None = None, - mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, - oauth2_headers: dict[str, str] | None = None, - raw_headers: dict[str, str] | None = None, - ) -> list[ResourceTemplate]: - """List all available MCP resource templates.""" - - if not MCP_AVAILABLE: - return [] - - managed_resource_templates: list[ResourceTemplate] = [] - try: - managed_resource_templates = await _get_resource_templates_from_mcp_servers( - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_servers=mcp_servers, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - ) - verbose_logger.debug( - "Successfully fetched %s resource templates from managed MCP servers", - len(managed_resource_templates), - ) - except Exception as e: - verbose_logger.exception( - "Error getting resource templates from managed MCP servers: %s", - str(e), - ) - - return managed_resource_templates - - def _resolve_display_name_to_original( - name: str, - allowed_mcp_servers: list[MCPServer], - ) -> str: - """Translate a display-name override back to the original prefixed tool name. - - When a client received a customised display name from tools/list (e.g. - "Get Pet") it will call tools/call with that same string. We need to - reverse-map it to the original prefixed name (e.g. - "petstore_mcp-getPetById") before any routing or permission logic runs. - """ - for server in allowed_mcp_servers: - display_map = server.tool_name_to_display_name or {} - for unprefixed_name, display_name in display_map.items(): - if display_name == name: - return add_server_prefix_to_name(unprefixed_name, get_server_prefix(server)) - return name - - async def _get_byok_credential( - mcp_server: MCPServer, - user_api_key_auth: UserAPIKeyAuth | None, - ) -> str | None: - """Retrieve the stored BYOK credential for a user+server pair, served from the worker cache within its TTL.""" - if not mcp_server.is_byok: - return None - user_id: Final = (user_api_key_auth.user_id if user_api_key_auth else None) or "" - if not user_id: - return None - - cached: Final = get_cached_byok_credential(user_id, mcp_server.server_id) - if cached is not None: - return cached.credential - - from litellm.proxy._experimental.mcp_server.db import get_user_credential - from litellm.proxy.proxy_server import prisma_client - - if prisma_client is None: - return None - credential: Final = await get_user_credential( - prisma_client=prisma_client, - user_id=user_id, - server_id=mcp_server.server_id, - ) - cache_byok_credential(user_id, mcp_server.server_id, credential) - return credential - - async def _check_byok_credential( - mcp_server: MCPServer, - user_api_key_auth: UserAPIKeyAuth | None, - ) -> None: - """ - If the MCP server is BYOK-enabled, verify that the requesting user has a - stored credential. When no credential is found, raise an HTTP 401 with a - WWW-Authenticate header that points the MCP client to our OAuth metadata - endpoint so it can drive the authorization flow. - """ - if not mcp_server.is_byok: - return - - user_id: Final = (user_api_key_auth.user_id if user_api_key_auth else None) or "" - if not user_id: - raise HTTPException( - status_code=401, - detail={ - "error": "byok_auth_required", - "server_id": mcp_server.server_id, - "server_name": mcp_server.server_name or mcp_server.name, - "message": "User identity is required for BYOK servers", - }, - headers={"WWW-Authenticate": get_byok_www_authenticate()}, - ) - - cached: Final = get_cached_byok_credential(user_id, mcp_server.server_id) - if cached is not None: - if cached.credential is None: - raise HTTPException( - status_code=401, - detail={ - "error": "byok_auth_required", - "server_id": mcp_server.server_id, - "server_name": mcp_server.server_name or mcp_server.name, - "message": ( - "No stored credential found for this BYOK server. " - "Complete the OAuth authorization flow to provide your API key." - ), - }, - headers={"WWW-Authenticate": get_byok_www_authenticate()}, - ) - return - - from litellm.proxy._experimental.mcp_server.db import get_user_credential - from litellm.proxy.proxy_server import prisma_client - - if prisma_client is None: - # Fail closed on DB unavailability: returning here previously - # bypassed the ownership check and let any proxy-authenticated - # caller invoke BYOK tools during outage windows. - raise HTTPException( - status_code=503, - detail={ - "error": "byok_auth_unavailable", - "server_id": mcp_server.server_id, - "server_name": mcp_server.server_name or mcp_server.name, - "message": "BYOK credential check requires a database connection.", - }, - ) - - credential: Final = await get_user_credential( - prisma_client=prisma_client, - user_id=user_id, - server_id=mcp_server.server_id, - ) - cache_byok_credential(user_id, mcp_server.server_id, credential) - if credential is None: - raise HTTPException( - status_code=401, - detail={ - "error": "byok_auth_required", - "server_id": mcp_server.server_id, - "server_name": mcp_server.server_name or mcp_server.name, - "message": ( - "No stored credential found for this BYOK server. " - "Complete the OAuth authorization flow to provide your API key." - ), - }, - headers={"WWW-Authenticate": get_byok_www_authenticate()}, - ) - - async def _list_tools_before_first_call( - server: MCPServer | None, - tool_name: str, - allowed_mcp_servers: list[MCPServer], - user_api_key_auth: UserAPIKeyAuth | None, - mcp_auth_header: str | None, - mcp_server_auth_headers: dict[str, dict[str, str]] | None, - oauth2_headers: dict[str, str] | None, - raw_headers: dict[str, str] | None, - ) -> None: - """List ``server`` with the caller's own credentials when it does not yet expose ``tool_name`` here. - - The startup fill skips a server whose upstream wants the caller's token, and mcp 2 no - longer lists before an uncached tools/call, so a worker that has not served tools/list - for this caller would otherwise answer 404 for a tool the caller can see. Gating on the - requested tool, not on any prior listing, keeps callers with different upstream catalogs - from masking each other. - """ - if server is None or global_mcp_server_manager.server_exposes_tool(server, tool_name): - return - if all(allowed.server_id != server.server_id for allowed in allowed_mcp_servers): - return - try: - await _get_tools_from_mcp_servers( - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_servers=[server.server_id], - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - ) - except Exception as e: # noqa: BLE001 # best effort: resolution below answers as it did before - verbose_logger.debug("MCP tools/call: listing %s before its first call failed: %s", server.name, e) - - async def execute_mcp_tool( - name: str, - arguments: dict[str, object], - allowed_mcp_servers: list[MCPServer], - start_time: datetime, - user_api_key_auth: UserAPIKeyAuth | None = None, - mcp_auth_header: str | None = None, - mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, - oauth2_headers: dict[str, str] | None = None, - raw_headers: dict[str, str] | None = None, - host_progress_callback: Callable | None = None, - guardrail_context: Mapping[str, object] | None = None, - **kwargs: Any, - ) -> CallToolResult: - """ - Execute MCP tool. - - This function assumes permission checks have already been performed. - - Args: - name: Tool name (may include server prefix) - arguments: Tool arguments - allowed_mcp_servers: Pre-validated list of servers the user can access - start_time: Start time for logging - user_api_key_auth: Optional user API key auth for logging - mcp_auth_header: Optional MCP auth header - mcp_server_auth_headers: Optional server-specific auth headers - oauth2_headers: Optional OAuth2 headers - raw_headers: Optional raw HTTP headers - **kwargs: Additional arguments (e.g., litellm_logging_obj) - - Returns: - CallToolResult: Tool execution result - """ - # Track resolved MCP server for both permission checks and dispatch - mcp_server: MCPServer | None = None - requested_server_id: Final[str | None] = kwargs.get("requested_server_id") - - # If the client called with a display-name override (e.g. "Get Pet"), - # translate it back to the original prefixed name before any routing. - name = _resolve_display_name_to_original(name, allowed_mcp_servers) - - # Remove prefix from tool name for logging and processing - original_tool_name, server_name = split_server_prefix_from_name(name) - - requested_server: MCPServer | None = None - if requested_server_id: - requested_server = next( - (s for s in allowed_mcp_servers if s.server_id == requested_server_id), - None, - ) - - name_is_prefixed = False - if requested_server is not None and MCP_TOOL_PREFIX_SEPARATOR in name: - all_registry_prefixes: Final[set[str]] = set() - for registry_server in global_mcp_server_manager.get_registry().values(): - for known_prefix in iter_known_server_prefixes(registry_server): - all_registry_prefixes.add(normalize_server_name(known_prefix)) - name_is_prefixed = is_tool_name_prefixed(name, known_server_prefixes=all_registry_prefixes) - - first_call_target: Final = ( - requested_server - if requested_server is not None and not name_is_prefixed - else global_mcp_server_manager.server_owning_tool_name_prefix(name) - ) - first_call_tool_name: Final = ( - name - if first_call_target is None or (requested_server is not None and not name_is_prefixed) - else strip_known_server_prefix(name, first_call_target) - ) - await _list_tools_before_first_call( - server=first_call_target, - tool_name=first_call_tool_name, - allowed_mcp_servers=allowed_mcp_servers, - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - ) - - if requested_server is not None and not name_is_prefixed: - # REST callers may pass server_id with the upstream tool name (no - # LiteLLM prefix). The first segment is not a registered server - # prefix, so the whole string is the upstream tool name and may - # legitimately contain the separator (e.g. "text-to-speech"). - # server_id is authoritative for routing and auth. - mcp_server = requested_server - server_name = requested_server.name - original_tool_name = name - else: - # Resolve from tool name (MCP JSON-RPC or prefixed REST tool names). - mcp_server = global_mcp_server_manager._get_mcp_server_from_tool_name(name) - if mcp_server is None and requested_server is not None: - for known_prefix in iter_known_server_prefixes(requested_server): - candidate = global_mcp_server_manager._get_mcp_server_from_tool_name( - add_server_prefix_to_name(name, known_prefix) - ) - if candidate is not None: - mcp_server = candidate - break - if mcp_server is not None: - server_name = mcp_server.name - original_tool_name = strip_known_server_prefix(name, mcp_server) - - if requested_server is not None: - if mcp_server is not None and mcp_server.server_id != requested_server.server_id: - raise HTTPException( - status_code=403, - detail={ - "error": "tool_server_mismatch", - "message": ( - f"Tool '{name}' belongs to MCP server " - f"'{mcp_server.name}' but request specified " - f"server_id for '{requested_server.name}'." - ), - }, - ) - if mcp_server is None: - mcp_server = requested_server - server_name = requested_server.name - original_tool_name = strip_known_server_prefix(name, requested_server) - - # Only enforce server-level permissions when we can resolve a server - if server_name: - if not MCPRequestHandler.is_tool_allowed( - allowed_mcp_servers=[server.name for server in allowed_mcp_servers], - server_name=server_name, - ): - raise HTTPException( - status_code=403, - detail="User not allowed to call this tool.", - ) - - standard_logging_mcp_tool_call: Final[StandardLoggingMCPToolCall] = _get_standard_logging_mcp_tool_call( - name=original_tool_name, # Use original name for logging - arguments=arguments, - server_name=server_name, - session_id=_mcp_session_id_from_headers(raw_headers), - ) - litellm_logging_obj: Final[LiteLLMLoggingObj | None] = kwargs.get("litellm_logging_obj", None) - if litellm_logging_obj: - litellm_logging_obj.model_call_details["mcp_tool_call_metadata"] = standard_logging_mcp_tool_call - litellm_logging_obj.model = f"MCP: {name}" - litellm_logging_obj.model_call_details["model"] = f"MCP: {name}" - # Resolve the MCP server early so BYOK checks and credential injection - # apply to ALL dispatch paths (local tool registry AND managed MCP server). - if mcp_server is None: - mcp_server = global_mcp_server_manager._get_mcp_server_from_tool_name(name) - - if mcp_server: - standard_logging_mcp_tool_call["mcp_server_cost_info"] = (mcp_server.mcp_info or {}).get( - "mcp_server_cost_info" - ) - if litellm_logging_obj: - litellm_logging_obj.model_call_details["mcp_tool_call_metadata"] = standard_logging_mcp_tool_call - - # BYOK: retrieve the stored per-user credential. A single DB call - # both checks existence and fetches the value, avoiding a double query. - if mcp_server.is_byok and not mcp_auth_header: - byok_cred: Final = await _get_byok_credential(mcp_server, user_api_key_auth) - if byok_cred is None: - raise HTTPException( - status_code=401, - detail={ - "error": "byok_auth_required", - "server_id": mcp_server.server_id, - "server_name": mcp_server.server_name or mcp_server.name, - "message": ( - "No stored credential found for this BYOK server. " - "Complete the OAuth authorization flow to provide your API key." - ), - }, - headers={"WWW-Authenticate": get_byok_www_authenticate()}, - ) - mcp_auth_header = byok_cred - elif mcp_server.is_byok: - # External auth header supplied; still enforce user-identity check. - await _check_byok_credential(mcp_server, user_api_key_auth) - - # Check if tool exists in local registry first (for OpenAPI-based tools) - # These tools are registered with their prefixed names - ######################################################### - local_tool: Final = global_mcp_tool_registry.get_tool(name) - if local_tool: - # OpenAPI-backed tools used to bypass `pre_call_tool_check` — - # only the managed path ran allowed/banned-tool checks, key/team - # tool permissions, and parameter validation. Run the same checks - # before dispatching to the local registry. Refuse the call if - # we cannot resolve a server: tools registered via - # openapi_to_mcp_generator are always tied to a server, so a - # missing mcp_server here means the tool->server mapping has - # not finished initializing or the registry entry is orphaned. - # Skipping the check would re-open the same authorization gap. - if mcp_server is None: - raise HTTPException( - status_code=503, - detail=( - f"MCP server for tool '{name}' is not available; " - "refusing to dispatch without authorization checks. " - "Retry once the server is registered." - ), - ) - - # `pre_call_tool_check` calls into `proxy_logging_obj` for the - # pre-call guardrail hooks, so source it from the canonical - # `proxy_server` module the same way `_handle_managed_mcp_tool` - # does. `kwargs.get("proxy_logging_obj")` is None on the MCP - # entry path and would crash with AttributeError after the - # security checks pass. - from litellm.proxy.proxy_server import proxy_logging_obj - - hook_result = await global_mcp_server_manager.pre_call_tool_check( - name=original_tool_name, - arguments=arguments or {}, - server_name=server_name or mcp_server.name, - user_api_key_auth=user_api_key_auth, - proxy_logging_obj=proxy_logging_obj, - server=mcp_server, - raw_headers=raw_headers, - litellm_logging_obj=litellm_logging_obj, - guardrail_context=guardrail_context, - ) - # `pre_call_tool_check` may return guardrail-modified - # arguments; honor them on the local path too. - if isinstance(hook_result, dict) and "arguments" in hook_result: - arguments = hook_result["arguments"] - - verbose_logger.debug("Executing local registry tool: %s", name) - # The credential rides ContextVars because the tool function has its - # headers baked into the closure at registration time. - auth_header_value, openapi_forwarded_headers, upstream_credential = _resolve_openapi_tool_auth( - mcp_server=mcp_server, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, - raw_headers=raw_headers, - user_api_key_auth=user_api_key_auth, - ) - ( - resolved_auth_headers, - forwarded_headers, - ) = await global_mcp_server_manager.resolve_openapi_upstream_auth( - mcp_server=mcp_server, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - mcp_auth_header=upstream_credential, - user_api_key_auth=user_api_key_auth, - forwarded_headers=openapi_forwarded_headers, - ) - - _auth_token: Final = _request_auth_header.set(auth_header_value) - _extra_token: Final = _request_extra_headers.set(forwarded_headers) - _resolved_token: Final = _request_resolved_auth_headers.set(resolved_auth_headers) - try: - response = await _handle_local_mcp_tool(name, arguments) - finally: - _request_auth_header.reset(_auth_token) - _request_extra_headers.reset(_extra_token) - _request_resolved_auth_headers.reset(_resolved_token) - - # Try managed MCP server tool (the name is bare; the prefix boundary was - # already resolved above against this server's registered prefixes) - # Primary and recommended way to use external MCP servers - ######################################################### - elif mcp_server: - response = await _handle_managed_mcp_tool( - server_name=server_name, - name=original_tool_name, - arguments=arguments, - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - litellm_logging_obj=litellm_logging_obj, - guardrail_context=guardrail_context, - host_progress_callback=host_progress_callback, - ) - - # Fall back to local tool registry with original name (legacy support) - ######################################################### - # Deprecated: Local MCP Server Tool - ######################################################### - else: - # Gate only what can actually dispatch. When the unprefixed name is - # not in the registry either, `_handle_local_mcp_tool` below reports - # 404 and nothing runs, so demanding a server here would turn every - # unknown tool name into a misleading 503. - if global_mcp_tool_registry.get_tool(original_tool_name) is not None: - # `mcp_server` is None here because the tool name is not in the - # tool -> server mapping, but the name still carries a prefix - # that the server-level check above compared against the - # caller's `allowed_mcp_servers` by exact `name`. So the named - # server is in that list and can carry the tool-level checks, - # even with the mapping cold. Resolve it from - # `allowed_mcp_servers` rather than the registry: the registry - # would happily return a server the caller holds no grant for, - # and matching anything other than `name` would accept a server - # the check never validated. - prefix_server: Final = next( - (candidate for candidate in allowed_mcp_servers if candidate.name == server_name), - None, - ) - if prefix_server is None: - # A non-empty prefix that passed the server-level check - # always matches here, so this arm only fires when the - # prefix was empty, which is exactly the case that check - # skips. Fail closed rather than dispatch with no server to - # evaluate a tool ceiling against. - raise HTTPException( - status_code=503, - detail=( - f"MCP server for tool '{original_tool_name}' is not available; " - "refusing to dispatch without authorization checks. " - "Retry once the server is registered." - ), - ) - - from litellm.proxy.proxy_server import proxy_logging_obj - - hook_result = await global_mcp_server_manager.pre_call_tool_check( - name=original_tool_name, - arguments=arguments, - server_name=server_name, - user_api_key_auth=user_api_key_auth, - proxy_logging_obj=proxy_logging_obj, - server=prefix_server, - raw_headers=raw_headers, - litellm_logging_obj=litellm_logging_obj, - guardrail_context=guardrail_context, - ) - if "arguments" in hook_result: - arguments = hook_result["arguments"] # pyright: ignore[reportAny] # hook returns untyped args - - response = await _handle_local_mcp_tool(original_tool_name, arguments) - - return await _run_post_mcp_call_guardrails( - result=response, - litellm_logging_obj=litellm_logging_obj, - user_api_key_auth=user_api_key_auth, - request_data=kwargs, - ) - - async def _run_post_mcp_call_guardrails( - result: CallToolResult, - litellm_logging_obj: LiteLLMLoggingObj | None, - user_api_key_auth: UserAPIKeyAuth | None, - request_data: Mapping[str, object], - ) -> CallToolResult: - """Run ``post_mcp_call`` guardrails over an executed tool result. - - Lives on ``execute_mcp_tool``'s return path rather than inside - ``_fire_mcp_tool_call_logging`` so enforcement never depends on logging - being configured, and so every dispatch route gets it: the MCP protocol - handler, the REST endpoint, and tool search all funnel through here. - A guardrail that rejects the result raises, matching ``pre_mcp_call``. - """ - from litellm.proxy.proxy_server import proxy_logging_obj - - if proxy_logging_obj is None: - return result - return await proxy_logging_obj.post_mcp_call_hook( - response=result, - request_data=( - litellm_logging_obj.model_call_details if litellm_logging_obj is not None else dict(request_data) - ), - user_api_key_dict=user_api_key_auth, - ) - - _MCP_CREDENTIAL_REQUEST_FIELDS: Final = frozenset( - { - "raw_headers", - "mcp_auth_header", - "mcp_server_auth_headers", - "oauth2_headers", - "user_api_key_auth", - } + from litellm.proxy._experimental.mcp_server.operations import ( + _MCP_CREDENTIAL_REQUEST_FIELDS, + _aggregate_server_key, + _check_byok_credential, + _fire_mcp_tool_call_logging, + _get_byok_credential, + _get_prompts_from_mcp_servers, + _get_resource_templates_from_mcp_servers, + _get_resources_from_mcp_servers, + _get_standard_logging_mcp_tool_call, + _get_tools_from_mcp_servers, + _handle_local_mcp_tool, + _handle_managed_mcp_tool, + _list_mcp_prompts, + _list_mcp_resource_templates, + _list_mcp_resources, + _list_mcp_tools, + _list_tools_before_first_call, + _resolve_display_name_to_original, + _run_post_mcp_call_guardrails, + call_mcp_tool, + execute_mcp_tool, + filter_tools_by_key_team_permissions, + fire_mcp_tool_call_failure_logging, + mcp_get_prompt, + mcp_read_resource, ) - async def _fire_mcp_tool_call_logging( - logging_obj: LiteLLMLoggingObj, - result: CallToolResult, - start_time: datetime, - end_time: datetime, - user_api_key_auth: UserAPIKeyAuth | None = None, - request_data: Mapping[str, object] | None = None, - ) -> CallToolResult: - """Fire post-call logging for an executed MCP tool call, returning the result to send. - - The returned result is what the caller must forward to the client: a - ``post_mcp_call`` guardrail may rewrite the tool output (e.g. mask - sensitive values) or reject it, in which case its exception propagates. - Guardrails run before the success/failure logging so the masked text, not - the raw one, is what gets logged. - - A result with ``is_error=True`` is logged as a failure (``status="failure"`` - payload, so OTel marks the span ERROR) while the HTTP wire behavior stays - 200 + ``isError: true`` per the MCP spec. The error check runs after - ``async_post_mcp_tool_call_hook`` because guardrails may flip the result - to ``is_error=True`` in that hook. Raised exceptions never reach here (the - ``@client`` wrapper and ``call_mcp_tool``'s except path log those), so - this cannot double-log a failure. - - ``request_data`` may carry credential-bearing fields (the REST path puts - ``raw_headers``, ``mcp_auth_header``, ``mcp_server_auth_headers``, and - ``oauth2_headers`` at the top level of its data dict), so those are - stripped before the dict is handed to ``post_call_failure_hook`` - callbacks. - """ - from litellm.proxy.proxy_server import proxy_logging_obj - - logging_obj.post_call(original_response=result) - await logging_obj.async_post_mcp_tool_call_hook( - kwargs=logging_obj.model_call_details, - response_obj=result, - start_time=start_time, - end_time=end_time, - ) - logging_obj.call_type = CallTypes.call_mcp_tool.value - error_message: Final = extract_mcp_tool_result_error_message(result) - if error_message is None: - await logging_obj.async_success_handler(result=result, start_time=start_time, end_time=end_time) - return result - - logging_obj.has_run_logging(event_type="sync_success") - logging_obj.has_run_logging(event_type="async_success") - tool_error: Final = MCPToolResultError(error_message) - logging_obj.failure_handler(tool_error, "", start_time, end_time) - await logging_obj.async_failure_handler(tool_error, "", start_time, end_time) - - if user_api_key_auth is None: - return result - - if proxy_logging_obj: - sanitized_request_data: Final = { - key: value for key, value in (request_data or {}).items() if key not in _MCP_CREDENTIAL_REQUEST_FIELDS - } - await proxy_logging_obj.post_call_failure_hook( - request_data=sanitized_request_data, - original_exception=tool_error, - user_api_key_dict=user_api_key_auth, - route="/mcp/call_tool", - ) - return result - - async def fire_mcp_tool_call_failure_logging( - logging_obj: LiteLLMLoggingObj | None, - exception: Exception, - start_time: datetime, - user_api_key_auth: UserAPIKeyAuth | None, - request_data: Mapping[str, object], - ) -> None: - """Failure logging shared by the ``/mcp`` path and the REST endpoint. Call from - inside the ``except`` block so the traceback is still available. - - The failure handlers run first because ``_ProxyDBLogger.async_post_call_failure_hook`` - builds the failure spend-log row from the ``standard_logging_object`` they produce; - both gate on ``should_run_logging``, so the ``@client`` wrapper does not log twice. - A relayed upstream 401 (``MCPUpstreamAuthError``) is an expected caller-must-reauth - signal and skips ``post_call_failure_hook``, which fires the ``llm_exceptions`` alert. - """ - from litellm.proxy.proxy_server import proxy_logging_obj - - traceback_str: Final = traceback.format_exc(limit=MAXIMUM_TRACEBACK_LINES_TO_LOG) - if logging_obj is not None: - end_time: Final = datetime.now() # noqa: DTZ005 # naive to match `start_time`, which it is subtracted from - logging_obj.failure_handler(exception, traceback_str, start_time, end_time) - await logging_obj.async_failure_handler(exception, traceback_str, start_time, end_time) - - if isinstance(exception, MCPUpstreamAuthError) or not proxy_logging_obj or user_api_key_auth is None: - return - sanitized_request_data: Final = { - key: value for key, value in request_data.items() if key not in _MCP_CREDENTIAL_REQUEST_FIELDS - } - await proxy_logging_obj.post_call_failure_hook( - request_data=sanitized_request_data, - original_exception=exception, - user_api_key_dict=user_api_key_auth, - route="/mcp/call_tool", - traceback_str=traceback_str, - ) - - @client - async def call_mcp_tool( - name: str, - arguments: dict[str, object] | None = None, - user_api_key_auth: UserAPIKeyAuth | None = None, - mcp_auth_header: str | None = None, - mcp_servers: list[str] | None = None, - mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, - oauth2_headers: dict[str, str] | None = None, - raw_headers: dict[str, str] | None = None, - client_ip: str | None = None, - **kwargs: Any, - ) -> CallToolResult: - """ - Call a specific tool with the provided arguments (handles prefixed tool names). - """ - start_time: Final = datetime.now() - litellm_logging_obj: Final[LiteLLMLoggingObj | None] = kwargs.get("litellm_logging_obj", None) - - try: - if arguments is None: - raise HTTPException(status_code=400, detail="Request arguments are required") - - ## CHECK IF USER IS ALLOWED TO CALL THIS TOOL - allowed_mcp_server_ids: Final = await global_mcp_server_manager.get_allowed_mcp_servers( - user_api_key_auth=user_api_key_auth, - ) - - allowed_mcp_servers: list[MCPServer] = [] - for allowed_mcp_server_id in allowed_mcp_server_ids: - allowed_server = global_mcp_server_manager.get_mcp_server_by_id(allowed_mcp_server_id) - if allowed_server is not None: - # Same request-time oauth2_flow backstop the listing path applies, - # so a null-flow M2M-shape row is treated as M2M on tool calls too. - allowed_server = MCPServerManager.resolve_oauth2_flow_for_request(allowed_server) - allowed_mcp_servers.append(allowed_server) - - allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names( - mcp_servers=mcp_servers, - allowed_mcp_servers=allowed_mcp_servers, - ) - if mcp_servers and not allowed_mcp_servers: - await raise_denied_scoped_mcp_access( - requested_names=mcp_servers, - user_api_key_auth=user_api_key_auth, - client_ip=client_ip, - ) - if not allowed_mcp_servers: - raise HTTPException( - status_code=403, - detail="User not allowed to call this tool.", - ) - - # Delegate to execute_mcp_tool for execution - response = await execute_mcp_tool( - name=name, - arguments=arguments, - allowed_mcp_servers=allowed_mcp_servers, - start_time=start_time, - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - **kwargs, - ) - except Exception as e: - await fire_mcp_tool_call_failure_logging(litellm_logging_obj, e, start_time, user_api_key_auth, kwargs) - raise - - if litellm_logging_obj: - response = await _fire_mcp_tool_call_logging( - logging_obj=litellm_logging_obj, - result=response, - start_time=start_time, - end_time=datetime.now(), - user_api_key_auth=user_api_key_auth, - request_data=kwargs, - ) - return response - - async def mcp_get_prompt( - name: str, - arguments: dict[str, object] | None = None, - user_api_key_auth: UserAPIKeyAuth | None = None, - mcp_auth_header: str | None = None, - mcp_servers: list[str] | None = None, - mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, - oauth2_headers: dict[str, str] | None = None, - raw_headers: dict[str, str] | None = None, - ) -> GetPromptResult: - """ - Fetch a specific MCP prompt, handling both prefixed and unprefixed names. - """ - allowed_mcp_servers: Final = await _get_allowed_mcp_servers( - user_api_key_auth=user_api_key_auth, - mcp_servers=mcp_servers, - ) - - if not allowed_mcp_servers: - raise HTTPException( - status_code=403, - detail="User not allowed to get this prompt.", - ) - - # Extract server name from prefixed prompt name - original_prompt_name, server_name = split_server_prefix_from_name(name) - - server: Final = next((s for s in allowed_mcp_servers if s.name == server_name), None) - if server is None: - raise HTTPException( - status_code=403, - detail="User not allowed to get this prompt.", - ) - - server_auth_header, extra_headers = _prepare_mcp_server_headers( - server=server, - mcp_server_auth_headers=mcp_server_auth_headers, - mcp_auth_header=mcp_auth_header, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - user_api_key_auth=user_api_key_auth, - ) - - return await global_mcp_server_manager.get_prompt_from_server( - server=server, - user_api_key_auth=user_api_key_auth, - prompt_name=original_prompt_name, - arguments=arguments, - mcp_auth_header=server_auth_header, - extra_headers=extra_headers, - raw_headers=raw_headers, - ) - - async def mcp_read_resource( - url: AnyUrl, - user_api_key_auth: UserAPIKeyAuth | None = None, - mcp_auth_header: str | None = None, - mcp_servers: list[str] | None = None, - mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, - oauth2_headers: dict[str, str] | None = None, - raw_headers: dict[str, str] | None = None, - ) -> ReadResourceResult: - """Read resource contents from upstream MCP servers.""" - - allowed_mcp_servers: Final = await _get_allowed_mcp_servers( - user_api_key_auth=user_api_key_auth, - mcp_servers=mcp_servers, - ) - - if not allowed_mcp_servers: - raise HTTPException( - status_code=403, - detail="User not allowed to read this resource.", - ) - - if len(allowed_mcp_servers) != 1: - raise HTTPException( - status_code=400, - detail=( - "Multiple MCP servers configured; read_resource currently supports exactly one allowed server." - ), - ) - - server: Final = allowed_mcp_servers[0] - - server_auth_header, extra_headers = _prepare_mcp_server_headers( - server=server, - mcp_server_auth_headers=mcp_server_auth_headers, - mcp_auth_header=mcp_auth_header, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - user_api_key_auth=user_api_key_auth, - ) - - return await global_mcp_server_manager.read_resource_from_server( - server=server, - user_api_key_auth=user_api_key_auth, - url=url, - mcp_auth_header=server_auth_header, - extra_headers=extra_headers, - raw_headers=raw_headers, - ) - - def _get_standard_logging_mcp_tool_call( - name: str, - arguments: dict[str, object], - server_name: str | None, - session_id: str | None = None, - ) -> StandardLoggingMCPToolCall: - mcp_server: Final = global_mcp_server_manager._get_mcp_server_from_tool_name( - add_server_prefix_to_name(name, server_name) if server_name else name - ) - namespaced_tool_name: Final = f"{server_name}/{name}" if server_name else name - if mcp_server: - mcp_info: Final = mcp_server.mcp_info or {} - return StandardLoggingMCPToolCall( - name=name, - arguments=arguments, - mcp_server_name=mcp_info.get("server_name"), - mcp_server_logo_url=mcp_info.get("logo_url"), - namespaced_tool_name=namespaced_tool_name, - mcp_session_id=session_id, - mcp_auth_mode=mcp_server.auth_type, - mcp_server_resource=_redact_mcp_resource_url(mcp_server.url), - ) - else: - return StandardLoggingMCPToolCall( - name=name, - arguments=arguments, - namespaced_tool_name=namespaced_tool_name, - mcp_session_id=session_id, - ) - - async def _handle_managed_mcp_tool( - server_name: str, - name: str, - arguments: dict[str, object], - user_api_key_auth: UserAPIKeyAuth | None = None, - mcp_auth_header: str | None = None, - mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, - oauth2_headers: dict[str, str] | None = None, - raw_headers: dict[str, str] | None = None, - litellm_logging_obj: LiteLLMLoggingObj | None = None, - host_progress_callback: Callable | None = None, - guardrail_context: Mapping[str, object] | None = None, - ) -> CallToolResult: - """Handle tool execution for managed server tools""" - # Import here to avoid circular import - from litellm.proxy.proxy_server import proxy_logging_obj - - call_tool_result: Final = await global_mcp_server_manager.call_tool( - server_name=server_name, - name=name, - arguments=arguments, - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - proxy_logging_obj=proxy_logging_obj, - host_progress_callback=host_progress_callback, - litellm_logging_obj=litellm_logging_obj, - guardrail_context=guardrail_context, - ) - verbose_logger.debug("CALL TOOL RESULT: %s", call_tool_result) - return call_tool_result - - async def _handle_local_mcp_tool(name: str, arguments: dict[str, object]) -> CallToolResult: - """Execute a local-registry tool and report whether it succeeded. - - Returns the result rather than bare content because the verdict is part of it: the content - alone cannot say whether the handler failed, so callers used to stamp is_error=False on every - outcome and an upstream rejection was served as tool output. - - A failure is reported as ``is_error=True`` here rather than raised, because the REST surface - turns an unrecognized exception into a 500 and an upstream 403 or 429 is not a gateway crash. - ``MCPUpstreamAuthError`` is the exception: it propagates so the caller is told to - re-authenticate, which both renderers already know how to say. - - Note: Local tools don't use prefixes, so we use the original name - """ - import inspect - - tool: Final = global_mcp_tool_registry.get_tool(name) - if not tool: - raise HTTPException(status_code=404, detail=f"Tool '{name}' not found") - - try: - if inspect.iscoroutinefunction(tool.handler): - result = await tool.handler(**arguments) - else: - result = tool.handler(**arguments) - except MCPUpstreamAuthError: - raise - except Exception as e: - verbose_logger.exception("Error executing local tool %s: %s", name, e) - return CallToolResult( - content=[TextContent(text=f"Error: {e}", type="text")], # mutable-ok: MCP result content - is_error=True, - ) - return CallToolResult( - content=[TextContent(text=str(result), type="text")], # mutable-ok: MCP result content - is_error=False, - ) - def _get_mcp_servers_in_path(path: str) -> list[str] | None: """ Get the MCP servers from the path @@ -4178,7 +1536,9 @@ if MCP_AVAILABLE: detail=f"API key does not have access to toolset '{toolset_id}'.", ) - tool_permissions = await global_mcp_server_manager.resolve_toolset_tool_permissions(toolset_ids=[toolset_id]) + tool_permissions = await operations.global_mcp_server_manager.resolve_toolset_tool_permissions( + toolset_ids=[toolset_id] + ) server_ids: Final = list(tool_permissions.keys()) existing_op: Final = user_api_key_auth.object_permission if existing_op is not None: @@ -4197,7 +1557,7 @@ if MCP_AVAILABLE: mcp_servers=server_ids, mcp_tool_permissions=tool_permissions, ) - return user_api_key_auth.model_copy(update={"object_permission": updated_op}) + return user_api_key_auth.model_copy(update={"object_permission": updated_op, "mcp_toolset_id": toolset_id}) async def _raise_preemptive_401_for_unauthenticated_servers( scope: Scope, @@ -4221,7 +1581,7 @@ if MCP_AVAILABLE: a server it will be 403'd on immediately after authentication. """ for server_name in mcp_servers or []: - server = global_mcp_server_manager.get_mcp_server_by_name(server_name, client_ip=client_ip) + server = operations.global_mcp_server_manager.get_mcp_server_by_name(server_name, client_ip=client_ip) if server is not None and allowed_server_ids is not None and server.server_id not in allowed_server_ids: # Caller's narrowed scope excludes this server — skip the # preemptive challenge and let downstream authorization @@ -4234,7 +1594,7 @@ if MCP_AVAILABLE: # authorization_url/token_url can change their inferred flow. continue if server is not None: - server = await global_mcp_server_manager.ensure_oauth_metadata_discovered(server) + server = await operations.global_mcp_server_manager.ensure_oauth_metadata_discovered(server) if server and server.auth_type == MCPAuth.oauth2: # The challenge decision is per oauth2 sub-mode, not per header: # gateway-managed modes (M2M and interactive authorization_code) @@ -4262,7 +1622,7 @@ if MCP_AVAILABLE: # authorization server is the gateway itself, vaulting via the # authorize interlude); the per-server relay advertised below # cannot vault without a litellm key on its token request. - if await global_mcp_server_manager.has_user_oauth_token(server, user_api_key_auth): + if await operations.global_mcp_server_manager.has_user_oauth_token(server, user_api_key_auth): continue if _is_mcp_admitted_user_subject(user_api_key_auth): @@ -4345,12 +1705,12 @@ if MCP_AVAILABLE: and server.server_id in frozenset( allowed.server_id - for allowed in await _get_allowed_mcp_servers( + for allowed in await operations._get_allowed_mcp_servers( user_api_key_auth=user_api_key_auth, mcp_servers=mcp_servers, client_ip=client_ip ) ) ): - await global_mcp_server_manager.preflight_token_exchange( + await operations.global_mcp_server_manager.preflight_token_exchange( server=server, oauth2_headers=oauth2_headers, user_api_key_auth=user_api_key_auth, @@ -4366,7 +1726,9 @@ if MCP_AVAILABLE: if ( server and server.is_oauth_passthrough - and not _client_has_passthrough_authorization(server, oauth2_headers, mcp_server_auth_headers) + and not operations._client_has_passthrough_authorization( + server, oauth2_headers, mcp_server_auth_headers + ) ): www_authenticate = get_passthrough_www_authenticate( scope=scope, @@ -4383,7 +1745,7 @@ if MCP_AVAILABLE: and server.is_oauth_delegate and len(mcp_servers or []) == 1 and _get_forwarded_auth_from_scope(scope) is None - and not _client_has_per_server_auth_header(server, mcp_server_auth_headers) + and not operations._client_has_per_server_auth_header(server, mcp_server_auth_headers) ): www_authenticate = get_passthrough_www_authenticate( scope=scope, @@ -4400,7 +1762,7 @@ if MCP_AVAILABLE: and server.is_true_passthrough and len(mcp_servers or []) == 1 and not _scope_has_authorization_header(scope) - and not _client_has_per_server_auth_header(server, mcp_server_auth_headers) + and not operations._client_has_per_server_auth_header(server, mcp_server_auth_headers) ): if server.is_dcr_bridge: raise HTTPException( @@ -4528,7 +1890,7 @@ if MCP_AVAILABLE: # Use the authorized server set, not the raw user-supplied names, so that # a caller cannot force a probe to a server their key is not allowed to use. - allowed_servers: Final = await _get_allowed_mcp_servers( + allowed_servers: Final = await operations._get_allowed_mcp_servers( user_api_key_auth=user_api_key_auth, mcp_servers=mcp_servers, client_ip=client_ip, diff --git a/litellm/proxy/_experimental/mcp_server/tool_search.py b/litellm/proxy/_experimental/mcp_server/tool_search.py index a482d02c31d..3650c722103 100644 --- a/litellm/proxy/_experimental/mcp_server/tool_search.py +++ b/litellm/proxy/_experimental/mcp_server/tool_search.py @@ -463,8 +463,8 @@ async def handle_mcp_tool_search( oauth2_headers: dict[str, str] | None = None, raw_headers: dict[str, str] | None = None, ) -> CallToolResult: - from litellm.proxy._experimental.mcp_server.server import ( - _list_mcp_tools, # pyright: ignore[reportPrivateUsage] # shared catalog owner + from litellm.proxy._experimental.mcp_server.operations import ( + _list_mcp_tools, ) from litellm.proxy.proxy_server import llm_router, proxy_logging_obj @@ -519,8 +519,8 @@ async def handle_mcp_proxy_tool( from jsonschema import validate from litellm.proxy import proxy_server - from litellm.proxy._experimental.mcp_server.server import ( # pyright: ignore[reportPrivateUsage] # shared catalog owner - _list_mcp_tools, # pyright: ignore[reportPrivateUsage] # shared catalog owner + from litellm.proxy._experimental.mcp_server.operations import ( + _list_mcp_tools, ) listing: Final = await _list_mcp_tools( @@ -607,7 +607,7 @@ async def handle_mcp_tool_call( requested_server_id: str | None = None, guardrail_context: Mapping[str, object] | None = None, ) -> CallToolResult: - from litellm.proxy._experimental.mcp_server.server import ( + from litellm.proxy._experimental.mcp_server.operations import ( _get_allowed_mcp_servers, execute_mcp_tool, raise_denied_scoped_mcp_access, @@ -643,6 +643,7 @@ async def handle_mcp_tool_call( mcp_server_auth_headers=mcp_server_auth_headers, oauth2_headers=oauth2_headers, raw_headers=raw_headers, + client_ip=client_ip, litellm_logging_obj=litellm_logging_obj, requested_server_id=requested_server_id, guardrail_context=guardrail_context, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 3f6ef89fbc6..03caca399c3 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -905,6 +905,7 @@ class LiteLLMRoutes(enum.Enum): "/claude_code_gateway/v1/traces", "/user/list", # org admins checked in endpoint; non-admins get 403 "/management/v1/users/bulk_delete", # proxy admins delete anyone, org admins only their orgs' users; others 403 + "/user/password/change", # endpoint only ever writes the caller's own row "/model/{model_id}/update", "/prompt/list", "/prompt/info", @@ -1865,6 +1866,17 @@ class NewUserRequest(GenerateRequestBase): send_invite_email: bool | None = None sso_user_id: str | None = None organizations: list[str] | None = None + password: str | None = None + + @field_validator("password") + @classmethod + def password_not_supported(cls, value: str | None) -> str | None: + if value is not None: + raise ValueError( + "password cannot be set via /user/new. Users set their own password through an " + "invitation link (POST /invitation/new)." + ) + return value class NewUserResponse(GenerateKeyResponse): @@ -1887,7 +1899,8 @@ class NewUserResponse(GenerateKeyResponse): class UpdateUserRequestNoUserIDorEmail(GenerateRequestBase): # shared with BulkUpdateUserRequest - password: str | None = None + # repr=False keeps the plaintext out of management-endpoint alerts, which str() the request model + password: str | None = Field(default=None, repr=False) spend: float | None = None metadata: dict | None = None user_alias: str | None = None @@ -1917,6 +1930,16 @@ class UpdateUserRequest(UpdateUserRequestNoUserIDorEmail): return values +class ChangePasswordRequest(LiteLLMPydanticObjectBase): + current_password: str = Field(repr=False) + new_password: str = Field(repr=False) + + +class ChangePasswordResponse(LiteLLMPydanticObjectBase): + user_id: str + message: str + + class DeleteUserRequest(LiteLLMPydanticObjectBase): user_ids: list[str] # required @@ -3239,6 +3262,7 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob # above; a forged value could at most narrow, but the stripping keeps the field's provenance # single-owner so its meaning stays trustworthy. mcp_session_resource_server_id: str | None = Field(default=None, exclude=True) + mcp_toolset_id: str | None = Field(default=None, exclude=True) via_virtual_key: bool = Field( default=False, exclude=True, @@ -3280,6 +3304,7 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob values.pop("mcp_admitted_user_subject", None) values.pop("mcp_source_team_rpm_limits", None) values.pop("mcp_session_resource_server_id", None) + values.pop("mcp_toolset_id", None) values.pop("via_virtual_key", None) if values.get("api_key") is not None: values.update({"token": cls._safe_hash_litellm_api_key(values.get("api_key"))}) @@ -3938,6 +3963,12 @@ class AllCallbacks(LiteLLMPydanticObjectBase): ) +class HTTPExceptionErrorDetail(TypedDict): + """The `{"error": }` shape most proxy endpoints raise as `HTTPException.detail`.""" + + error: ReadOnly[str] + + class SpendLogsRouterMetadata(TypedDict): """ Router provenance stamped on spend logs for deployments flagged with diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py index e0d599b0017..4c2b5d3d0fe 100644 --- a/litellm/proxy/auth/login_utils.py +++ b/litellm/proxy/auth/login_utils.py @@ -10,14 +10,16 @@ import secrets from collections.abc import Mapping from datetime import datetime, timedelta, timezone from types import MappingProxyType -from typing import Final, Literal, cast +from typing import TYPE_CHECKING, Final, Literal, cast import jwt from fastapi import HTTPException import litellm +from litellm._logging import verbose_proxy_logger from litellm.constants import LITELLM_PROXY_ADMIN_NAME, LITELLM_UI_SESSION_DURATION from litellm.litellm_core_utils.duration_parser import duration_in_seconds +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy._types import ( LiteLLM_UserTable, LitellmUserRoles, @@ -28,6 +30,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.auth.auth_utils import is_sso_provider_fully_configured from litellm.proxy.auth.login_throttle import LoginAttempt, LoginThrottle +from litellm.proxy.auth.password_policy import is_breach_check_enabled, is_password_breached from litellm.proxy.management_endpoints.internal_user_endpoints import user_update from litellm.proxy.management_endpoints.key_management_endpoints import ( generate_key_helper_fn, @@ -50,6 +53,57 @@ INVALID_UI_CREDENTIALS_MESSAGE: Final = ( ) INVALID_USER_PASSWORD_MESSAGE: Final = "Invalid credentials used to access UI. Check the password set for your user" +if TYPE_CHECKING: + from prisma import types as prisma_types + +BREACH_RECHECK_INTERVAL: Final = timedelta(hours=24) +PASSWORD_RESET_ALLOWED_ROUTES: Final = ("/user/password/change",) +PASSWORD_SESSION_METADATA: Final = MappingProxyType({"login_method": "username_password"}) + + +def _breach_recheck_due(last_breach_check_at: datetime | None) -> bool: + if last_breach_check_at is None: + return True + last_checked_utc: Final = ( + last_breach_check_at + if last_breach_check_at.tzinfo is not None + else last_breach_check_at.replace(tzinfo=timezone.utc) + ) + return datetime.now(timezone.utc) - last_checked_utc >= BREACH_RECHECK_INTERVAL + + +async def screen_login_password_for_breach( + user_id: str, + password: str, + last_breach_check_at: datetime | None, + general_settings: Mapping[str, object], + prisma_client: PrismaClient, + client: AsyncHTTPHandler | None = None, +) -> bool: + """Screens a successfully verified login password against HIBP, stamps + ``password_reset_required`` when breached, and returns whether a breach was + found so the login it runs in can restrict the session it is about to mint. + Fails open (HIBP or DB trouble never fails the login) and rechecks a given + user at most once per ``BREACH_RECHECK_INTERVAL``.""" + if not is_breach_check_enabled(general_settings): + return False + if not _breach_recheck_due(last_breach_check_at): + return False + breached: Final = await is_password_breached(password, general_settings, client) + checked_at: Final = datetime.now(timezone.utc) + breached_update: Final[prisma_types.LiteLLM_UserTableUpdateInput] = { + "last_breach_check_at": checked_at, + "password_reset_required": True, + } + recheck_update: Final[prisma_types.LiteLLM_UserTableUpdateInput] = {"last_breach_check_at": checked_at} + update_data: Final = breached_update if breached else recheck_update + find_user: Final[prisma_types.LiteLLM_UserTableWhereInput] = {"user_id": user_id} + try: + await UserRepository(prisma_client).table.update(where=find_user, data=update_data) + except Exception as e: # noqa: BLE001 # a failed stamp must never surface into the login + verbose_proxy_logger.warning("Login-time breach screening could not update user %s: %s", user_id, e) + return breached + async def _rehash_password_if_needed(user_id: str, password: str, stored: str) -> None: """Rehash legacy password (SHA256) to scrypt on successful login.""" @@ -137,6 +191,7 @@ class LoginResult: user_email: str | None user_role: str login_method: Literal["sso", "username_password"] + password_reset_required: bool def __init__( self, @@ -145,12 +200,14 @@ class LoginResult: user_email: str | None, user_role: str, login_method: Literal["sso", "username_password"] = "username_password", + password_reset_required: bool = False, ): self.user_id = user_id self.key = key self.user_email = user_email self.user_role = user_role self.login_method = login_method + self.password_reset_required = password_reset_required async def authenticate_user( @@ -356,20 +413,28 @@ async def _sign_in( if verify_password(password, _password): await _rehash_password_if_needed(_user_row.user_id, password, _password) + breached_now: Final = prisma_client is not None and await screen_login_password_for_breach( + user_id=_user_row.user_id, + password=password, + last_breach_check_at=getattr(_user_row, "last_breach_check_at", None), + general_settings=general_settings, + prisma_client=prisma_client, + ) + password_reset_required: Final = breached_now or getattr(_user_row, "password_reset_required", None) is True if os.getenv("DATABASE_URL") is not None: response = await generate_key_helper_fn( llm_router=None, request_type="key", - **{ - "user_role": user_role, - "duration": LITELLM_UI_SESSION_DURATION, - "key_max_budget": litellm.max_ui_session_budget, - "models": [], - "aliases": {}, - "config": {}, - "spend": 0, - "user_id": user_id, - "team_id": "litellm-dashboard", + user_role=user_role, + duration=LITELLM_UI_SESSION_DURATION, + key_max_budget=litellm.max_ui_session_budget, + spend=0, + user_id=user_id, + team_id="litellm-dashboard", + allowed_routes=list(PASSWORD_RESET_ALLOWED_ROUTES) if password_reset_required else None, + metadata={ + **PASSWORD_SESSION_METADATA, + **({"password_reset_required": True} if password_reset_required else {}), }, ) else: @@ -390,6 +455,7 @@ async def _sign_in( user_email=user_email, user_role=cast(str, user_role), login_method="username_password", + password_reset_required=password_reset_required, ) else: await attempt.failed() @@ -460,4 +526,5 @@ def create_ui_token_object( auth_header_name=general_settings.get("litellm_key_header_name", "Authorization"), disabled_non_admin_personal_key_creation=disabled_non_admin_personal_key_creation, server_root_path=get_server_root_path(), + password_reset_required=login_result.password_reset_required, ) diff --git a/litellm/proxy/auth/password_policy.py b/litellm/proxy/auth/password_policy.py index ab7a565894a..7f06a0993d3 100644 --- a/litellm/proxy/auth/password_policy.py +++ b/litellm/proxy/auth/password_policy.py @@ -4,13 +4,28 @@ Applied at every path that persists a new or changed password for a DB-backed user (``/user/update``, ``/user/bulk_update``, and the invitation onboarding claim flow), so the strength bar is configured in one place instead of per-endpoint. + +Also screens new passwords against known data breaches via the +haveibeenpwned.com (HIBP) k-anonymity range API: only the first 5 characters +of the password's SHA-1 hash ever leave the proxy, and the check fails open +(allows the password) when HIBP is unreachable. """ -from collections.abc import Mapping +import asyncio +import hashlib +from collections.abc import Mapping, Sequence from dataclasses import dataclass +from types import MappingProxyType from typing import Final +from litellm._logging import verbose_proxy_logger +from litellm._version import version +from litellm.constants import HIBP_RANGE_API_BASE +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client from litellm.proxy._types import ProxyErrorTypes, ProxyException +from litellm.types.llms.custom_http import httpxSpecialProvider + +HIBP_TIMEOUT_SECONDS: Final = 5.0 DEFAULT_MIN_LENGTH: Final = 12 MIN_ALLOWED_LENGTH: Final = 8 @@ -90,3 +105,114 @@ def validate_password_policy(password: str, general_settings: Mapping[str, objec param="password", code=400, ) + + +def _hibp_client() -> AsyncHTTPHandler: + return get_async_httpx_client( + llm_provider=httpxSpecialProvider.PasswordBreachCheck, + params={"timeout": HIBP_TIMEOUT_SECONDS}, # mutable-ok: callee takes a bare dict (PEP 589) + ) + + +def _is_suffix_in_range_response(response_body: str, hash_suffix: str) -> bool: + for line in response_body.upper().splitlines(): + entry_suffix, _, count = line.strip().partition(":") + if entry_suffix == hash_suffix: + return int(count.strip() or "0") > 0 + return False + + +async def _is_password_breached(password: str, client: AsyncHTTPHandler) -> bool: + # usedforsecurity=False: SHA-1 is only a lookup key into the HIBP dataset, so no security property rests on it + sha1_hex: Final = hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper() + headers: Final = { # mutable-ok: callee takes a bare dict (PEP 589) + "Add-Padding": "true", + "User-Agent": f"litellm-proxy/{version}", + } + try: + response: Final = await client.get( + f"{HIBP_RANGE_API_BASE}/{sha1_hex[:5]}", + headers=headers, + ) + response.raise_for_status() + breached: Final = _is_suffix_in_range_response(response.text, sha1_hex[5:]) + except Exception as e: # noqa: BLE001 # fail-open: any HIBP failure skips the check, never breaks the caller + verbose_proxy_logger.warning("Breached-password check skipped, HIBP lookup failed: %s", e) + return False + return breached + + +def is_breach_check_enabled(general_settings: Mapping[str, object]) -> bool: + return general_settings.get("password_policy_check_breached_passwords", True) is not False + + +async def is_password_breached( + password: str, + general_settings: Mapping[str, object], + client: AsyncHTTPHandler | None = None, +) -> bool: + """False when the check is disabled, the password is absent from the HIBP + corpus, or HIBP is unreachable (fail open).""" + if not is_breach_check_enabled(general_settings): + return False + return await _is_password_breached(password, client if client is not None else _hibp_client()) + + +def breached_password_error() -> ProxyException: + return ProxyException( + message=( + "This password appears in known data breaches and cannot be used. Please choose a different password." + ), + type=ProxyErrorTypes.validation_error, + param="password", + code=400, + ) + + +async def validate_password_not_breached( + password: str, + general_settings: Mapping[str, object], + client: AsyncHTTPHandler | None = None, +) -> None: + """Raise ``ProxyException`` (400) if ``password`` appears in a known data breach. + + Fails open: an unreachable or misbehaving HIBP allows the password.""" + if not await is_password_breached(password, general_settings, client): + return + raise breached_password_error() + + +def _strength_verdict(password: str, general_settings: Mapping[str, object]) -> ProxyException | None: + try: + validate_password_policy(password, general_settings) + except ProxyException as e: + return e + return None + + +async def validate_passwords_bulk( + passwords: Sequence[str], + general_settings: Mapping[str, object], + client: AsyncHTTPHandler | None = None, +) -> Mapping[str, ProxyException | None]: + """Per-unique-password policy verdicts for a batch: the ProxyException to + surface, or None when the password is acceptable. + + Deduplicates first, then issues every needed HIBP lookup concurrently, so a + batch caller pays one HIBP timeout window in the worst case instead of one + per password (each lookup still fails open independently).""" + unique_passwords: Final = tuple(dict.fromkeys(passwords)) + strength_verdicts: Final[Mapping[str, ProxyException | None]] = MappingProxyType( + {password: _strength_verdict(password, general_settings) for password in unique_passwords} + ) + to_screen: Final = tuple(password for password in unique_passwords if strength_verdicts[password] is None) + breached_flags: Final = await asyncio.gather( + *(is_password_breached(password, general_settings, client) for password in to_screen) + ) + breached_passwords: Final = frozenset(password for password, breached in zip(to_screen, breached_flags) if breached) + return MappingProxyType( + { + password: breached_password_error() if password in breached_passwords else strength_verdicts[password] + for password in unique_passwords + } + ) diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 1b9fd7c42bf..afddb4866a9 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -194,6 +194,16 @@ class RouteChecks: if denied_auth_enforced_pass_through_route: raise RouteChecks._auth_pass_through_denied_exception(route=route) + if valid_token.metadata.get("password_reset_required") is True: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=( + "This account's password must be changed before the session can be used: " + "it was either found in a known data breach or set by an admin. " + "Change it via POST /user/password/change (UI: /ui/change-password), then log in again." + ), + ) + raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=f"Virtual key is not allowed to call this route. Only allowed to call routes: {valid_token.allowed_routes}. Tried to call route: {route}", @@ -812,7 +822,8 @@ class RouteChecks: in the codebase is automatically readable by Admin Viewer without needing to remember to add it to an allowlist. 3. Unsafe HTTP method (POST/PUT/PATCH/DELETE): - - Allow `/user/update` only when restricted to user_email/password. + - Allow `/user/update` only when restricted to user_email. + - Allow `/user/password/change` (endpoint only writes the caller's own row). - Block all explicit writes in `_ADMIN_VIEWER_BLOCKED_WRITE_ROUTES`. - Otherwise allow only if the route is in admin_viewer_routes / global_spend_tracking_routes (legacy explicit-allow set). @@ -832,10 +843,10 @@ class RouteChecks: if request_data is not None and isinstance(request_data, dict): _params_updated: Final = request_data.keys() for param in _params_updated: - if param not in ["user_email", "password"]: + if param != "user_email": raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, - detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route} and updating invalid param: {param}. only user_email and password can be updated", + detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route} and updating invalid param: {param}. only user_email can be updated", ) elif RouteChecks.check_route_access(route=route, allowed_routes=_PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES) or ( route.startswith("/key/") and route.endswith(_PROXY_ADMIN_VIEW_ONLY_BLOCKED_KEY_SUFFIXES) @@ -854,21 +865,25 @@ class RouteChecks: return # ── Unsafe HTTP method: explicit checks ────────────────────────── - # Allow `/user/update` for self-service email / password change. + # Allow `/user/update` for self-service email change. if route == "/user/update": if request_data is not None and isinstance(request_data, dict): for param in request_data: - if param not in ["user_email", "password"]: + if param != "user_email": raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=( f"user not allowed to access this route, role= {_user_role}. " f"Trying to access: {route} and updating invalid param: {param}. " - "only user_email and password can be updated" + "only user_email can be updated" ), ) return + # Self-service password change; the endpoint only writes the caller's own row. + if route == "/user/password/change": + return + # Hard-block known write routes regardless of HTTP method (defensive # — these are POSTs in practice, but pinning them here protects # against future GET-shaped writes). diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index c6b89096ca9..60ac7e55eaf 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -28,6 +28,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy._types import * from litellm.proxy.auth.auth_checks import ( delete_cache_key_objects, @@ -35,7 +36,11 @@ from litellm.proxy.auth.auth_checks import ( get_team_object, get_user_object, ) -from litellm.proxy.auth.password_policy import validate_password_policy +from litellm.proxy.auth.password_policy import ( + validate_password_not_breached, + validate_password_policy, + validate_passwords_bulk, +) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast from litellm.proxy.common_utils.user_api_key_cache import ( @@ -173,11 +178,23 @@ def _team_membership_table( return team_membership_table -def _hash_password_in_dict(data: dict, general_settings: Mapping[str, object]) -> None: - """Validate and hash password field in-place if present.""" +async def _hash_password_in_dict( + data: dict, general_settings: Mapping[str, object], password_prevalidated: bool = False +) -> None: + """Validate and hash password field in-place if present. + + ``password_prevalidated`` skips the policy checks for callers that already + validated the password (the bulk path screens its whole batch upfront). + + An admin-set password is known to whoever set it, so the user is also + flagged for a forced password change at next login.""" if "password" in data and data["password"] is not None: - validate_password_policy(data["password"], general_settings) + if not password_prevalidated: + validate_password_policy(data["password"], general_settings) + await validate_password_not_breached(data["password"], general_settings) data["password"] = hash_password(data["password"]) + data["password_reset_required"] = True + data["last_breach_check_at"] = None def _strip_password_from_response(response) -> None: @@ -505,6 +522,7 @@ async def new_user( - prompts: Optional[List[str]] - List of allowed prompts for the user. If specified, the user will only be able to use these specific prompts. - organizations: List[str] - List of organization id's the user is a member of - budget_limits: Optional[list] - List of concurrent budget windows for the user. Each window specifies a budget_limit, time_period, and optional budget_duration. Example - [{"budget_limit": 10.0, "time_period": "1d"}, {"budget_limit": 50.0, "time_period": "7d"}]. + - password: Optional[str] - Not supported; any value is rejected with a 422. Users set their own password through an invitation link (POST /invitation/new). Returns: - key: (str) The generated api key for the user - expires: (datetime) Datetime object for when key expires. @@ -524,7 +542,7 @@ async def new_user( ``` """ try: - from litellm.proxy.proxy_server import _license_check, general_settings, prisma_client + from litellm.proxy.proxy_server import _license_check, prisma_client if prisma_client is None: raise HTTPException(status_code=400, detail=CommonProxyErrors.db_not_connected_error.value) @@ -572,7 +590,7 @@ async def new_user( # generate_key_helper_fn only forwards object_permission_id, so without this the entitlement # the caller sent would be dropped on the floor. data_json = await _set_object_permission(data_json=data_json, prisma_client=prisma_client) - _hash_password_in_dict(data_json, general_settings) + data_json.pop("password", None) teams = data.teams if teams is None: teams = check_if_default_team_set() @@ -1438,6 +1456,7 @@ async def _update_single_user_helper( user_request: UpdateUserRequest, user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: str | None = None, + password_prevalidated: bool = False, ) -> dict[str, Any]: """ Helper function to update a single user. @@ -1460,7 +1479,7 @@ async def _update_single_user_helper( data_json: Final[dict] = user_request.model_dump(exclude_unset=True) non_default_values = _update_internal_user_params(data_json=data_json, data=user_request) - _hash_password_in_dict(non_default_values, general_settings) + await _hash_password_in_dict(non_default_values, general_settings, password_prevalidated=password_prevalidated) existing_user_row: BaseModel | None = None if user_request.user_id: @@ -1641,7 +1660,7 @@ async def user_update( Parameters: - user_id: Optional[str] - Specify a user id. If not set, a unique id will be generated. - user_email: Optional[str] - Specify a user email. - - password: Optional[str] - Specify a user password. + - password: Optional[str] - Set the user's password (admin only). Must satisfy the configured password policy. The user is required to change it at their next login. Users change their own password with POST /user/password/change. - user_alias: Optional[str] - A descriptive name for you to know who this user id refers to. - teams: Optional[list] - specify a list of team id's a user belongs to. - send_invite_email: Optional[bool] - Specify if an invite email should be sent. @@ -1709,19 +1728,38 @@ async def bulk_update_processed_users( users_to_update: list[UpdateUserRequest], user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: str | None = None, + hibp_client: AsyncHTTPHandler | None = None, ) -> BulkUpdateUserResponse: + from litellm.proxy.proxy_server import general_settings + results: Final[list[UserUpdateResult]] = [] successful_updates = 0 failed_updates = 0 + # Screen the batch's passwords upfront and concurrently: done per-user + # inside the loop below, each HIBP lookup would be awaited serially and a + # degraded-slow HIBP could stretch a full batch to minutes, timing out the + # request after some updates already persisted. + password_verdicts: Final = await validate_passwords_bulk( + tuple(u.password for u in users_to_update if u.password is not None), + general_settings, + client=hibp_client, + ) + # Process each user update independently try: for user_request in users_to_update: try: + if ( + user_request.password is not None + and (password_error := password_verdicts.get(user_request.password)) is not None + ): + raise password_error response = await _update_single_user_helper( user_request=user_request, user_api_key_dict=user_api_key_dict, litellm_changed_by=litellm_changed_by, + password_prevalidated=True, ) # Record success results.append( @@ -1859,6 +1897,14 @@ async def bulk_user_update( status_code=403, detail="Only proxy admins can update all users at once.", ) + if data.user_updates.password is not None: + bulk_password_error: Final[HTTPExceptionErrorDetail] = { + "error": ( + "Setting one password for all users is not supported. " + "Use per-user updates via the 'users' list instead." + ) + } + raise HTTPException(status_code=400, detail=bulk_password_error) # Optimized path for updating all users directly in database all_users_in_db: Final = await _user_table(prisma_client).find_many(order={"created_at": "desc"}) diff --git a/litellm/proxy/management_endpoints/password_endpoints.py b/litellm/proxy/management_endpoints/password_endpoints.py new file mode 100644 index 00000000000..03a8b4c4010 --- /dev/null +++ b/litellm/proxy/management_endpoints/password_endpoints.py @@ -0,0 +1,154 @@ +""" +Self-service password management. + +/user/password/change + +Deliberately NOT wrapped in `management_endpoint_wrapper`: the wrapper emits +request kwargs to OTEL spans, which would log plaintext passwords. The audit +signal is emitted by hand below, with field names only, never values. +""" + +from typing import TYPE_CHECKING, Annotated, Final + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import TypeAdapter + +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import ( + UI_TEAM_ID, + ChangePasswordRequest, + ChangePasswordResponse, + CommonProxyErrors, + HTTPExceptionErrorDetail, + LitellmTableNames, + UserAPIKeyAuth, +) +from litellm.proxy.auth.login_utils import PASSWORD_SESSION_METADATA +from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.management_helpers.audit_logs import create_object_audit_log +from litellm.proxy.utils import hash_password, verify_password +from litellm.repositories.prisma_protocols import TableActions +from litellm.repositories.user_repository import UserRepository + +if TYPE_CHECKING: + from prisma import models as prisma_models + from prisma import types as prisma_types + + from litellm.proxy.utils import PrismaClient + +router: Final = APIRouter() + +_PASSWORD_CHANGED_AUDIT_VALUES: Final = '{"fields_changed": ["password"]}' +_KEY_METADATA: Final = TypeAdapter(dict[str, object]) + + +def _error_detail(message: str) -> HTTPExceptionErrorDetail: + detail: Final[HTTPExceptionErrorDetail] = {"error": message} + return detail + + +def _is_password_login_session(user_api_key_dict: UserAPIKeyAuth) -> bool: + if user_api_key_dict.team_id != UI_TEAM_ID: + return False + key_metadata: Final = _KEY_METADATA.validate_python(user_api_key_dict.metadata) + return all(key_metadata.get(k) == v for k, v in PASSWORD_SESSION_METADATA.items()) + + +def _user_table( + prisma_client: "PrismaClient | None", +) -> "TableActions[prisma_models.LiteLLM_UserTable]": + user_table: Final[TableActions[prisma_models.LiteLLM_UserTable]] = UserRepository(prisma_client).table + return user_table + + +@router.post( + "/user/password/change", + tags=("Internal User management",), + dependencies=(Depends(user_api_key_auth),), +) +async def change_password( + data: ChangePasswordRequest, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> ChangePasswordResponse: + """ + Change the calling user's own password. + + Only callable with the dashboard session issued by a username/password + login; SSO sessions and virtual keys are rejected with 403. Requires the + current password. The new password must differ from the + current one and satisfy the configured password policy + (`general_settings.password_policy_*`: minimum length, character classes, + and, when enabled, breached-password screening via haveibeenpwned.com). + A successful change lifts any pending forced password reset + (`password_reset_required`) on the account. + + Parameters: + - current_password: str - The user's current password. + - new_password: str - The password to change to. + """ + from litellm.proxy.proxy_server import general_settings, litellm_proxy_admin_name, prisma_client + + if prisma_client is None: + raise HTTPException( + status_code=500, + detail=_error_detail(CommonProxyErrors.db_not_connected_error.value), + ) + + if not _is_password_login_session(user_api_key_dict): + raise HTTPException( + status_code=403, + detail=_error_detail( + "Passwords can only be changed from a dashboard session created by logging in with a password." + ), + ) + + user_id: Final = user_api_key_dict.user_id + if user_id is None: + raise HTTPException( + status_code=400, + detail=_error_detail("No user is associated with this session, so there is no password to change."), + ) + + find_user: Final[prisma_types.LiteLLM_UserTableWhereInput] = {"user_id": user_id} + user_row: Final = await _user_table(prisma_client).find_first(where=find_user) + stored_password: Final = user_row.password if user_row is not None else None + if stored_password is None: + raise HTTPException( + status_code=400, + detail=_error_detail( + "This account has no password set, so there is no password to change. " + "Passwords are set through an invitation link (POST /invitation/new)." + ), + ) + + if not verify_password(data.current_password, stored_password): + raise HTTPException(status_code=400, detail=_error_detail("Current password is incorrect.")) + + if data.new_password == data.current_password: + raise HTTPException( + status_code=400, + detail=_error_detail("New password must be different from the current password."), + ) + + validate_password_policy(data.new_password, general_settings) + await validate_password_not_breached(data.new_password, general_settings) + + password_update: Final[prisma_types.LiteLLM_UserTableUpdateInput] = { + "password": hash_password(data.new_password), + "password_reset_required": False, + "last_breach_check_at": None, + } + await _user_table(prisma_client).update(where=find_user, data=password_update) + + verbose_proxy_logger.info("Password changed via /user/password/change for user_id=%s", user_id) + await create_object_audit_log( + object_id=user_id, + action="updated", + litellm_changed_by=None, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + table_name=LitellmTableNames.USER_TABLE_NAME, + after_value=_PASSWORD_CHANGED_AUDIT_VALUES, + ) + return ChangePasswordResponse(user_id=user_id, message="Password updated successfully.") diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 00cf357d89d..7859c678c07 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -3665,6 +3665,7 @@ class SSOAuthenticationHandler: auth_header_name=general_settings.get("litellm_key_header_name", "Authorization"), disabled_non_admin_personal_key_creation=disabled_non_admin_personal_key_creation, server_root_path=get_server_root_path(), + password_reset_required=False, ) from litellm.proxy.auth.login_utils import encode_ui_session_jwt diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index dac1a8dd001..8111e21e114 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -359,7 +359,7 @@ from litellm.proxy.auth.model_checks import ( get_mcp_server_ids, get_team_models, ) -from litellm.proxy.auth.password_policy import validate_password_policy +from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy from litellm.proxy.auth.user_api_key_auth import ( _fetch_global_spend_with_event_coordination, user_api_key_auth, @@ -601,6 +601,9 @@ from litellm.proxy.management_endpoints.model_management_endpoints import ( from litellm.proxy.management_endpoints.organization_endpoints import ( router as organization_router, ) +from litellm.proxy.management_endpoints.password_endpoints import ( + router as password_management_router, +) from litellm.proxy.management_endpoints.prompt_caching_requests import ( router as prompt_caching_requests_router, ) @@ -16647,6 +16650,7 @@ async def onboarding(invite_link: str, request: Request): auth_header_name=general_settings.get("litellm_key_header_name", "Authorization"), disabled_non_admin_personal_key_creation=disabled_non_admin_personal_key_creation, server_root_path=get_server_root_path(), + password_reset_required=False, ) jwt_token: Final = jwt.encode( cast(dict, returned_ui_token_object), @@ -16757,6 +16761,7 @@ async def _generate_onboarding_ui_session_token(user_obj: _UserTableRow) -> str: auth_header_name=general_settings.get("litellm_key_header_name", "Authorization"), disabled_non_admin_personal_key_creation=disabled_non_admin_personal_key_creation, server_root_path=get_server_root_path(), + password_reset_required=False, ) assert master_key is not None return jwt.encode( @@ -16827,6 +16832,7 @@ async def claim_onboarding_link(data: InvitationClaim, request: Request): ) validate_password_policy(data.password, general_settings) + await validate_password_not_breached(data.password, general_settings) hashed_pw: Final = hash_password(data.password) current_time = litellm.utils.get_utc_datetime() async with prisma_client.db.tx() as tx: @@ -16846,7 +16852,12 @@ async def claim_onboarding_link(data: InvitationClaim, request: Request): ### UPDATE USER OBJECT ### user_obj: Final[_UserTableRow | None] = await tx.litellm_usertable.update( - where={"user_id": invite_obj.user_id}, data={"password": hashed_pw} + where={"user_id": invite_obj.user_id}, + data={ + "password": hashed_pw, + "password_reset_required": False, + "last_breach_check_at": None, + }, ) if user_obj is None: @@ -19284,6 +19295,7 @@ app.include_router(pass_through_router) app.include_router(health_router) app.include_router(key_management_router) app.include_router(internal_user_router) +app.include_router(password_management_router) app.include_router(team_router) app.include_router(ui_sso_router) app.include_router(organization_router) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 2d7e557a9d1..368864f9cd1 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -246,6 +246,8 @@ model LiteLLM_UserTable { organization_id String? object_permission_id String? password String? + password_reset_required Boolean? + last_breach_check_at DateTime? teams String[] @default([]) user_role String? max_budget Float? diff --git a/litellm/types/llms/custom_http.py b/litellm/types/llms/custom_http.py index d80d7410aae..793893451df 100644 --- a/litellm/types/llms/custom_http.py +++ b/litellm/types/llms/custom_http.py @@ -31,6 +31,7 @@ class httpxSpecialProvider(str, Enum): UI = "ui" Sandbox = "sandbox" ModelCostMap = "model_cost_map" + PasswordBreachCheck = "password_breach_check" VerifyTypes = str | bool | ssl.SSLContext diff --git a/litellm/types/proxy/ui_sso.py b/litellm/types/proxy/ui_sso.py index 0d7e0b99cf0..03b0b92a4d1 100644 --- a/litellm/types/proxy/ui_sso.py +++ b/litellm/types/proxy/ui_sso.py @@ -1,6 +1,6 @@ from typing import Literal -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict class ReturnedUITokenObject(TypedDict): @@ -17,6 +17,7 @@ class ReturnedUITokenObject(TypedDict): auth_header_name: str disabled_non_admin_personal_key_creation: bool server_root_path: str # e.g. `/litellm` + password_reset_required: ReadOnly[bool] class ParsedOpenIDResult(TypedDict, total=False): diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6b49b1d47a5..beb90c2fb8c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -43037,21 +43037,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 8.95578e-07, + "input_cost_per_token": 8.92272e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.791156e-06, + "output_cost_per_token": 1.784544e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.46315e-08, + "cache_read_input_token_cost": 7.4356e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -68212,13 +68212,13 @@ "supports_web_search": false }, "openrouter/z-ai/glm-5.3-flash": { - "input_cost_per_token": 7.5e-08, - "output_cost_per_token": 2.5e-07, - "cache_read_input_token_cost": 2e-08, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5e-07, + "cache_read_input_token_cost": 5e-08, "litellm_provider": "openrouter", "max_input_tokens": 1310720, - "max_output_tokens": 102400, - "max_tokens": 102400, + "max_output_tokens": 943718, + "max_tokens": 943718, "mode": "chat", "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, @@ -73252,14 +73252,14 @@ "supports_web_search": true }, "openrouter/~z-ai/glm-flash-latest": { - "cache_read_input_token_cost": 2e-08, - "input_cost_per_token": 7.5e-08, + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 1.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 1310720, - "max_output_tokens": 102400, - "max_tokens": 102400, + "max_output_tokens": 943718, + "max_tokens": 943718, "mode": "chat", - "output_cost_per_token": 2.5e-07, + "output_cost_per_token": 5e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, diff --git a/schema.prisma b/schema.prisma index 2d7e557a9d1..368864f9cd1 100644 --- a/schema.prisma +++ b/schema.prisma @@ -246,6 +246,8 @@ model LiteLLM_UserTable { organization_id String? object_permission_id String? password String? + password_reset_required Boolean? + last_breach_check_at DateTime? teams String[] @default([]) user_role String? max_budget Float? diff --git a/scripts/check_mcp_operation_boundary.py b/scripts/check_mcp_operation_boundary.py new file mode 100644 index 00000000000..b6c9dcefabf --- /dev/null +++ b/scripts/check_mcp_operation_boundary.py @@ -0,0 +1,65 @@ +import ast +import sys +from pathlib import Path +from typing import Final + +PACKAGE: Final = Path("litellm/proxy/_experimental/mcp_server") +LEGACY_ADAPTERS: Final = frozenset({"server.py", "legacy_callbacks.py", "mcp_context.py", "mcp_debug.py"}) +CONFINED_NAMES: Final = frozenset( + { + "auth_context_var", + "active_mcp_session_var", + "active_mcp_request_ctx_var", + "get_active_auth_context", + "get_active_mcp_session", + "get_active_mcp_request_ctx", + "get_or_extract_auth_context", + "_session_obj_auth_storage", + "WeakKeyDictionary", + "_mcp_active_toolset_id", + "_mcp_gateway_initialize_instructions", + "_mcp_gateway_server_name", + "_mcp_proxy_mode", + } +) + + +def is_confined(name: str) -> bool: + return name in CONFINED_NAMES or name.startswith("_stateful_session_") + + +def violations(path: Path, source: str) -> tuple[str, ...]: + if path.name in LEGACY_ADAPTERS: + return () + tree: Final = ast.parse(source, filename=str(path)) + return tuple( + f"{path}:{node.lineno}: MCP request/session state belongs in a legacy adapter" + for node in ast.walk(tree) + if ( + isinstance(node, ast.ImportFrom) + and ( + (node.module or "").endswith(".mcp_context") + or any(is_confined(alias.name) for alias in node.names) + or (path.name in {"operations.py", "contracts.py"} and (node.module or "").endswith(".server")) + ) + or isinstance(node, ast.Name) + and is_confined(node.id) + or isinstance(node, ast.Attribute) + and is_confined(node.attr) + ) + ) + + +def main() -> int: + findings: Final = tuple( + finding for path in sorted(PACKAGE.rglob("*.py")) for finding in violations(path, path.read_text()) + ) + if findings: + print("\n".join(findings), file=sys.stderr) + return 1 + print("MCP operation boundary: passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/pre_commit_lint.sh b/scripts/pre_commit_lint.sh index 1abd415d237..22cc38f841c 100755 --- a/scripts/pre_commit_lint.sh +++ b/scripts/pre_commit_lint.sh @@ -102,6 +102,9 @@ ui_prettier_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs|json|css|s ui_eslint_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs)$' litellm_py_files=$(scope_match "$litellm_py_pattern") +if [ -n "$(scope_match '^(litellm/proxy/_experimental/mcp_server/|scripts/check_mcp_operation_boundary\.py)')" ]; then + uv run --no-sync python scripts/check_mcp_operation_boundary.py || exit 1 +fi e2e_py_files=$(scope_match "$e2e_py_pattern") test_tree_files=$(scope_match "$test_tree_pattern") # ruff format (and CI's format step) skip enterprise; the rest of make lint covers it. diff --git a/terraform/provider/tools/endpointaudit/coverage_allowlist.txt b/terraform/provider/tools/endpointaudit/coverage_allowlist.txt index 4ea64b152f1..f8277c83a64 100644 --- a/terraform/provider/tools/endpointaudit/coverage_allowlist.txt +++ b/terraform/provider/tools/endpointaudit/coverage_allowlist.txt @@ -86,6 +86,7 @@ POST /team/key/bulk_update POST /team/permissions_bulk_update POST /team/{team_id}/disable_logging POST /user/bulk_update +POST /user/password/change # Alternate method or path for functionality the provider already manages elsewhere GET /credentials/by_model/{model_id} diff --git a/tests/mcp_tests/test_mcp_logging.py b/tests/mcp_tests/test_mcp_logging.py index ed8829945e5..41d0e2cb59b 100644 --- a/tests/mcp_tests/test_mcp_logging.py +++ b/tests/mcp_tests/test_mcp_logging.py @@ -142,7 +142,7 @@ async def test_mcp_cost_tracking(): local_mcp_server_manager, ), patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", local_mcp_server_manager, ), ): @@ -293,7 +293,7 @@ async def test_mcp_cost_tracking_per_tool(): local_mcp_server_manager, ), patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", local_mcp_server_manager, ), ): @@ -451,7 +451,7 @@ async def test_mcp_tool_call_hook(): local_mcp_server_manager, ), patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", local_mcp_server_manager, ), ): diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/mcp_tests/test_mcp_server.py index 94cf35b675d..2b92367f186 100644 --- a/tests/mcp_tests/test_mcp_server.py +++ b/tests/mcp_tests/test_mcp_server.py @@ -922,7 +922,7 @@ async def test_get_tools_from_mcp_servers(): ) with patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", mock_manager, ): # Test with specific servers @@ -950,6 +950,7 @@ async def test_get_tools_from_mcp_servers(): extra_headers=None, add_prefix=False, raw_headers=None, + client_ip=None, user_api_key_auth=None, oauth2_headers=None, ): @@ -966,7 +967,7 @@ async def test_get_tools_from_mcp_servers(): ) with patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", mock_manager_2, ): result = await _get_tools_from_mcp_servers( @@ -998,7 +999,7 @@ async def test_get_tools_from_mcp_servers(): ) with patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", mock_manager, ): with patch( @@ -1981,6 +1982,7 @@ async def test_get_tools_for_single_server(): extra_headers=None, add_prefix=False, raw_headers=None, + client_ip=None, user_api_key_auth=None, ) @@ -2076,7 +2078,7 @@ async def test_rest_listing_hides_key_grants_dispatch_would_refuse(): with patch( "litellm.proxy._experimental.mcp_server.rest_endpoints.global_mcp_server_manager" ) as mock_manager, patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager" + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager" ) as mock_server_manager, patch.object( MCPRequestHandler, "get_allowed_tools_for_server", @@ -2473,7 +2475,7 @@ async def test_filter_tools_by_allowed_tools_integration(): # Mock the global MCP server manager with patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager" + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager" ) as mock_manager: # Mock manager methods mock_manager.get_allowed_mcp_servers = AsyncMock( @@ -2588,7 +2590,7 @@ async def test_filter_tools_by_disallowed_tools_integration(): # Mock the global MCP server manager with patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager" + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager" ) as mock_manager: # Mock manager methods mock_manager.get_allowed_mcp_servers = AsyncMock( @@ -2689,7 +2691,7 @@ async def test_filter_tools_no_restrictions_integration(): # Mock the global MCP server manager with patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager" + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager" ) as mock_manager: # Mock manager methods mock_manager.get_allowed_mcp_servers = AsyncMock( @@ -2970,10 +2972,10 @@ async def test_call_mcp_tool_uses_manager_permission_lookup(): return_value=mock_server, ) as mock_get_server, patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_tool_registry" + "litellm.proxy._experimental.mcp_server.operations.global_mcp_tool_registry" ) as mock_tool_registry, patch( - "litellm.proxy._experimental.mcp_server.server._handle_managed_mcp_tool", + "litellm.proxy._experimental.mcp_server.operations._handle_managed_mcp_tool", new_callable=AsyncMock, ) as mock_handle_managed, patch( @@ -3046,10 +3048,10 @@ async def test_call_mcp_tool_resolves_unprefixed_tool_name_and_checks_permission return_value=mock_server, ) as mock_get_server, patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_tool_registry" + "litellm.proxy._experimental.mcp_server.operations.global_mcp_tool_registry" ) as mock_tool_registry, patch( - "litellm.proxy._experimental.mcp_server.server._handle_managed_mcp_tool", + "litellm.proxy._experimental.mcp_server.operations._handle_managed_mcp_tool", new_callable=AsyncMock, ) as mock_handle_managed, patch( diff --git a/tests/otel_tests/test_e2e_budgeting.py b/tests/otel_tests/test_e2e_budgeting.py index ca5058818e4..ae8f0ddc3ec 100644 --- a/tests/otel_tests/test_e2e_budgeting.py +++ b/tests/otel_tests/test_e2e_budgeting.py @@ -5,6 +5,7 @@ import uuid from typing import Any, Optional import aiohttp +import openai import pytest from httpx import AsyncClient @@ -23,7 +24,7 @@ async def make_calls_until_budget_exceeded(session, key: str, call_function, **k call_count += 1 await asyncio.sleep(0.1) # allow spend tracking to catch up pytest.fail(f"Budget was not exceeded after {MAX_CALLS} calls") - except Exception as e: + except openai.APIStatusError as e: print("vars: ", vars(e)) print("e.body: ", e.body) @@ -32,8 +33,8 @@ async def make_calls_until_budget_exceeded(session, key: str, call_function, **k # Check error structure and values that should be consistent assert ( - error_dict["code"] == "429" - ), f"Expected error code 429, got: {error_dict['code']}" + error_dict["code"] == "422" + ), f"Expected error code 422, got: {error_dict['code']}" assert ( error_dict["type"] == "budget_exceeded" ), f"Expected error type budget_exceeded, got: {error_dict['type']}" @@ -506,9 +507,9 @@ async def make_calls_until_team_budget_exceeded_cli_sso( call_count += 1 await asyncio.sleep(0.1) pytest.fail(f"Budget was not exceeded after {MAX_CALLS} calls") - except Exception as e: + except openai.APIStatusError as e: error_dict = e.body - assert error_dict["code"] == "429" + assert error_dict["code"] == "422" assert error_dict["type"] == "budget_exceeded" message = error_dict["message"] assert "Budget has been exceeded!" in message @@ -556,7 +557,7 @@ async def test_team_budget_enforcement_cli_sso_token(): 1. Create team with a tiny max_budget and a user on that team 2. Obtain a CLI SSO JWT (HTTP poll flow when Redis is shared, else mint) 3. Make chat completion calls until the team budget is exceeded - 4. Verify HTTP 429 budget_exceeded names the team + 4. Verify HTTP 422 budget_exceeded names the team """ user_id = f"cli-budget-user-{uuid.uuid4().hex[:8]}" user_email = f"{user_id}@example.com" diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py index b45cd2ec299..a6b930db7a9 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -2297,6 +2297,56 @@ class TestStructuredMessagesWriteBack: } assert result["input"][3] == {"role": "user", "content": "What is the codename?"} + @pytest.mark.asyncio + async def test_codex_custom_tool_items_survive_tool_output_compression(self): + handler = OpenAIResponsesHandler() + additional_tools_item = { + "type": "additional_tools", + "tools": [{"type": "custom", "name": "exec", "description": "Run a JavaScript snippet"}], + } + reasoning_item = { + "id": "rs_456", + "type": "reasoning", + "summary": [], + "encrypted_content": "gAAAAA-signed-reasoning", + } + custom_tool_call_item = { + "id": "ctc_456", + "type": "custom_tool_call", + "call_id": "call_exec", + "name": "exec", + "input": 'const r = await tools.exec_command({"cmd": "cat memo.txt"});\ntext(r.output);', + "status": "completed", + } + data = { + "model": "gpt-5.6", + "input": [ + additional_tools_item, + {"role": "user", "content": "What is the codename?"}, + reasoning_item, + custom_tool_call_item, + { + "type": "custom_tool_call_output", + "call_id": "call_exec", + "output": [ + {"type": "input_text", "text": "Script completed\nOutput:\n"}, + {"type": "input_text", "text": "memo " * 400}, + ], + }, + ], + } + + result = await handler.process_input_messages(data, ToolOutputRewriteGuardrail()) + + assert result["input"][0] is additional_tools_item + assert result["input"][1] == {"role": "user", "content": "What is the codename?"} + assert result["input"][2] is reasoning_item + assert result["input"][3] is custom_tool_call_item + assert result["input"][4]["type"] == "custom_tool_call_output" + assert result["input"][4]["call_id"] == "call_exec" + assert COMPRESSED_MARKER in str(result["input"][4]["output"]) + assert len(result["input"]) == 5 + @pytest.mark.asyncio async def test_web_search_call_item_preserved_verbatim(self): handler = OpenAIResponsesHandler() diff --git a/tests/test_litellm/llms/xai/test_xai_chat_transformation.py b/tests/test_litellm/llms/xai/test_xai_chat_transformation.py index 290cd3dcb3a..3fd666e4f50 100644 --- a/tests/test_litellm/llms/xai/test_xai_chat_transformation.py +++ b/tests/test_litellm/llms/xai/test_xai_chat_transformation.py @@ -90,6 +90,16 @@ class TestXAIReasoningTokenFolding: assert response.usage.total_tokens == 999 +def test_max_completion_tokens_is_accepted_and_mapped_to_max_tokens() -> None: + optional_params = litellm.get_optional_params( + model="grok-4.20", + custom_llm_provider="xai", + max_completion_tokens=64, + ) + assert optional_params["max_tokens"] == 64, optional_params + assert "max_completion_tokens" not in optional_params, optional_params + + class TestXAIParallelToolCalls: """Test suite for XAI parallel tool calls functionality.""" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py index 87e23893616..a77b4c8d565 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_byok_oauth_endpoints.py @@ -1,3 +1,4 @@ +from litellm.proxy._experimental.mcp_server import operations as mcp_operations """ Unit tests for the BYOK OAuth 2.1 authorization server endpoints. @@ -592,7 +593,7 @@ async def test_check_byok_credential_missing_credential(monkeypatch): monkeypatch.delenv("PROXY_BASE_URL", raising=False) monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) - server_module.byok_credential_cache.flush_cache() + mcp_operations.byok_credential_cache.flush_cache() mock_prisma = MagicMock() with ( @@ -628,13 +629,13 @@ async def test_execute_byok_tool_missing_credential_advertises_api_key_flow(monk from litellm.types.mcp_server.mcp_server_manager import MCPServer monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com/proxy") - mcp_module.byok_credential_cache.flush_cache() + mcp_operations.byok_credential_cache.flush_cache() server = MCPServer(server_id="byok-discovery", name="byok-discovery", transport=MCPTransport.http, is_byok=True) prisma = MagicMock() prisma.db.litellm_mcpusercredentials.find_unique = AsyncMock(return_value=None) monkeypatch.setattr(proxy_server, "prisma_client", prisma) with pytest.raises(HTTPException) as exc_info: - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name="list_regions", arguments={}, allowed_mcp_servers=[server], @@ -687,7 +688,7 @@ async def test_invalidate_byok_cred_cache_evicts_locally_and_broadcasts_the_same server = MCPServer(server_id="byok-revoke", name="byok-server", transport=MCPTransport.http, is_byok=True) user_auth = UserAPIKeyAuth(user_id="mallory", api_key="sk-test") - server_module.byok_credential_cache.flush_cache() + mcp_operations.byok_credential_cache.flush_cache() db_lookup = AsyncMock(side_effect=["sk-before-revoke", None]) publish = AsyncMock() @@ -699,13 +700,13 @@ async def test_invalidate_byok_cred_cache_evicts_locally_and_broadcasts_the_same "litellm.proxy.proxy_server.prisma_client", MagicMock() ), patch.object( # test-quality-ok: the redis publisher is module-level; asserting the broadcast without a redis - server_module, "publish_auth_cache_invalidation", new=publish + mcp_operations, "publish_auth_cache_invalidation", new=publish ), ): - assert await server_module._get_byok_credential(server, user_auth) == "sk-before-revoke" - assert await server_module._get_byok_credential(server, user_auth) == "sk-before-revoke" - await server_module._invalidate_byok_cred_cache("mallory", "byok-revoke") - assert await server_module._get_byok_credential(server, user_auth) is None + assert await mcp_operations._get_byok_credential(server, user_auth) == "sk-before-revoke" + assert await mcp_operations._get_byok_credential(server, user_auth) == "sk-before-revoke" + await mcp_operations._invalidate_byok_cred_cache("mallory", "byok-revoke") + assert await mcp_operations._get_byok_credential(server, user_auth) is None assert db_lookup.await_count == 2 publish.assert_awaited_once_with(cache_key=byok_credential_cache_key("mallory", "byok-revoke")) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_contracts.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_contracts.py new file mode 100644 index 00000000000..e13ecdfcce9 --- /dev/null +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_contracts.py @@ -0,0 +1,60 @@ +from dataclasses import FrozenInstanceError + +import pytest + +from litellm.proxy._experimental.mcp_server.operations import prepare_context +from litellm.proxy._types import UserAPIKeyAuth + + +def test_operation_context_isolates_nested_headers_and_caller_permissions(): + caller = UserAPIKeyAuth(user_id="alpha", models=["allowed"]) + caller.mcp_admitted_user_subject = True + caller.mcp_session_resource_server_id = "alpha-server" + caller.mcp_toolset_id = "toolset-alpha" + caller.mcp_source_team_rpm_limits = {"team": {"alpha-server": 2}} + headers = {"x-caller": "alpha"} + server_headers = {"alpha-server": {"authorization": "alpha-token"}} + context = prepare_context(caller, raw_headers=headers, mcp_server_auth_headers=server_headers) + + caller.models.append("forbidden") + caller.mcp_source_team_rpm_limits["team"]["alpha-server"] = 999 + headers["x-caller"] = "bravo" + server_headers["alpha-server"]["authorization"] = "bravo-token" + captured = context.user_api_key_auth + assert captured is not None + assert captured.models == ["allowed"] + assert captured.mcp_admitted_user_subject is True + assert captured.mcp_session_resource_server_id == "alpha-server" + assert captured.mcp_toolset_id == "toolset-alpha" + assert captured.mcp_source_team_rpm_limits == {"team": {"alpha-server": 2}} + captured.models.append("also-forbidden") + assert context.user_api_key_auth.models == ["allowed"] + assert context.raw_headers == {"x-caller": "alpha"} + assert context.mcp_server_auth_headers == {"alpha-server": {"authorization": "alpha-token"}} + with pytest.raises(TypeError): + context.raw_headers["x-caller"] = "changed" + with pytest.raises(TypeError): + context.mcp_server_auth_headers["alpha-server"]["authorization"] = "changed" + with pytest.raises(FrozenInstanceError): + context.client_ip = "untrusted" + + +def test_operation_context_preserves_missing_and_empty_inputs(): + missing = prepare_context() + empty = prepare_context(mcp_servers=[], raw_headers={}, oauth2_headers={}, mcp_server_auth_headers={}) + assert missing.user_api_key_auth is None + assert missing.mcp_servers is None + assert missing.raw_headers is None + assert missing.oauth2_headers is None + assert missing.mcp_server_auth_headers is None + assert empty.mcp_servers == () + assert empty.raw_headers == {} + assert empty.oauth2_headers == {} + assert empty.mcp_server_auth_headers == {} + + +def test_toolset_request_marker_cannot_be_supplied_by_caller_or_serialized(): + auth = UserAPIKeyAuth.model_validate({"user_id": "alpha", "mcp_toolset_id": "forged"}) + assert auth.mcp_toolset_id is None + auth.mcp_toolset_id = "server-resolved" + assert "mcp_toolset_id" not in auth.model_dump() diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_block_recording.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_block_recording.py index 64d926bc5e3..b8aadef430f 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_block_recording.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_block_recording.py @@ -1,5 +1,6 @@ +from litellm.proxy._experimental.mcp_server import operations as mcp_operations """Tests for guardrail-block recording in -``litellm.proxy._experimental.mcp_server.server.call_mcp_tool``. +``litellm.proxy._experimental.mcp_server.operations.call_mcp_tool``. A pre-call MCP guardrail block *raises* into ``call_mcp_tool``'s ``except Exception``. The failure spend-log row that the Guardrails Monitor's @@ -70,7 +71,7 @@ async def _call_block(logging_obj, order: list, *, user_api_key_auth=mock.sentin with mock.patch.dict(sys.modules, {"litellm.proxy.proxy_server": fake_proxy_server}): with contextlib.suppress(HTTPException): - await server.call_mcp_tool.__wrapped__( + await mcp_operations.call_mcp_tool.__wrapped__( name="t", arguments=None, user_api_key_auth=user_api_key_auth, diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py index 28faf375ab8..9659eb1cbc2 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py @@ -1229,7 +1229,7 @@ class TestResolveByokMcpAuthHeader: user_auth = UserAPIKeyAuth(user_id="user-1", api_key="sk-dashboard") with patch( - "litellm.proxy._experimental.mcp_server.server._get_byok_credential", + "litellm.proxy._experimental.mcp_server.operations._get_byok_credential", new=AsyncMock(return_value="stored-cred"), ): result = await _resolve_byok_mcp_auth_header(server, user_auth, None) @@ -1249,7 +1249,7 @@ class TestResolveByokMcpAuthHeader: user_auth = UserAPIKeyAuth(user_id="user-1", api_key="sk-dashboard") with patch( - "litellm.proxy._experimental.mcp_server.server._get_byok_credential", + "litellm.proxy._experimental.mcp_server.operations._get_byok_credential", new=AsyncMock(return_value=None), ): with pytest.raises(HTTPException) as exc_info: @@ -1272,7 +1272,7 @@ class TestResolveByokMcpAuthHeader: check_mock = AsyncMock(return_value=None) with patch( - "litellm.proxy._experimental.mcp_server.server._check_byok_credential", + "litellm.proxy._experimental.mcp_server.operations._check_byok_credential", new=check_mock, ): result = await _resolve_byok_mcp_auth_header(server, user_auth, "caller-header") diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py index 3f5d4ad83ea..1909e3306a2 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough_tools.py @@ -1,3 +1,4 @@ +from litellm.proxy._experimental.mcp_server import operations as mcp_operations """Unit tests for MCP OAuth passthrough tool-fetch behavior.""" import logging @@ -339,16 +340,16 @@ async def test_aggregate_list_tools_absorbs_one_unauthenticated_server(): raise MCPUpstreamAuthError(status_code=401, www_authenticate=None, server_name=server.name) return [good_tool] - with patch.object(mcp_server, "_get_allowed_mcp_servers", AsyncMock(return_value=[delegate, working])), patch.object( - mcp_server, "_prefetch_oauth_creds_for_user", AsyncMock(return_value={}) - ), patch.object(mcp_server, "_prepare_mcp_server_headers", MagicMock(return_value=(None, None))), patch.object( - mcp_server, "_get_user_oauth_extra_headers_from_db", AsyncMock(return_value=None) + with patch.object(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[delegate, working])), patch.object( + mcp_operations, "_prefetch_oauth_creds_for_user", AsyncMock(return_value={}) + ), patch.object(mcp_operations, "_prepare_mcp_server_headers", MagicMock(return_value=(None, None))), patch.object( + mcp_operations, "_get_user_oauth_extra_headers_from_db", AsyncMock(return_value=None) ), patch.object( - mcp_server, "filter_tools_by_key_team_permissions", AsyncMock(side_effect=lambda tools, **k: tools) + mcp_operations, "filter_tools_by_key_team_permissions", AsyncMock(side_effect=lambda tools, **k: tools) ), patch.object( - mcp_server.global_mcp_server_manager, "_get_tools_from_server", AsyncMock(side_effect=fake_get_tools) + mcp_operations.global_mcp_server_manager, "_get_tools_from_server", AsyncMock(side_effect=fake_get_tools) ): - listing = await mcp_server._get_tools_from_mcp_servers( + listing = await mcp_operations._get_tools_from_mcp_servers( user_api_key_auth=UserAPIKeyAuth(token="h", user_id="u1"), mcp_auth_header=None, mcp_servers=None, @@ -382,14 +383,14 @@ async def test_single_server_route_also_absorbs_upstream_auth_error(): # //mcp sets the path-derived single-server scope; absorption must hold even then. token = _mcp_gateway_server_name.set("delegate_docs") try: - with patch.object(mcp_server, "_get_allowed_mcp_servers", AsyncMock(return_value=[delegate])), patch.object( - mcp_server, "_prefetch_oauth_creds_for_user", AsyncMock(return_value={}) - ), patch.object(mcp_server, "_prepare_mcp_server_headers", MagicMock(return_value=(None, None))), patch.object( - mcp_server, "_get_user_oauth_extra_headers_from_db", AsyncMock(return_value=None) + with patch.object(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[delegate])), patch.object( + mcp_operations, "_prefetch_oauth_creds_for_user", AsyncMock(return_value={}) + ), patch.object(mcp_operations, "_prepare_mcp_server_headers", MagicMock(return_value=(None, None))), patch.object( + mcp_operations, "_get_user_oauth_extra_headers_from_db", AsyncMock(return_value=None) ), patch.object( - mcp_server.global_mcp_server_manager, "_get_tools_from_server", AsyncMock(side_effect=fake_get_tools) + mcp_operations.global_mcp_server_manager, "_get_tools_from_server", AsyncMock(side_effect=fake_get_tools) ): - listing = await mcp_server._get_tools_from_mcp_servers( + listing = await mcp_operations._get_tools_from_mcp_servers( user_api_key_auth=UserAPIKeyAuth(token="h", user_id="u1"), mcp_auth_header=None, mcp_servers=["delegate_docs"], @@ -419,15 +420,15 @@ async def test_aggregate_with_single_accessible_server_still_absorbs(): async def fake_get_tools(server, **kwargs): raise MCPUpstreamAuthError(status_code=401, www_authenticate=None, server_name=server.name) - with patch.object(mcp_server, "_get_allowed_mcp_servers", AsyncMock(return_value=[delegate])), patch.object( - mcp_server, "_prefetch_oauth_creds_for_user", AsyncMock(return_value={}) - ), patch.object(mcp_server, "_prepare_mcp_server_headers", MagicMock(return_value=(None, None))), patch.object( - mcp_server, "_get_user_oauth_extra_headers_from_db", AsyncMock(return_value=None) + with patch.object(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[delegate])), patch.object( + mcp_operations, "_prefetch_oauth_creds_for_user", AsyncMock(return_value={}) + ), patch.object(mcp_operations, "_prepare_mcp_server_headers", MagicMock(return_value=(None, None))), patch.object( + mcp_operations, "_get_user_oauth_extra_headers_from_db", AsyncMock(return_value=None) ), patch.object( - mcp_server.global_mcp_server_manager, "_get_tools_from_server", AsyncMock(side_effect=fake_get_tools) + mcp_operations.global_mcp_server_manager, "_get_tools_from_server", AsyncMock(side_effect=fake_get_tools) ): # Aggregate route: no explicit server filter, even though only one server is accessible. - listing = await mcp_server._get_tools_from_mcp_servers( + listing = await mcp_operations._get_tools_from_mcp_servers( user_api_key_auth=UserAPIKeyAuth(token="h", user_id="u1"), mcp_auth_header=None, mcp_servers=None, @@ -475,3 +476,25 @@ async def test_client_creation_failure_logs_sanitized_exchange(monkeypatch, capl await manager._get_tools_from_server(server) assert "POST https://upstream/ -> HTTP 500" in caplog.text assert "missing_scope" in caplog.text and "query-secret" not in caplog.text + + +@pytest.mark.parametrize( + "oauth_headers,server_headers,authorized", + [ + ({"Authorization": "Bearer upstream"}, None, True), + ({"AUTHORIZATION": "Bearer upstream"}, None, True), + ({"x-unrelated": "present"}, None, False), + (None, {"catalog": {"Authorization": "Bearer scoped"}}, True), + (None, {"other-server": {"Authorization": "Bearer unrelated"}}, False), + (None, {"catalog": {"x-unrelated": "present"}}, False), + (None, {"catalog": "Bearer legacy"}, True), + (None, {"catalog": " "}, False), + ], +) +def test_passthrough_admission_recognizes_only_matching_authorization(oauth_headers, server_headers, authorized): + from litellm.proxy._experimental.mcp_server.operations import _client_has_passthrough_authorization + from litellm.types.mcp import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server = MCPServer(server_id="catalog", name="catalog", alias="catalog", transport=MCPTransport.http) + assert _client_has_passthrough_authorization(server, oauth_headers, server_headers) is authorized diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py index 84d4f1fd083..ed5d67164bd 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_proxy_mode.py @@ -1,3 +1,4 @@ +from litellm.proxy._experimental.mcp_server import operations as mcp_operations import json from datetime import datetime @@ -27,8 +28,8 @@ def proxy_mode(): @pytest.mark.asyncio @pytest.mark.usefixtures("proxy_mode") async def test_proxy_call_rejects_non_proxy_tool_names() -> None: - result = await server._dispatch_virtual_mcp_tool( - name="math_stdio-add", arguments={"a": 1, "b": 2}, user_api_key_auth=AUTH, client_ip=None + result = await mcp_operations._dispatch_virtual_mcp_tool( + name="math_stdio-add", arguments={"a": 1, "b": 2}, user_api_key_auth=AUTH, client_ip=None, mcp_proxy_mode=True ) assert result is not None @@ -105,12 +106,13 @@ async def test_proxy_scope_exception_emits_failure_log(monkeypatch: pytest.Monke arguments = {"tool_id": "denied-scope", "arguments": {}} with pytest.raises(HTTPException) as denied: - await server._dispatch_virtual_mcp_tool( + await mcp_operations._dispatch_virtual_mcp_tool( name="call_tool", arguments=arguments, user_api_key_auth=auth, client_ip=None, mcp_servers=["ungranted"], + mcp_proxy_mode=True, raw_headers={"authorization": "Bearer raw-scope-secret", "x-litellm-call-id": "scope-denial"}, ) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index b2eded67430..b715fe67e20 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -1,3 +1,4 @@ +from litellm.proxy._experimental.mcp_server import operations as mcp_operations import asyncio import contextlib import contextvars @@ -138,7 +139,7 @@ async def test_mcp_server_tool_call_body_contains_request_data(_mcp_request_ctx) mock_add_litellm_data_to_request, ): with patch( - "litellm.proxy._experimental.mcp_server.server.call_mcp_tool", + "litellm.proxy._experimental.mcp_server.operations.call_mcp_tool", mock_call_mcp_tool, ): with patch( @@ -194,7 +195,7 @@ async def test_mcp_server_tool_call_forwards_client_headers_to_logging(_mcp_requ mock_add_litellm_data_to_request, ): with patch( - "litellm.proxy._experimental.mcp_server.server.call_mcp_tool", + "litellm.proxy._experimental.mcp_server.operations.call_mcp_tool", mock_call_mcp_tool, ): with patch("litellm.proxy.proxy_server.proxy_config", MagicMock()): @@ -241,7 +242,7 @@ async def test_mcp_server_tool_call_strips_custom_litellm_key_header(_mcp_reques capturing_add_litellm_data_to_request, ): with patch( - "litellm.proxy._experimental.mcp_server.server.call_mcp_tool", + "litellm.proxy._experimental.mcp_server.operations.call_mcp_tool", mock_call_mcp_tool, ): with patch("litellm.proxy.proxy_server.proxy_config", MagicMock()): @@ -287,11 +288,11 @@ async def test_mcp_server_tool_call_relays_upstream_auth_error_as_iserror(_mcp_r mock_add_litellm_data_to_request, ): with patch( - "litellm.proxy._experimental.mcp_server.server.call_mcp_tool", + "litellm.proxy._experimental.mcp_server.operations.call_mcp_tool", mock_call_mcp_tool, ): with patch("litellm.proxy.proxy_server.proxy_config", MagicMock()): - with patch("litellm.proxy._experimental.mcp_server.server.verbose_logger", mock_logger): + with patch("litellm.proxy._experimental.mcp_server.operations.verbose_logger", mock_logger): result = await mcp_server_tool_call(_mcp_request_ctx(), _call_tool_params("test_tool", {"param": "value"})) assert result.is_error is True @@ -867,15 +868,15 @@ async def test_get_prompts_from_mcp_servers_success(): with ( patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", AsyncMock(return_value=[server_a, server_b]), ) as mock_allowed, patch( - "litellm.proxy._experimental.mcp_server.server._prepare_mcp_server_headers", + "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers", return_value=(None, None), ) as mock_headers, patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", ) as mock_manager, ): mock_manager.get_prompts_from_server = AsyncMock( @@ -927,15 +928,15 @@ async def test_get_resources_from_mcp_servers_success(): with ( patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", AsyncMock(return_value=[server_a, server_b]), ) as mock_allowed, patch( - "litellm.proxy._experimental.mcp_server.server._prepare_mcp_server_headers", + "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers", return_value=(None, None), ) as mock_headers, patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", ) as mock_manager, ): mock_manager.get_resources_from_server = AsyncMock( @@ -992,15 +993,15 @@ async def test_get_resource_templates_from_mcp_servers_success(): with ( patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", AsyncMock(return_value=[server]), ) as mock_allowed, patch( - "litellm.proxy._experimental.mcp_server.server._prepare_mcp_server_headers", + "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers", return_value=(None, None), ) as mock_headers, patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", ) as mock_manager, ): mock_manager.get_resource_templates_from_server = AsyncMock( @@ -1042,15 +1043,15 @@ async def test_mcp_get_prompt_success(): with ( patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", AsyncMock(return_value=[server]), ) as mock_allowed, patch( - "litellm.proxy._experimental.mcp_server.server._prepare_mcp_server_headers", + "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers", return_value=({"Authorization": "token"}, {"X-Test": "1"}), ) as mock_headers, patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", ) as mock_manager, ): mock_manager.get_prompt_from_server = AsyncMock(return_value=prompt_result) @@ -1078,6 +1079,7 @@ async def test_mcp_get_prompt_success(): mcp_auth_header={"Authorization": "token"}, extra_headers={"X-Test": "1"}, raw_headers=None, + client_ip=None, ) assert result is prompt_result @@ -1106,15 +1108,15 @@ async def test_mcp_read_resource_success(): with ( patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", AsyncMock(return_value=[server]), ) as mock_allowed, patch( - "litellm.proxy._experimental.mcp_server.server._prepare_mcp_server_headers", + "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers", return_value=({"Authorization": "token"}, {"X-Test": "1"}), ) as mock_headers, patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", ) as mock_manager, ): mock_manager.read_resource_from_server = AsyncMock(return_value=read_result) @@ -1140,6 +1142,7 @@ async def test_mcp_read_resource_success(): mcp_auth_header={"Authorization": "token"}, extra_headers={"X-Test": "1"}, raw_headers=None, + client_ip=None, ) assert result is read_result @@ -1264,7 +1267,7 @@ async def test_mcp_read_resource_multiple_servers_error(): server_b.name = "server_b" with patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", AsyncMock(return_value=[server_a, server_b]), ) as mock_allowed: with pytest.raises(HTTPException) as exc_info: @@ -1354,11 +1357,11 @@ async def test_get_tools_from_mcp_servers_continues_when_one_server_fails(): mock_manager._get_tools_from_server = mock_get_tools_from_server with patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", mock_manager, ): with patch( - "litellm.proxy._experimental.mcp_server.server.verbose_logger", + "litellm.proxy._experimental.mcp_server.operations.verbose_logger", ) as mock_logger: # Test with server-specific auth headers mcp_server_auth_headers = { @@ -1450,11 +1453,11 @@ async def test_get_tools_from_mcp_servers_handles_all_servers_failing(): mock_manager._get_tools_from_server = mock_get_tools_from_server with patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", mock_manager, ): with patch( - "litellm.proxy._experimental.mcp_server.server.verbose_logger", + "litellm.proxy._experimental.mcp_server.operations.verbose_logger", ) as mock_logger: # Test with server-specific auth headers mcp_server_auth_headers = { @@ -1524,11 +1527,11 @@ async def _denied_scoped_list( with ( patch( # test-quality-ok: the permission resolver is a module-level function; the suite's only seam - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", resolver, ), patch( # test-quality-ok: the server registry is a module-level singleton; the suite's only seam - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", mock_manager, ), ): @@ -1575,11 +1578,11 @@ async def test_empty_scope_lists_nothing_instead_of_raising_a_nameless_denial(): with ( patch( # test-quality-ok: the permission resolver is a module-level function; the suite's only seam - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", resolver, ), patch( # test-quality-ok: the server registry is a module-level singleton; the suite's only seam - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", _denied_scope_manager({"github": "srv-github"}), ), ): @@ -1721,7 +1724,8 @@ async def test_scoped_list_agent_veto_attributed_for_differently_cased_server_na @pytest.mark.asyncio -async def test_handle_list_tools_converts_permission_httpexception_to_mcp_error(_mcp_request_ctx): +@pytest.mark.parametrize("denial_at_auth", [False, True]) +async def test_handle_list_tools_converts_permission_httpexception_to_mcp_error(_mcp_request_ctx, denial_at_auth): """The MCP protocol handler surfaces a permission HTTPException as a clean JSON-RPC error (MCPError, INVALID_REQUEST) carrying the denial message, instead of a raw 500.""" try: @@ -1738,10 +1742,10 @@ async def test_handle_list_tools_converts_permission_httpexception_to_mcp_error( with ( patch( # test-quality-ok: the protocol handler reads auth from module context; no injection seam "litellm.proxy._experimental.mcp_server.server.get_or_extract_auth_context", - new=AsyncMock(return_value=(None, None, None, None, None, None, None)), + new=AsyncMock(return_value=(None, None, None, None, None, None, None), side_effect=denial if denial_at_auth else None), ), patch( # test-quality-ok: the listing helper is the handler's only collaborator; the suite's seam - "litellm.proxy._experimental.mcp_server.server._list_mcp_tools", + "litellm.proxy._experimental.mcp_server.operations._list_mcp_tools", new=AsyncMock(side_effect=denial), ), ): @@ -1768,7 +1772,7 @@ async def test_mcp_server_tool_call_renders_denial_message_not_detail_dict(_mcp_ new=AsyncMock(return_value=(None, None, None, None, None, None, None)), ), patch( # test-quality-ok: the tool-call helper is the handler's only collaborator; the suite's seam - "litellm.proxy._experimental.mcp_server.server.call_mcp_tool", + "litellm.proxy._experimental.mcp_server.operations.call_mcp_tool", new=AsyncMock(side_effect=denial), ), ): @@ -1819,7 +1823,7 @@ async def test_mcp_server_tool_call_body_with_none_arguments(_mcp_request_ctx): mock_add_litellm_data_to_request, ): with patch( - "litellm.proxy._experimental.mcp_server.server.call_mcp_tool", + "litellm.proxy._experimental.mcp_server.operations.call_mcp_tool", mock_call_mcp_tool, ): with patch( @@ -1893,7 +1897,7 @@ async def test_concurrent_initialize_session_managers(): "run", return_value=mock_cm_sse, ) as mock_sse_run, - patch("litellm.proxy._experimental.mcp_server.server.verbose_logger"), + patch("litellm.proxy._experimental.mcp_server.operations.verbose_logger"), ): # Create multiple concurrent tasks that call initialize_session_managers async def init_task(): @@ -2334,7 +2338,7 @@ async def test_mcp_routing_chunked_initialize_to_stateful(): "litellm.proxy._experimental.mcp_server.server.set_auth_context", ), patch( # test-quality-ok: registry is empty in unit tests; key owns one server - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[MagicMock()], ), @@ -2886,7 +2890,7 @@ async def test_initialize_request_tracks_active_session_after_response_header(): return_value=(owner_auth, None, None, None, None, None), ), patch( # test-quality-ok: registry is empty in unit tests; key owns one server - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[MagicMock()], ), @@ -3039,7 +3043,7 @@ async def test_initialize_request_records_client_name_in_gateway_sessions_report return_value=(owner_auth, None, None, None, None, None), ), patch( # test-quality-ok: registry is empty in unit tests; key owns one server - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[MagicMock()], ), @@ -3514,7 +3518,7 @@ async def test_initialize_request_with_existing_session_tracks_new_session(): ), ), patch( # test-quality-ok: registry is empty in unit tests; key owns one server - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[MagicMock()], ), @@ -4248,7 +4252,7 @@ async def test_mcp_routing_with_conflicting_alias_and_group_name(): with ( patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_allowed_mcp_servers", mock_get_allowed, ), patch( @@ -4256,7 +4260,7 @@ async def test_mcp_routing_with_conflicting_alias_and_group_name(): mock_db_lookup, ), patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager._get_tools_from_server", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager._get_tools_from_server", mock_get_tools_spy, ), ): @@ -4365,16 +4369,16 @@ async def test_oauth2_caller_headers_not_forwarded_for_migrated_server(): side_effect=mock_fetch_tools_with_timeout, ), patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", AsyncMock(return_value=[oauth2_server]), ), patch( - "litellm.proxy._experimental.mcp_server.server._prefetch_oauth_creds_for_user", + "litellm.proxy._experimental.mcp_server.operations._prefetch_oauth_creds_for_user", new_callable=AsyncMock, return_value={}, ), patch( - "litellm.proxy._experimental.mcp_server.server._get_user_oauth_extra_headers_from_db", + "litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db", new_callable=AsyncMock, return_value=None, ), @@ -4456,7 +4460,7 @@ async def test_list_tools_single_server_unprefixed_names(): mock_manager._get_tools_from_server = mock_get_tools_from_server with patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", mock_manager, ): listing = await _get_tools_from_mcp_servers( @@ -4535,7 +4539,7 @@ async def test_list_tools_multiple_servers_prefixed_names(): mock_manager._get_tools_from_server = mock_get_tools_from_server with patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", mock_manager, ): listing = await _get_tools_from_mcp_servers( @@ -4715,7 +4719,7 @@ async def test_call_mcp_tool_user_unauthorized_access(): AsyncMock(return_value=["allowed_server", "another_server"]), ), patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_id", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_id", side_effect=mock_get_server_by_id, ), ): @@ -4745,11 +4749,11 @@ async def test_call_mcp_tool_scoped_denial_names_the_binding_agent(): with ( patch( # test-quality-ok: the server registry is a module-level singleton; the suite's only seam - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_allowed_mcp_servers", AsyncMock(return_value=[]), ), patch( # test-quality-ok: the permission resolver is a module-level function; the suite's only seam - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", _scope_resolver({"github": "srv-github"}), ), ): @@ -4821,7 +4825,7 @@ async def test_call_mcp_tool_unauthorized_403_does_not_leak_server_credentials() AsyncMock(return_value=["allowed_server"]), ), patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_id", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_id", side_effect=mock_get_server_by_id, ), ): @@ -4964,7 +4968,7 @@ async def test_list_tools_filters_by_key_team_permissions(): mock_manager._get_tools_from_server = mock_get_tools_from_server with patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", mock_manager, ): listing = await _get_tools_from_mcp_servers( @@ -5075,7 +5079,7 @@ async def test_list_tools_with_team_tool_permissions_inheritance(): mock_manager._get_tools_from_server = mock_get_tools_from_server with patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", mock_manager, ): # Mock the team object permission retrieval @@ -5167,7 +5171,7 @@ async def test_list_tools_with_no_tool_permissions_shows_all(): mock_manager._get_tools_from_server = mock_get_tools_from_server with patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", mock_manager, ): listing = await _get_tools_from_mcp_servers( @@ -5273,7 +5277,7 @@ async def test_list_tools_strips_prefix_when_matching_permissions(): mock_manager._get_tools_from_server = mock_get_tools_from_server with patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", mock_manager, ): listing = await _get_tools_from_mcp_servers( @@ -5715,12 +5719,12 @@ async def test_call_mcp_tool_logs_failure_via_post_call_failure_hook(): return_value=mock_server, ), patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers_from_mcp_server_names", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers_from_mcp_server_names", new_callable=AsyncMock, return_value=[mock_server], ), patch( - "litellm.proxy._experimental.mcp_server.server.execute_mcp_tool", + "litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool", new_callable=AsyncMock, side_effect=Exception("boom"), ), @@ -5784,26 +5788,26 @@ async def test_get_tools_from_mcp_servers_logs_list_tools_to_spendlogs_when_enab with ( patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=[server_a]), ), patch( - "litellm.proxy._experimental.mcp_server.server._prepare_mcp_server_headers", + "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers", return_value=(None, None), ), patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", ) as mock_manager, patch( - "litellm.proxy._experimental.mcp_server.server.filter_tools_by_allowed_tools", + "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_allowed_tools", side_effect=lambda tools, _server: tools, ), patch( - "litellm.proxy._experimental.mcp_server.server.filter_tools_by_key_team_permissions", + "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_key_team_permissions", new=AsyncMock(side_effect=lambda tools, **_: tools), ), patch( - "litellm.proxy._experimental.mcp_server.server.function_setup", + "litellm.proxy._experimental.mcp_server.operations.function_setup", side_effect=_capture_function_setup, ), ): @@ -5866,26 +5870,26 @@ async def test_get_tools_from_mcp_servers_returns_tools_when_success_logging_fai with ( patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=[server_a]), ), patch( - "litellm.proxy._experimental.mcp_server.server._prepare_mcp_server_headers", + "litellm.proxy._experimental.mcp_server.operations._prepare_mcp_server_headers", return_value=(None, None), ), patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", ) as mock_manager, patch( - "litellm.proxy._experimental.mcp_server.server.filter_tools_by_allowed_tools", + "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_allowed_tools", side_effect=lambda tools, _server: tools, ), patch( - "litellm.proxy._experimental.mcp_server.server.filter_tools_by_key_team_permissions", + "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_key_team_permissions", new=AsyncMock(side_effect=lambda tools, **_: tools), ), patch( - "litellm.proxy._experimental.mcp_server.server.function_setup", + "litellm.proxy._experimental.mcp_server.operations.function_setup", return_value=(dummy_logging_obj, None), ), ): @@ -6186,23 +6190,23 @@ async def test_get_tools_from_mcp_servers_injects_stored_oauth2_token(): with ( patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=[oauth2_server]), ), patch( # Patch the bulk prefetch so no real DB connection is needed - "litellm.proxy._experimental.mcp_server.server._prefetch_oauth_creds_for_user", + "litellm.proxy._experimental.mcp_server.operations._prefetch_oauth_creds_for_user", new=AsyncMock(return_value=prefetched_creds), ) as mock_prefetch, patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", ) as mock_manager, patch( - "litellm.proxy._experimental.mcp_server.server.filter_tools_by_allowed_tools", + "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_allowed_tools", side_effect=lambda tools, _server: tools, ), patch( - "litellm.proxy._experimental.mcp_server.server.filter_tools_by_key_team_permissions", + "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_key_team_permissions", new=AsyncMock(side_effect=lambda tools, **_: tools), ), ): @@ -6534,7 +6538,7 @@ class TestGatewayCreateInitializationOptions: with ( patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[scoped_server], ), @@ -6564,7 +6568,7 @@ class TestGatewayCreateInitializationOptions: from litellm.proxy._types import UserAPIKeyAuth with patch( # test-quality-ok: grant resolution is the input under test - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[], ): @@ -6590,7 +6594,7 @@ class TestGatewayCreateInitializationOptions: from litellm.proxy._types import UserAPIKeyAuth with patch( # test-quality-ok: grant resolution is the input under test - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[], ): @@ -6615,7 +6619,7 @@ class TestGatewayCreateInitializationOptions: from litellm.proxy._types import UserAPIKeyAuth with patch( # test-quality-ok: grant resolution is the input under test - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[], ): @@ -6671,7 +6675,7 @@ class TestGatewayCreateInitializationOptions: ), ), patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[scoped_server], ), @@ -6806,14 +6810,14 @@ async def test_list_tools_with_legacy_db_m2m_server_resolves_oauth2_flow(): with ( patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", ) as mock_manager, patch( - "litellm.proxy._experimental.mcp_server.server.filter_tools_by_allowed_tools", + "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_allowed_tools", side_effect=lambda tools, _server: tools, ), patch( - "litellm.proxy._experimental.mcp_server.server.filter_tools_by_key_team_permissions", + "litellm.proxy._experimental.mcp_server.operations.filter_tools_by_key_team_permissions", new=AsyncMock(side_effect=lambda tools, **_: tools), ), ): @@ -7076,7 +7080,7 @@ def _patch_delegate_resolver(server: MCPServer, *resolvable_names: str): return server if name in resolvable_names else None return patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", side_effect=_resolve, ) @@ -7095,7 +7099,7 @@ async def test_legacy_delegate_bare_token_is_not_probed_upstream(): # test-qual with ( _patch_delegate_resolver(server, "delegate_test"), patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=[server]), ), patch( @@ -7131,7 +7135,7 @@ async def test_legacy_delegate_dual_credentials_are_not_probed_upstream(): # te with ( patch( # test-quality-ok: isolate authorized-server resolution so this test targets the preflight boundary - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=[server]), ), patch( # test-quality-ok: the removed probe call is the security regression under test @@ -7178,7 +7182,7 @@ async def test_oauth_passthrough_preflight_preserves_status_contract(probe_statu with ( patch( # test-quality-ok: isolate authorized-server resolution so this test exercises the preflight contract - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=[server]), ), patch( # test-quality-ok: the upstream transport boundary is the behavior being mapped to an HTTP response @@ -7224,7 +7228,7 @@ async def test_delegate_tokenless_request_not_probed(): with ( _patch_delegate_resolver(server, "delegate_test"), patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=[server]), ), patch( @@ -7257,7 +7261,7 @@ async def test_delegate_preflight_skipped_on_multi_server_routes(): with ( _patch_delegate_resolver(servers[0], "delegate_test", "other_server"), patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=servers), ), patch( @@ -7300,7 +7304,7 @@ async def test_bare_authorization_never_probes_passthrough_servers(): with ( _patch_delegate_resolver(passthrough_server, "pt_server"), patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=[passthrough_server]), ), patch( @@ -7346,7 +7350,7 @@ async def test_delegate_not_probed_when_named_only_via_server_id(): with ( _patch_delegate_resolver(server, "delegate_test"), patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=[server]), ), patch( @@ -7379,7 +7383,7 @@ async def test_delegate_probe_not_fanned_out_to_access_group_members(): with ( _patch_delegate_resolver(group_member, "delegate_test"), patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(return_value=[group_member]), ), patch( @@ -7475,11 +7479,11 @@ async def test_execute_mcp_tool_rest_server_id_authoritative_for_unprefixed_tool with ( patch.dict( - mcp_module.global_mcp_server_manager.tool_name_to_mcp_server_name_mapping, + mcp_operations.global_mcp_server_manager.tool_name_to_mcp_server_name_mapping, {"echo": oauth_server.name}, ), patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "get_registry", return_value={ api_key_server.server_id: api_key_server, @@ -7487,13 +7491,12 @@ async def test_execute_mcp_tool_rest_server_id_authoritative_for_unprefixed_tool }, ), patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "_get_mcp_server_from_tool_name", return_value=oauth_server, ), patch.object( - mcp_module, - "_handle_managed_mcp_tool", + mcp_operations, "_handle_managed_mcp_tool", new=fake_handle_managed_mcp_tool, ), patch.object( @@ -7502,12 +7505,12 @@ async def test_execute_mcp_tool_rest_server_id_authoritative_for_unprefixed_tool return_value=True, ), patch.object( - mcp_module.global_mcp_tool_registry, + mcp_operations.global_mcp_tool_registry, "get_tool", return_value=None, ), ): - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name="echo", arguments={"message": "hello"}, allowed_mcp_servers=[api_key_server, oauth_server], @@ -7540,7 +7543,7 @@ def _worker_that_never_listed(server: MCPServer, upstream_tools: tuple[str, ...] from litellm.proxy._experimental.mcp_server import server as mcp_module - mcp_module.global_mcp_server_manager.registry[server.server_id] = server + mcp_operations.global_mcp_server_manager.registry[server.server_id] = server dispatched: dict[str, object] = {} async def fake_handle_managed_mcp_tool(**kwargs): @@ -7552,17 +7555,17 @@ def _worker_that_never_listed(server: MCPServer, upstream_tools: tuple[str, ...] with ( patch.object( # test-quality-ok: the upstream MCP session is the boundary; a real one needs an initialize handshake over a live server - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "_create_mcp_client", new=AsyncMock(return_value=MagicMock()), ) as create_client, patch.object( # test-quality-ok: same boundary, this is the tools/list answer the upstream would give - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "_fetch_tools_with_timeout", side_effect=fake_fetch_tools, ) as fetch_tools, patch.object( # test-quality-ok: records the resolved server and bare name the managed call would forward upstream - mcp_module, "_handle_managed_mcp_tool", new=fake_handle_managed_mcp_tool + mcp_operations, "_handle_managed_mcp_tool", new=fake_handle_managed_mcp_tool ), ): yield SimpleNamespace(create_client=create_client, fetch_tools=fetch_tools, dispatched=dispatched) @@ -7576,7 +7579,7 @@ async def test_execute_mcp_tool_lists_never_listed_passthrough_server_with_calle server = _never_listed_passthrough_server() with _worker_that_never_listed(server, upstream_tools=("add",)) as worker: - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name="lazy_map-add", arguments={"a": 1, "b": 2}, allowed_mcp_servers=[server], @@ -7597,7 +7600,7 @@ async def test_execute_mcp_tool_rest_server_id_lists_never_listed_server_first() server = _never_listed_passthrough_server() with _worker_that_never_listed(server, upstream_tools=("add",)) as worker: - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name="add", arguments={"a": 1, "b": 2}, allowed_mcp_servers=[server], @@ -7620,7 +7623,7 @@ async def test_execute_mcp_tool_unknown_tool_on_never_listed_server_lists_once_t _worker_that_never_listed(server, upstream_tools=("add",)) as worker, pytest.raises(HTTPException) as exc_info, ): - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name="lazy_map-nope", arguments={}, allowed_mcp_servers=[server], @@ -7641,8 +7644,8 @@ async def test_execute_mcp_tool_does_not_relist_a_server_this_worker_already_lis server = _never_listed_passthrough_server() with _worker_that_never_listed(server, upstream_tools=("add",)) as worker: - mcp_module.global_mcp_server_manager._create_prefixed_tools([MCPTool(name="add", inputSchema={})], server) - await mcp_module.execute_mcp_tool( + mcp_operations.global_mcp_server_manager._create_prefixed_tools([MCPTool(name="add", inputSchema={})], server) + await mcp_operations.execute_mcp_tool( name="lazy_map-add", arguments={"a": 1, "b": 2}, allowed_mcp_servers=[server], @@ -7664,8 +7667,8 @@ async def test_execute_mcp_tool_lists_a_tool_this_worker_has_not_yet_seen_on_a_l server = _never_listed_passthrough_server() with _worker_that_never_listed(server, upstream_tools=("add", "multiply")) as worker: - mcp_module.global_mcp_server_manager._create_prefixed_tools([MCPTool(name="add", inputSchema={})], server) - await mcp_module.execute_mcp_tool( + mcp_operations.global_mcp_server_manager._create_prefixed_tools([MCPTool(name="add", inputSchema={})], server) + await mcp_operations.execute_mcp_tool( name="lazy_map-multiply", arguments={"a": 1, "b": 2}, allowed_mcp_servers=[server], @@ -7688,7 +7691,7 @@ async def test_execute_mcp_tool_never_lists_a_server_the_caller_cannot_access(): _worker_that_never_listed(server, upstream_tools=("add",)) as worker, pytest.raises(HTTPException) as exc_info, ): - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name="lazy_map-add", arguments={"a": 1, "b": 2}, allowed_mcp_servers=[other_server], @@ -7734,13 +7737,12 @@ async def test_execute_mcp_tool_strips_a_prefix_that_contains_the_separator(): with ( patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "_get_mcp_server_from_tool_name", return_value=alias_less_server, ), patch.object( - mcp_module, - "_handle_managed_mcp_tool", + mcp_operations, "_handle_managed_mcp_tool", new=fake_handle_managed_mcp_tool, ), patch.object( @@ -7749,12 +7751,12 @@ async def test_execute_mcp_tool_strips_a_prefix_that_contains_the_separator(): return_value=True, ), patch.object( - mcp_module.global_mcp_tool_registry, + mcp_operations.global_mcp_tool_registry, "get_tool", return_value=None, ), ): - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name=f"{server_id}-read_wiki_contents", arguments={"repoName": "acme/wiki"}, allowed_mcp_servers=[alias_less_server], @@ -7808,11 +7810,11 @@ async def test_execute_mcp_tool_rest_server_id_injects_requested_server_credenti with ( patch.dict( - mcp_module.global_mcp_server_manager.tool_name_to_mcp_server_name_mapping, + mcp_operations.global_mcp_server_manager.tool_name_to_mcp_server_name_mapping, {"echo": collision_server.name, "echo_requested-echo": requested_server.name}, ), patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "get_registry", return_value={ requested_server.server_id: requested_server, @@ -7820,7 +7822,7 @@ async def test_execute_mcp_tool_rest_server_id_injects_requested_server_credenti }, ), patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "_create_mcp_client", new=fake_create_mcp_client, ), @@ -7830,13 +7832,13 @@ async def test_execute_mcp_tool_rest_server_id_injects_requested_server_credenti return_value=True, ), patch.object( - mcp_module.global_mcp_tool_registry, + mcp_operations.global_mcp_tool_registry, "get_tool", return_value=None, ), patch("litellm.proxy.proxy_server.proxy_logging_obj", None), ): - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name="echo", arguments={"message": "hello"}, allowed_mcp_servers=[requested_server, collision_server], @@ -7879,7 +7881,7 @@ async def test_execute_mcp_tool_rest_prefixed_tool_still_validates_server_id(): with ( patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "get_registry", return_value={ api_key_server.server_id: api_key_server, @@ -7887,7 +7889,7 @@ async def test_execute_mcp_tool_rest_prefixed_tool_still_validates_server_id(): }, ), patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "_get_mcp_server_from_tool_name", return_value=oauth_server, ), @@ -7897,13 +7899,13 @@ async def test_execute_mcp_tool_rest_prefixed_tool_still_validates_server_id(): return_value=True, ), patch.object( - mcp_module.global_mcp_tool_registry, + mcp_operations.global_mcp_tool_registry, "get_tool", return_value=None, ), pytest.raises(HTTPException) as exc_info, ): - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name="echo_oauth_m2m-echo", arguments={"message": "hello"}, allowed_mcp_servers=[api_key_server, oauth_server], @@ -7941,7 +7943,7 @@ async def test_execute_mcp_tool_rest_unauthorized_prefix_still_mismatches(): with ( patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "get_registry", return_value={ api_key_server.server_id: api_key_server, @@ -7949,7 +7951,7 @@ async def test_execute_mcp_tool_rest_unauthorized_prefix_still_mismatches(): }, ), patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "_get_mcp_server_from_tool_name", return_value=restricted_server, ), @@ -7959,13 +7961,13 @@ async def test_execute_mcp_tool_rest_unauthorized_prefix_still_mismatches(): return_value=True, ), patch.object( - mcp_module.global_mcp_tool_registry, + mcp_operations.global_mcp_tool_registry, "get_tool", return_value=None, ), pytest.raises(HTTPException) as exc_info, ): - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name="restricted_server-echo", arguments={"message": "hello"}, allowed_mcp_servers=[api_key_server], @@ -8005,18 +8007,17 @@ async def test_execute_mcp_tool_rest_hyphenated_upstream_tool_name_routes_to_req with ( patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "get_registry", return_value={api_key_server.server_id: api_key_server}, ), patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "_get_mcp_server_from_tool_name", return_value=None, ), patch.object( - mcp_module, - "_handle_managed_mcp_tool", + mcp_operations, "_handle_managed_mcp_tool", new=fake_handle_managed_mcp_tool, ), patch.object( @@ -8025,12 +8026,12 @@ async def test_execute_mcp_tool_rest_hyphenated_upstream_tool_name_routes_to_req return_value=True, ), patch.object( - mcp_module.global_mcp_tool_registry, + mcp_operations.global_mcp_tool_registry, "get_tool", return_value=None, ), ): - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name="text-to-speech", arguments={"message": "hello"}, allowed_mcp_servers=[api_key_server], @@ -8089,22 +8090,22 @@ async def test_execute_mcp_tool_sets_model_in_model_call_details(): with ( patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "_get_mcp_server_from_tool_name", return_value=fake_server, ), patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "pre_call_tool_check", new=AsyncMock(return_value={}), ), patch.object( - mcp_module.global_mcp_tool_registry, + mcp_operations.global_mcp_tool_registry, "get_tool", return_value=fake_tool, ), patch( - "litellm.proxy._experimental.mcp_server.server._handle_local_mcp_tool", + "litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool", new=AsyncMock(return_value=[]), ), patch( @@ -8112,7 +8113,7 @@ async def test_execute_mcp_tool_sets_model_in_model_call_details(): return_value=True, ), ): - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name="list_pets", arguments={"limit": 10}, allowed_mcp_servers=[fake_server], @@ -8168,7 +8169,7 @@ async def test_execute_mcp_tool_rest_unresolved_prefixed_name_routes_to_requeste with ( patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "get_registry", return_value={ requested_server.server_id: requested_server, @@ -8176,13 +8177,12 @@ async def test_execute_mcp_tool_rest_unresolved_prefixed_name_routes_to_requeste }, ), patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "_get_mcp_server_from_tool_name", return_value=None, ), patch.object( - mcp_module, - "_handle_managed_mcp_tool", + mcp_operations, "_handle_managed_mcp_tool", new=fake_handle_managed_mcp_tool, ), patch.object( @@ -8191,12 +8191,12 @@ async def test_execute_mcp_tool_rest_unresolved_prefixed_name_routes_to_requeste return_value=True, ), patch.object( - mcp_module.global_mcp_tool_registry, + mcp_operations.global_mcp_tool_registry, "get_tool", return_value=None, ), ): - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name="known_prefix-list_things", arguments={"message": "hello"}, allowed_mcp_servers=[requested_server, prefix_owner], @@ -8248,7 +8248,7 @@ async def test_execute_mcp_tool_rest_prefix_retry_resolution_still_enforces_serv with ( patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "get_registry", return_value={ requested_server.server_id: requested_server, @@ -8256,7 +8256,7 @@ async def test_execute_mcp_tool_rest_prefix_retry_resolution_still_enforces_serv }, ), patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "_get_mcp_server_from_tool_name", side_effect=resolve_only_when_requested_prefix_added, ), @@ -8266,13 +8266,13 @@ async def test_execute_mcp_tool_rest_prefix_retry_resolution_still_enforces_serv return_value=True, ), patch.object( - mcp_module.global_mcp_tool_registry, + mcp_operations.global_mcp_tool_registry, "get_tool", return_value=None, ), pytest.raises(HTTPException) as exc_info, ): - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name="known_prefix-echo", arguments={"message": "hello"}, allowed_mcp_servers=[requested_server, prefix_owner], @@ -9175,14 +9175,14 @@ async def test_call_tool_with_legacy_db_m2m_server_resolves_oauth2_flow(): with ( patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", ) as mock_manager, patch( - "litellm.proxy._experimental.mcp_server.server.execute_mcp_tool", + "litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool", side_effect=capture_execute, ), patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers_from_mcp_server_names", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers_from_mcp_server_names", new=AsyncMock(side_effect=lambda mcp_servers, allowed_mcp_servers: allowed_mcp_servers), ), ): @@ -9260,12 +9260,12 @@ async def test_call_mcp_tool_skips_failure_hook_for_upstream_auth_error(): ), patch.object(global_mcp_server_manager, "get_mcp_server_by_id", return_value=mock_server), patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers_from_mcp_server_names", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers_from_mcp_server_names", new_callable=AsyncMock, return_value=[mock_server], ), patch( - "litellm.proxy._experimental.mcp_server.server.execute_mcp_tool", + "litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool", new_callable=AsyncMock, side_effect=MCPUpstreamAuthError(status_code=401, www_authenticate="Bearer", server_name="test_server"), ), @@ -9345,7 +9345,7 @@ async def test_aggregate_listing_reports_per_server_outcomes(): mock_manager._get_tools_from_server = mock_get_tools_from_server with patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", mock_manager, ): listing = await _get_tools_from_mcp_servers( @@ -9419,7 +9419,7 @@ async def test_handle_list_tools_attaches_outcome_meta(_mcp_request_ctx): new=AsyncMock(return_value=(None, None, None, None, None, None, None)), ), patch( - "litellm.proxy._experimental.mcp_server.server._list_mcp_tools", + "litellm.proxy._experimental.mcp_server.operations._list_mcp_tools", new=AsyncMock(return_value=listing), ), ): @@ -9485,12 +9485,12 @@ class TestPreemptive401ModeAware: with ( patch.object( - server_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "get_mcp_server_by_name", return_value=server, ), patch.object( - server_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "has_user_oauth_token", new_callable=AsyncMock, return_value=has_stored_token, @@ -9509,7 +9509,7 @@ class TestPreemptive401ModeAware: async def test_deferred_discovery_runs_before_delegate_challenge(self): from litellm.proxy._experimental.mcp_server import server as server_module - manager = server_module.global_mcp_server_manager + manager = mcp_operations.global_mcp_server_manager server = _make_oauth2_server( "lazy_delegate", oauth2_flow="authorization_code", @@ -9541,7 +9541,7 @@ class TestPreemptive401ModeAware: async def test_stamped_m2m_challenge_skips_deferred_discovery(self): from litellm.proxy._experimental.mcp_server import server as server_module - manager = server_module.global_mcp_server_manager + manager = mcp_operations.global_mcp_server_manager server = _make_oauth2_server("stamped_m2m", oauth2_flow="client_credentials") with patch.object( @@ -9584,12 +9584,12 @@ class TestPreemptive401ModeAware: with ( patch.dict(os.environ, {"SERVER_ROOT_PATH": "/litellm"}), patch.object( - server_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "get_mcp_server_by_name", return_value=server, ), patch.object( - server_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "has_user_oauth_token", new_callable=AsyncMock, return_value=False, @@ -9691,17 +9691,17 @@ class TestSingleServerPreflightReachesIdJag: with ( patch.object( # test-quality-ok: route wiring must use the manager's configured server - server_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "get_mcp_server_by_name", return_value=server, ), patch.object( # test-quality-ok: route wiring must invoke the manager preflight - server_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "preflight_token_exchange", preflight, ), patch.object( # test-quality-ok: allowed-set resolution needs the DB; the test controls its answer - server_module, "_get_allowed_mcp_servers", AsyncMock(return_value=[server]) + mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[server]) ), ): await server_module._raise_preemptive_401_for_unauthenticated_servers( @@ -9751,12 +9751,12 @@ class TestSingleServerPreflightReachesIdJag: with ( patch.object( # test-quality-ok: route wiring must use the manager's configured server - server_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "get_mcp_server_by_name", return_value=token_exchange, ), patch.object( # test-quality-ok: route wiring must invoke the manager preflight - server_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "preflight_token_exchange", preflight, ), @@ -9817,13 +9817,13 @@ class TestOboPreflightScopedToAllowedServers: preflight = AsyncMock() with ( patch.object( # test-quality-ok: route handler reads the module-level manager, no injection seam - server_module.global_mcp_server_manager, "get_mcp_server_by_name", return_value=requested + mcp_operations.global_mcp_server_manager, "get_mcp_server_by_name", return_value=requested ), patch.object( # test-quality-ok: the exchanger is the observable; a real one would call an IdP - server_module.global_mcp_server_manager, "preflight_token_exchange", preflight + mcp_operations.global_mcp_server_manager, "preflight_token_exchange", preflight ), patch.object( # test-quality-ok: allowed-set resolution needs the DB; the test controls its answer - server_module, "_get_allowed_mcp_servers", allowed_lookup + mcp_operations, "_get_allowed_mcp_servers", allowed_lookup ), ): await server_module._raise_preemptive_401_for_unauthenticated_servers( @@ -10132,7 +10132,7 @@ class TestListFiltersHonorThePrefixBoundary: with ( patch.object(MCPRequestHandler, "get_allowed_tools_for_server", AsyncMock(return_value=grants)), - patch("litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager") as mock_manager, + patch("litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager") as mock_manager, ): mock_manager.get_mcp_server_by_id.return_value = server @@ -10195,11 +10195,11 @@ async def test_list_tools_injects_byok_credential_for_non_oauth2_auth_types(auth with ( patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", mock_manager, ), patch( - "litellm.proxy._experimental.mcp_server.server._get_byok_credential", + "litellm.proxy._experimental.mcp_server.operations._get_byok_credential", AsyncMock(return_value="personal-api-key"), ), ): @@ -10292,3 +10292,44 @@ async def test_streamable_http_rejects_modern_protocol_version(header_value: str assert header_value in body["error"]["message"] for version in body["error"]["message"].split("supported: ")[1].split(", "): assert version in HANDSHAKE_PROTOCOL_VERSIONS + + +@pytest.mark.asyncio +@pytest.mark.parametrize("handler_name,field", [ + ("handle_list_tools", "tools"), + ("list_prompts", "prompts"), + ("list_resources", "resources"), + ("list_resource_templates", "resource_templates"), +]) +async def test_native_listing_preserves_empty_result_on_auth_failure(_mcp_request_ctx, handler_name, field): + from litellm.proxy._experimental.mcp_server import server + + with patch.object(server, "get_or_extract_auth_context", AsyncMock(side_effect=RuntimeError("auth failure"))): + result = await getattr(server, handler_name)(_mcp_request_ctx(), _paged_params()) + assert getattr(result, field) == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure_hook_raises", [False, True]) +async def test_tool_listing_preserves_permission_denial_when_failure_logging_fails(failure_hook_raises): + from litellm.proxy._experimental.mcp_server import operations + from litellm.proxy import proxy_server + + auth = UserAPIKeyAuth(user_id="denied-caller") + denial = HTTPException(status_code=403, detail="scope denied") + logger = MagicMock() + logger.post_call_failure_hook = AsyncMock(side_effect=RuntimeError("log unavailable") if failure_hook_raises else None) + upstream = AsyncMock() + with ( + patch.object(operations, "_get_allowed_mcp_servers", AsyncMock(side_effect=denial)), + patch.object(operations, "function_setup", return_value=(None, None)), + patch.object(proxy_server, "proxy_logging_obj", logger), + patch.object(operations.global_mcp_server_manager, "_get_tools_from_server", upstream), + ): + with pytest.raises(HTTPException) as rejected: + await operations._get_tools_from_mcp_servers(user_api_key_auth=auth, mcp_auth_header=None, mcp_servers=["catalog"], log_list_tools_to_spendlogs=True) + assert rejected.value is denial + upstream.assert_not_awaited() + logger.post_call_failure_hook.assert_awaited_once() + assert logger.post_call_failure_hook.await_args.kwargs["original_exception"] is denial + assert logger.post_call_failure_hook.await_args.kwargs["user_api_key_dict"] == auth diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index ac23831b5f4..9f42a523350 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -73,6 +73,135 @@ from litellm.proxy.utils import ProxyLogging from litellm.types.guardrails import GuardrailEventHooks +@pytest.mark.asyncio +async def test_manager_sampling_preserves_explicit_headers_without_ambient_context(): + from litellm.proxy._experimental.mcp_server import server as legacy_server + + caller = UserAPIKeyAuth(user_id="sampling-caller") + upstream = MCPServer( + server_id="sampling-context", + name="sampling_context", + url="https://example.invalid/mcp", + transport=MCPTransport.http, + allow_sampling=True, + ) + sampling = AsyncMock() + client = MagicMock() + client.call_tool = AsyncMock(return_value=CallToolResult(content=[])) + assert legacy_server.get_active_auth_context() is None + with ( + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.MCPClient", return_value=client) as factory, + patch("litellm.proxy._experimental.mcp_server.sampling_handler.handle_sampling_create_message", sampling), + ): + await MCPServerManager()._call_regular_mcp_tool( + mcp_server=upstream, + original_tool_name="probe", + arguments={}, + tasks=[], + mcp_auth_header=None, + mcp_server_auth_headers=None, + oauth2_headers=None, + raw_headers={"x-test-caller": "sampling-caller"}, + proxy_logging_obj=None, + user_api_key_auth=caller, + ) + callback = factory.call_args.kwargs["sampling_callback"] + await callback(None, None) + assert sampling.await_args.kwargs["user_api_key_auth"].user_id == "sampling-caller" + assert sampling.await_args.kwargs["raw_headers"] == {"x-test-caller": "sampling-caller"} + + + +@pytest.mark.asyncio +async def test_sampling_callback_keeps_creation_context_after_caller_switch(): + from mcp.server.auth.middleware.auth_context import auth_context_var + + from litellm.proxy._experimental.mcp_server import server as legacy_server + from litellm.proxy._experimental.mcp_server.mcp_server_manager import _create_sampling_callback + + token = auth_context_var.set(None) + recorder = AsyncMock() + try: + original = UserAPIKeyAuth(user_id="alpha", models=["alpha-model"]) + original.mcp_admitted_user_subject = True + headers = {"x-caller": "alpha"} + legacy_server.set_auth_context(original, raw_headers=headers, client_ip="192.0.2.1") + callback = _create_sampling_callback() + original.models.append("bravo-model") + headers["x-caller"] = "bravo" + legacy_server.set_auth_context(UserAPIKeyAuth(user_id="bravo"), raw_headers={"x-caller": "bravo"}) + with patch("litellm.proxy._experimental.mcp_server.sampling_handler.handle_sampling_create_message", recorder): + await callback(None, None) + observed = recorder.await_args.kwargs + assert observed["user_api_key_auth"].user_id == "alpha" + assert observed["user_api_key_auth"].models == ["alpha-model"] + assert observed["user_api_key_auth"].mcp_admitted_user_subject is True + assert observed["raw_headers"] == {"x-caller": "alpha"} + assert observed["client_ip"] == "192.0.2.1" + finally: + auth_context_var.reset(token) + + +@pytest.mark.asyncio +async def test_elicitation_callback_keeps_initiating_session(): + from litellm.proxy._experimental.mcp_server import server as legacy_server + from litellm.proxy._experimental.mcp_server.mcp_server_manager import _create_elicitation_callback + + initiating = MagicMock() + replacement = MagicMock() + recorder = AsyncMock() + token = legacy_server.active_mcp_session_var.set(initiating) + try: + callback = _create_elicitation_callback() + legacy_server.active_mcp_session_var.set(replacement) + with patch("litellm.proxy._experimental.mcp_server.elicitation_handler.handle_elicitation_request", recorder): + await callback(None, None) + assert recorder.await_args.kwargs["downstream_session"] is initiating + assert recorder.await_args.kwargs["downstream_capabilities"] is initiating.capabilities + finally: + legacy_server.active_mcp_session_var.reset(token) + + +@pytest.mark.asyncio +async def test_sampling_callbacks_isolate_callers_and_cancellation(): + from mcp.types import ErrorData + + from litellm.proxy._experimental.mcp_server.mcp_server_manager import _create_sampling_callback + + started = asyncio.Event() + cancelled = asyncio.Event() + observed = {} + + async def record_sampling(*, user_api_key_auth, raw_headers, **kwargs): + label = user_api_key_auth.user_id + if label == "cancelled": + started.set() + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + cancelled.set() + raise + await asyncio.sleep(0) + observed[label] = raw_headers["x-caller"] + return ErrorData(code=-1, message=label) + + callbacks = tuple( + _create_sampling_callback(UserAPIKeyAuth(user_id=label), raw_headers={"x-caller": label}) + for label in ("alpha", "bravo", "cancelled") + ) + with patch( + "litellm.proxy._experimental.mcp_server.sampling_handler.handle_sampling_create_message", record_sampling + ): + tasks = tuple(asyncio.create_task(callback(None, None)) for callback in callbacks) + await asyncio.wait_for(started.wait(), timeout=2) + tasks[2].cancel() + results = await asyncio.gather(*tasks, return_exceptions=True) + assert observed == {"alpha": "alpha", "bravo": "bravo"} + assert [result.message for result in results[:2]] == ["alpha", "bravo"] + assert isinstance(results[2], asyncio.CancelledError) + assert cancelled.is_set() + + def _reload_mcp_manager_module(): utils_module = sys.modules["litellm.proxy._experimental.mcp_server.utils"] manager_module = sys.modules["litellm.proxy._experimental.mcp_server.mcp_server_manager"] @@ -84,6 +213,9 @@ def _reload_mcp_manager_module(): server_module = sys.modules.get("litellm.proxy._experimental.mcp_server.server") if server_module is not None and hasattr(server_module, "global_mcp_server_manager"): server_module.global_mcp_server_manager = reloaded.global_mcp_server_manager + operations_module = sys.modules.get("litellm.proxy._experimental.mcp_server.operations") + if operations_module is not None: + operations_module.global_mcp_server_manager = reloaded.global_mcp_server_manager return reloaded @@ -3923,6 +4055,7 @@ class TestMCPServerManager: result = await manager.get_resource_templates_from_server( server=server, user_api_key_auth=None, + raw_headers=None, mcp_auth_header="auth", extra_headers=None, add_prefix=False, @@ -3935,6 +4068,8 @@ class TestMCPServerManager: stdio_env=None, subject_token=None, user_api_key_auth=None, + raw_headers=None, + client_ip=None, ) mock_client.list_resource_templates.assert_awaited_once() assert result == expected_templates @@ -5849,7 +5984,7 @@ class TestMCPServerManager: stored = {"Authorization": "Bearer stored-user-token"} with patch( - "litellm.proxy._experimental.mcp_server.server._get_user_oauth_extra_headers_from_db", + "litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db", new=AsyncMock(return_value=stored), ) as mock_lookup: result = await manager._resolve_oauth2_headers_for_tool_call( @@ -5876,7 +6011,7 @@ class TestMCPServerManager: user_auth = UserAPIKeyAuth(api_key="sk-test", user_id="alice") with patch( - "litellm.proxy._experimental.mcp_server.server._get_user_oauth_extra_headers_from_db", + "litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db", new=AsyncMock(return_value={"Authorization": "Bearer should-not-be-used"}), ) as mock_lookup: result = await manager._resolve_oauth2_headers_for_tool_call( @@ -5902,7 +6037,7 @@ class TestMCPServerManager: user_auth = UserAPIKeyAuth(api_key="sk-test", user_id="alice") with patch( - "litellm.proxy._experimental.mcp_server.server._get_user_oauth_extra_headers_from_db", + "litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db", new=AsyncMock(side_effect=RuntimeError("redis down")), ): result = await manager._resolve_oauth2_headers_for_tool_call( @@ -6058,7 +6193,7 @@ class TestMCPServerManager: user_auth = UserAPIKeyAuth(api_key="sk-test") with patch( - "litellm.proxy._experimental.mcp_server.server._get_user_oauth_extra_headers_from_db", + "litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db", new=AsyncMock(return_value={"Authorization": "Bearer x"}), ) as mock_lookup: result = await manager._resolve_oauth2_headers_for_tool_call( @@ -6862,7 +6997,8 @@ class TestMCPServerManager: } user_api_key_auth = UserAPIKeyAuth(api_key="sk-test", user_id="user-123") - token = _mcp_active_toolset_id.set("toolset-abc") + user_api_key_auth.mcp_toolset_id = "toolset-abc" + token = _mcp_active_toolset_id.set("unrelated-ambient-toolset") try: with ( patch.object(proxy_server_module, "user_api_key_cache", cache), @@ -14332,3 +14468,36 @@ async def test_request_selected_during_guardrail_runs_concurrently_with_tool(mon assert guardrail_started.is_set() is selected assert result.is_error is False assert result.content[0].text == "executed" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("with_caller,legacy_factory", [(True, False), (False, False), (True, True)]) +async def test_client_sampling_does_not_fill_explicit_context_from_another_ambient_caller(with_caller, legacy_factory): + from mcp.server.auth.middleware.auth_context import auth_context_var + from litellm.proxy._experimental.mcp_server import server as legacy_server + from litellm.proxy._experimental.mcp_server.mcp_server_manager import _create_sampling_callback + + upstream = MCPServer(server_id="explicit-empty", name="explicit_empty", url="https://example.invalid/mcp", transport=MCPTransport.http, allow_sampling=True) + token = auth_context_var.set(None) + sampling = AsyncMock() + try: + legacy_server.set_auth_context(UserAPIKeyAuth(user_id="unrelated"), raw_headers={"authorization": "unrelated-credential"}, client_ip="192.0.2.99") + with ( + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.MCPClient") as factory, + patch("litellm.proxy._experimental.mcp_server.sampling_handler.handle_sampling_create_message", sampling), + ): + if legacy_factory: + callback = _create_sampling_callback(user_api_key_auth=UserAPIKeyAuth(user_id="explicit")) + else: + await MCPServerManager()._create_mcp_client(upstream, user_api_key_auth=UserAPIKeyAuth(user_id="explicit") if with_caller else None) + callback = factory.call_args.kwargs["sampling_callback"] + await callback(None, None) + captured = sampling.await_args.kwargs + if with_caller: + assert captured["user_api_key_auth"].user_id == "explicit" + else: + assert captured["user_api_key_auth"] is None + assert captured["raw_headers"] is None + assert captured["client_ip"] is None + finally: + auth_context_var.reset(token) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py index 9420eecd222..ec6fdef69ee 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py @@ -639,12 +639,12 @@ async def test_per_user_oauth_missing_stored_token_returns_preemptive_401(): return_value=False, ), patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.has_user_oauth_token", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token", new_callable=AsyncMock, return_value=False, ) as mock_has_token, patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=oauth_server, ), patch.object( @@ -727,12 +727,12 @@ async def test_admitted_subject_missing_stored_token_challenged_with_resource_me return_value=False, ), patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.has_user_oauth_token", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token", new_callable=AsyncMock, return_value=False, ) as mock_has_token, patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=oauth_server, ), patch.object( @@ -833,11 +833,11 @@ async def test_client_credentials_server_is_not_preemptively_challenged(m2m_fiel return_value=False, ), patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.has_user_oauth_token", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token", new_callable=AsyncMock, ) as mock_has_token, patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=m2m_server, ), patch.object(session_manager_stateless, "handle_request", new_callable=AsyncMock) as mock_handle_request, @@ -929,16 +929,16 @@ async def test_handle_streamable_http_mcp_delegated_server_surfaces_upstream_cha return_value=False, ), patch( - "litellm.proxy._experimental.mcp_server.server._get_user_oauth_extra_headers_from_db", + "litellm.proxy._experimental.mcp_server.operations._get_user_oauth_extra_headers_from_db", new_callable=AsyncMock, return_value=None, ), patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=delegated_server, ), patch( # test-quality-ok: registry is empty in unit tests; key owns the delegated server - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[delegated_server], ), @@ -1022,12 +1022,12 @@ async def test_per_user_oauth_with_stored_token_skips_preemptive_401(): return_value=False, ), patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.has_user_oauth_token", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token", new_callable=AsyncMock, return_value=True, ) as mock_has_token, patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=oauth_server, ), patch.object( @@ -1126,11 +1126,11 @@ async def test_handle_streamable_http_mcp_delegated_server_without_token_returns return_value=False, ), patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.has_user_oauth_token", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.has_user_oauth_token", new_callable=AsyncMock, ) as mock_has_token, patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=delegated_server, ), patch.object( @@ -1218,7 +1218,7 @@ async def test_handle_streamable_http_mcp_token_exchange_without_subject_returns return_value=False, ), patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=obo_server, ), patch.object( @@ -1317,7 +1317,7 @@ async def test_handle_streamable_http_mcp_oauth_delegate_without_token_returns_g True, ), patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=od_server, ), patch.object( @@ -1391,7 +1391,7 @@ async def test_handle_streamable_http_mcp_oauth_delegate_with_forwarded_token_sk new_callable=AsyncMock, ), patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=od_server, ), patch.object( @@ -1453,7 +1453,7 @@ async def _run_passthrough_connect( new_callable=AsyncMock, ), patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=server, ), patch.object(session_manager_stateless, "handle_request", new_callable=AsyncMock) as mock_handle_request, @@ -1574,7 +1574,7 @@ async def test_handle_streamable_http_mcp_true_passthrough_without_token_surface return_value=probe_client, ), patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=tp_server, ), patch.object( @@ -1642,7 +1642,7 @@ async def test_handle_streamable_http_mcp_true_passthrough_dcr_bridge_challenges return_value=probe_client, ), patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=bridge_server, ), patch.object( @@ -1720,7 +1720,7 @@ async def test_handle_streamable_http_mcp_true_passthrough_with_token_skips_prob return_value=probe_client, ), patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager.get_mcp_server_by_name", return_value=tp_server, ), patch.object( diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py index cb43d2c2592..4575741aa8b 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py @@ -1,3 +1,4 @@ +from litellm.proxy._experimental.mcp_server import operations as mcp_operations """ Tests for MCP tool search feature. @@ -572,7 +573,7 @@ class TestCallToolRestApiVirtualTools: mock_tool.input_schema = {"type": "object", "properties": {}} with patch( - "litellm.proxy._experimental.mcp_server.server._list_mcp_tools", + "litellm.proxy._experimental.mcp_server.operations._list_mcp_tools", new_callable=AsyncMock, return_value=AggregateToolListing(tools=[mock_tool], outcomes={}), ): @@ -616,12 +617,12 @@ class TestCallToolRestApiVirtualTools: with ( patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[MagicMock()], ), patch( - "litellm.proxy._experimental.mcp_server.server.execute_mcp_tool", + "litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool", new_callable=AsyncMock, return_value=fake_result, ) as mock_execute, @@ -669,12 +670,12 @@ class TestCallToolRestApiVirtualTools: return_value="203.0.113.7", ), patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[MagicMock()], ) as mock_allowed, patch( - "litellm.proxy._experimental.mcp_server.server.execute_mcp_tool", + "litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool", new_callable=AsyncMock, return_value=fake_result, ), @@ -699,7 +700,7 @@ class TestCallToolRestApiVirtualTools: return_value="203.0.113.7", ), patch( - "litellm.proxy._experimental.mcp_server.server._list_mcp_tools", + "litellm.proxy._experimental.mcp_server.operations._list_mcp_tools", new_callable=AsyncMock, return_value=AggregateToolListing(tools=[], outcomes={}), ) as mock_list, @@ -832,7 +833,7 @@ class TestCallToolRestApiVirtualTools: "litellm.proxy.proxy_server.proxy_logging_obj", key_limits ), patch( # test-quality-ok: the authorized catalog is the seam every virtual tool shares; the ranking under test stays real - "litellm.proxy._experimental.mcp_server.server._list_mcp_tools", + "litellm.proxy._experimental.mcp_server.operations._list_mcp_tools", new_callable=AsyncMock, return_value=AggregateToolListing(tools=list(CATALOG), outcomes={}), ) as mock_list, @@ -939,7 +940,7 @@ class TestDispatchVirtualMcpTool: new_callable=AsyncMock, return_value="SEARCH_RESULT", ) as mock_search: - result = await srv._dispatch_virtual_mcp_tool( + result = await mcp_operations._dispatch_virtual_mcp_tool( name=MCP_TOOL_SEARCH_TOOL_NAME, arguments={"query": "q", "top_k": 3}, user_api_key_auth=uak, @@ -961,7 +962,7 @@ class TestDispatchVirtualMcpTool: new_callable=AsyncMock, return_value="AGENT_RESULT", ) as mock_agent_search: - result = await srv._dispatch_virtual_mcp_tool( + result = await mcp_operations._dispatch_virtual_mcp_tool( name=AGENT_SEARCH_TOOL_NAME, arguments={"query": "translate a document", "top_k": "2"}, user_api_key_auth=uak, @@ -996,7 +997,7 @@ class TestDispatchVirtualMcpTool: new_callable=AsyncMock, return_value="CALL_RESULT", ) as mock_call: - result = await srv._dispatch_virtual_mcp_tool( + result = await mcp_operations._dispatch_virtual_mcp_tool( name=MCP_TOOL_CALL_TOOL_NAME, arguments={"tool_name": "math-add", "arguments": {"a": 1, "b": 2}}, user_api_key_auth=uak, @@ -1027,8 +1028,7 @@ class TestDispatchVirtualMcpTool: sentinel_logging_obj = object() with ( patch.object( - srv, - "_build_virtual_call_logging_obj", + mcp_operations, "_build_virtual_call_logging_obj", new_callable=AsyncMock, return_value=sentinel_logging_obj, ) as mock_build, @@ -1038,7 +1038,7 @@ class TestDispatchVirtualMcpTool: return_value="CALL_RESULT", ) as mock_call, ): - await srv._dispatch_virtual_mcp_tool( + await mcp_operations._dispatch_virtual_mcp_tool( name=MCP_TOOL_CALL_TOOL_NAME, arguments={"tool_name": "math-add", "arguments": {"a": 1}}, user_api_key_auth=uak, @@ -1060,7 +1060,7 @@ class TestDispatchVirtualMcpTool: new_callable=AsyncMock, return_value="SEARCH_RESULT", ) as mock_search: - await srv._dispatch_virtual_mcp_tool( + await mcp_operations._dispatch_virtual_mcp_tool( name=MCP_TOOL_SEARCH_TOOL_NAME, arguments={"query": "issue", "top_k": "not-a-number"}, user_api_key_auth=uak, @@ -1083,12 +1083,12 @@ class TestDispatchVirtualMcpTool: fake = CallToolResult(content=[TextContent(type="text", text="ok")], isError=False) with ( patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[MagicMock()], ) as mock_allowed, patch( - "litellm.proxy._experimental.mcp_server.server.execute_mcp_tool", + "litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool", new_callable=AsyncMock, return_value=fake, ) as mock_exec, @@ -1130,12 +1130,12 @@ class TestDispatchVirtualMcpTool: uak = UserAPIKeyAuth(api_key="k", object_permission=_make_perm(mcp_tool_search_enabled=True)) with ( patch( - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new_callable=AsyncMock, return_value=[], ), patch( - "litellm.proxy._experimental.mcp_server.server.execute_mcp_tool", + "litellm.proxy._experimental.mcp_server.operations.execute_mcp_tool", new_callable=AsyncMock, ) as mock_exec, ): @@ -1217,7 +1217,7 @@ class TestMcpServerToolCallErrorHandling: return_value=(uak, None, None, None, None, None, None), ), patch( - "litellm.proxy._experimental.mcp_server.server._dispatch_virtual_mcp_tool", + "litellm.proxy._experimental.mcp_server.operations._dispatch_virtual_mcp_tool", new_callable=AsyncMock, side_effect=HTTPException(status_code=403, detail="User not allowed to call this tool"), ), @@ -1254,7 +1254,7 @@ async def test_handle_mcp_tool_call_scoped_denial_names_the_binding_agent() -> N ] with patch( # test-quality-ok: the permission resolver is a module-level function; the suite's only seam - "litellm.proxy._experimental.mcp_server.server._get_allowed_mcp_servers", + "litellm.proxy._experimental.mcp_server.operations._get_allowed_mcp_servers", new=AsyncMock(side_effect=resolve), ): with pytest.raises(HTTPException) as exc_info: diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_toolset_scope.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_toolset_scope.py index 519acc241c6..1398884783e 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_toolset_scope.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_toolset_scope.py @@ -58,6 +58,22 @@ class TestApplyToolsetScope: assert set(op.mcp_servers or []) == {"server-a", "server-b"} assert op.mcp_tool_permissions == toolset_perms + from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + from litellm.proxy._experimental.mcp_server.operations import prepare_context + + manager = MCPServerManager() + unscoped_open = await manager.operator_open_server_ids( + auth, allow_all_server_ids=["operator-open-outside-toolset"], submitted_server_ids=[] + ) + scoped_open = await manager.operator_open_server_ids( + prepare_context(result).user_api_key_auth, + allow_all_server_ids=["operator-open-outside-toolset"], + submitted_server_ids=[], + ) + assert unscoped_open == {"operator-open-outside-toolset"} + assert scoped_open == set() + assert auth.mcp_toolset_id is None + @pytest.mark.asyncio async def test_admin_creates_object_permission_when_none(self): """Admin key with object_permission=None can access any toolset.""" @@ -564,7 +580,7 @@ class TestMCPActiveToolsetContextVar: MagicMock(get_mcp_client_ip=MagicMock(return_value="127.0.0.1")), ), patch( - "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager", + "litellm.proxy._experimental.mcp_server.operations.global_mcp_server_manager", MagicMock(get_mcp_server_by_name=MagicMock(return_value=None)), ), patch( diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py index ac716bace3c..bb70f38285c 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_tool_auth.py @@ -1,3 +1,4 @@ +from litellm.proxy._experimental.mcp_server import operations as mcp_operations """ VERIA-7 regression: OpenAPI-backed (local-registry) MCP tools must run through `pre_call_tool_check` before dispatch, the same as managed @@ -49,22 +50,22 @@ async def test_openapi_local_tool_runs_pre_call_tool_check(): with ( patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "_get_mcp_server_from_tool_name", return_value=fake_server, ), patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "pre_call_tool_check", new=pre_call, ), patch.object( - mcp_module.global_mcp_tool_registry, + mcp_operations.global_mcp_tool_registry, "get_tool", return_value=fake_tool, ), patch( - "litellm.proxy._experimental.mcp_server.server._handle_local_mcp_tool", + "litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool", new=handle_local, ), patch( @@ -72,7 +73,7 @@ async def test_openapi_local_tool_runs_pre_call_tool_check(): return_value=True, ), ): - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name="list_pets", arguments={"limit": 10}, allowed_mcp_servers=[fake_server], @@ -92,7 +93,7 @@ async def test_openapi_local_tool_runs_pre_call_tool_check(): assert pre_call_kwargs["guardrail_context"] == {"metadata": {"guardrails": ("block-all",)}} assert pre_call_kwargs["name"] == "list_pets" assert pre_call_kwargs["server"] is fake_server - assert pre_call_kwargs["user_api_key_auth"] is user + assert pre_call_kwargs["user_api_key_auth"] == user # `proxy_logging_obj` must be sourced from the canonical proxy_server # module (same as the managed path) — passing None would crash the # downstream `_create_mcp_request_object_from_kwargs` call with @@ -134,22 +135,22 @@ async def test_openapi_local_tool_blocked_when_pre_call_check_raises(): with ( patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "_get_mcp_server_from_tool_name", return_value=fake_server, ), patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "pre_call_tool_check", new=pre_call, ), patch.object( - mcp_module.global_mcp_tool_registry, + mcp_operations.global_mcp_tool_registry, "get_tool", return_value=fake_tool, ), patch( - "litellm.proxy._experimental.mcp_server.server._handle_local_mcp_tool", + "litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool", new=handle_local, ), patch( @@ -158,7 +159,7 @@ async def test_openapi_local_tool_blocked_when_pre_call_check_raises(): ), ): with pytest.raises(HTTPException) as exc: - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name="delete_pet", arguments={}, allowed_mcp_servers=[fake_server], @@ -195,24 +196,24 @@ async def test_openapi_local_tool_denied_when_server_not_resolvable(): # `_get_mcp_server_from_tool_name` returns None — no server context. with ( - patch.object(mcp_module, "_resolve_openapi_tool_auth", new=resolve_auth), + patch.object(mcp_operations, "_resolve_openapi_tool_auth", new=resolve_auth), patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "_get_mcp_server_from_tool_name", return_value=None, ), patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "pre_call_tool_check", new=pre_call, ), patch.object( - mcp_module.global_mcp_tool_registry, + mcp_operations.global_mcp_tool_registry, "get_tool", return_value=fake_tool, ), patch( - "litellm.proxy._experimental.mcp_server.server._handle_local_mcp_tool", + "litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool", new=handle_local, ), patch( @@ -221,7 +222,7 @@ async def test_openapi_local_tool_denied_when_server_not_resolvable(): ), ): with pytest.raises(HTTPException) as exc: - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name="list_pets", arguments={}, allowed_mcp_servers=[], @@ -280,27 +281,27 @@ async def test_openapi_local_tool_injects_resolved_oauth_token(): with ( patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "_get_mcp_server_from_tool_name", return_value=oauth_server, ), patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "pre_call_tool_check", new=AsyncMock(return_value={}), ), patch.object( - mcp_module.global_mcp_tool_registry, + mcp_operations.global_mcp_tool_registry, "get_tool", return_value=fake_tool, ), patch.object( - mcp_module.global_mcp_server_manager._cred_provider, + mcp_operations.global_mcp_server_manager._cred_provider, "resolve_credentials", new=AsyncMock(return_value=Ok(StaticHeaderAuth("Bearer stored-user-token"))), ), patch( - "litellm.proxy._experimental.mcp_server.server._handle_local_mcp_tool", + "litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool", new=handle_local, ), patch( @@ -308,7 +309,7 @@ async def test_openapi_local_tool_injects_resolved_oauth_token(): return_value=True, ), ): - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name="get_values", arguments={}, allowed_mcp_servers=[oauth_server], @@ -417,7 +418,7 @@ async def test_legacy_local_tool_fallback_refuses_unentitled_caller(legacy_local ) with pytest.raises(HTTPException) as exc: - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name=f"{LEGACY_SERVER_NAME}-{LEGACY_TOOL}", arguments={}, allowed_mcp_servers=[server], @@ -451,7 +452,7 @@ async def test_legacy_local_tool_fallback_still_dispatches_entitled_caller( server, executed = legacy_local_tool user = _caller_entitled_to([LEGACY_TOOL]) - result = await mcp_module.execute_mcp_tool( + result = await mcp_operations.execute_mcp_tool( name=f"{LEGACY_SERVER_NAME}-{LEGACY_TOOL}", arguments={}, allowed_mcp_servers=[server], @@ -481,7 +482,7 @@ async def test_legacy_local_tool_fallback_fails_closed_on_empty_prefix( _server, executed = legacy_local_tool with pytest.raises(HTTPException) as exc: - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name=f"-{LEGACY_TOOL}", arguments={}, allowed_mcp_servers=[], @@ -523,7 +524,7 @@ async def test_legacy_local_tool_fallback_fails_closed_when_prefix_names_no_serv return_value=True, ): with pytest.raises(HTTPException) as exc: - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name=f"{LEGACY_SERVER_NAME}-{LEGACY_TOOL}", arguments={}, allowed_mcp_servers=[other_server], @@ -546,7 +547,7 @@ async def test_unknown_tool_name_still_reports_not_found(): from litellm.proxy._experimental.mcp_server import server as mcp_module with pytest.raises(HTTPException) as exc: - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name="tool_no_registry_knows", arguments={}, allowed_mcp_servers=[], @@ -610,7 +611,7 @@ async def test_per_server_auth_header_reaches_both_openapi_dispatch_arms(dispatc captured["injected"] = _request_auth_header.get() return [] - manager = mcp_module.global_mcp_server_manager + manager = mcp_operations.global_mcp_server_manager with ( patch.object(manager, "resolve_openapi_upstream_auth", new=fake_resolver), patch.object(manager, "pre_call_tool_check", new=AsyncMock(return_value={})), @@ -620,9 +621,9 @@ async def test_per_server_auth_header_reaches_both_openapi_dispatch_arms(dispatc fake_tool.name = "list_reports" with ( patch.object(manager, "_get_mcp_server_from_tool_name", return_value=server), - patch.object(mcp_module.global_mcp_tool_registry, "get_tool", return_value=fake_tool), + patch.object(mcp_operations.global_mcp_tool_registry, "get_tool", return_value=fake_tool), patch( - "litellm.proxy._experimental.mcp_server.server._handle_local_mcp_tool", + "litellm.proxy._experimental.mcp_server.operations._handle_local_mcp_tool", new=capture_local, ), patch( @@ -630,7 +631,7 @@ async def test_per_server_auth_header_reaches_both_openapi_dispatch_arms(dispatc return_value=True, ), ): - await mcp_module.execute_mcp_tool( + await mcp_operations.execute_mcp_tool( name="list_reports", arguments={}, allowed_mcp_servers=[server], @@ -702,11 +703,11 @@ async def test_local_dispatch_reports_the_outcome_instead_of_success(failure: st user = UserAPIKeyAuth(api_key="sk-user", user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER.value) with ( - patch.object(mcp_module.global_mcp_server_manager, "_get_mcp_server_from_tool_name", return_value=server), - patch.object(mcp_module.global_mcp_server_manager, "pre_call_tool_check", new=AsyncMock(return_value={})), - patch.object(mcp_module.global_mcp_tool_registry, "get_tool", return_value=fake_tool), + patch.object(mcp_operations.global_mcp_server_manager, "_get_mcp_server_from_tool_name", return_value=server), + patch.object(mcp_operations.global_mcp_server_manager, "pre_call_tool_check", new=AsyncMock(return_value={})), + patch.object(mcp_operations.global_mcp_tool_registry, "get_tool", return_value=fake_tool), patch.object( - mcp_module.global_mcp_server_manager, + mcp_operations.global_mcp_server_manager, "resolve_openapi_upstream_auth", new=AsyncMock(return_value=(None, None)), ), @@ -715,7 +716,7 @@ async def test_local_dispatch_reports_the_outcome_instead_of_success(failure: st return_value=True, ), ): - call = mcp_module.execute_mcp_tool( + call = mcp_operations.execute_mcp_tool( name="list_reports", arguments={}, allowed_mcp_servers=[server], diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_operations.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_operations.py new file mode 100644 index 00000000000..abb925ddc77 --- /dev/null +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_operations.py @@ -0,0 +1,365 @@ +import asyncio +from unittest.mock import AsyncMock, patch + +import pytest +from mcp.types import GetPromptRequest, GetPromptRequestParams, GetPromptResult + +from litellm.proxy._experimental.mcp_server.operations import GatewayOperations, prepare_context +from litellm.proxy._types import UserAPIKeyAuth + + +@pytest.mark.asyncio +async def test_oauth_prefetch_failure_does_not_log_caller_or_exception_text(caplog): + from litellm.proxy._experimental.mcp_server.operations import _prefetch_oauth_creds_for_user + + user_id = "caller\nFORGED-USER-LINE" + fetch = AsyncMock(side_effect=RuntimeError("database\nFORGED-ERROR-LINE")) + database = object() + with ( + patch("litellm.proxy.utils.get_prisma_client_or_throw", return_value=database), + patch("litellm.proxy._experimental.mcp_server.db.list_user_oauth_credentials", fetch), + caplog.at_level("WARNING", logger="LiteLLM"), + ): + result = await _prefetch_oauth_creds_for_user(UserAPIKeyAuth(user_id=user_id)) + assert result == {} + fetch.assert_awaited_once_with(database, user_id) + warnings = [record.getMessage() for record in caplog.records if "prefetch" in record.getMessage()] + assert len(warnings) == 1 + assert "failed" in warnings[0] + assert "\n" not in warnings[0] + assert "FORGED" not in warnings[0] + + +@pytest.mark.asyncio +async def test_dispatch_uses_explicit_context_when_ambient_caller_differs(): + from mcp.server.auth.middleware.auth_context import auth_context_var + from litellm.proxy._experimental.mcp_server.server import set_auth_context + + context = prepare_context( + UserAPIKeyAuth(user_id="alpha"), + raw_headers={"x-caller": "alpha"}, + mcp_servers=["alpha-server"], + client_ip="192.0.2.1", + ) + token = auth_context_var.set(None) + handler = AsyncMock(return_value=GetPromptResult(messages=[])) + try: + set_auth_context(UserAPIKeyAuth(user_id="bravo"), raw_headers={"x-caller": "bravo"}) + with patch("litellm.proxy._experimental.mcp_server.operations.mcp_get_prompt", handler): + result = await GatewayOperations().execute( + GetPromptRequest(params=GetPromptRequestParams(name="alpha-prompt")), context + ) + assert result.messages == [] + assert handler.await_args.kwargs["name"] == "alpha-prompt" + assert handler.await_args.kwargs["user_api_key_auth"].user_id == "alpha" + assert handler.await_args.kwargs["raw_headers"] == {"x-caller": "alpha"} + assert handler.await_args.kwargs["mcp_servers"] == ["alpha-server"] + assert handler.await_args.kwargs["client_ip"] == "192.0.2.1" + finally: + auth_context_var.reset(token) + + +@pytest.mark.asyncio +async def test_legacy_adapter_cleans_context_after_cancelled_operation(): + from types import SimpleNamespace + from litellm.proxy._experimental.mcp_server import server + from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var + + previous_session = server.active_mcp_session_var.get() + previous_request = active_mcp_request_ctx_var.get() + request = SimpleNamespace(session=object()) + auth = (None, None, None, None, None, None, None) + + async def cancelled_operation(): + async with server._legacy_operation_context(request, trace=False): + assert server.active_mcp_session_var.get() is request.session + assert active_mcp_request_ctx_var.get() is request + raise asyncio.CancelledError + + with patch( + "litellm.proxy._experimental.mcp_server.server.get_or_extract_auth_context", AsyncMock(return_value=auth) + ): + with pytest.raises(asyncio.CancelledError): + await cancelled_operation() + assert server.active_mcp_session_var.get() is previous_session + assert active_mcp_request_ctx_var.get() is previous_request + + +@pytest.mark.asyncio +async def test_legacy_adapter_cleans_context_when_trace_setup_fails(): + from types import SimpleNamespace + from litellm.proxy._experimental.mcp_server import server + from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var + + previous_session = server.active_mcp_session_var.get() + previous_request = active_mcp_request_ctx_var.get() + request = SimpleNamespace(session=object()) + + async def enter_operation(): + async with server._legacy_operation_context(request, trace=True): + pytest.fail("Trace setup failure must prevent dispatch") + + with patch.object(server, "_otel_set_mcp_transport_span", side_effect=RuntimeError("trace failure")): + with pytest.raises(RuntimeError, match="trace failure"): + await enter_operation() + assert server.active_mcp_session_var.get() is previous_session + assert active_mcp_request_ctx_var.get() is previous_request + + +@pytest.mark.asyncio +async def test_prompt_sampling_receives_explicit_operation_caller_headers_and_ip(): + from unittest.mock import MagicMock + from litellm.proxy._experimental.mcp_server import operations + from litellm.types.mcp import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + upstream = MCPServer( + server_id="explicit-prompt", + name="explicit_prompt", + url="https://example.invalid/mcp", + transport=MCPTransport.http, + allow_sampling=True, + ) + context = prepare_context( + UserAPIKeyAuth(user_id="prompt-caller"), + raw_headers={"x-caller": "prompt-caller"}, + client_ip="192.0.2.41", + ) + client = MagicMock() + client.get_prompt = AsyncMock(return_value=GetPromptResult(messages=[])) + sampling = AsyncMock() + with ( + patch.object(operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[upstream])), + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.MCPClient", return_value=client) as factory, + patch("litellm.proxy._experimental.mcp_server.sampling_handler.handle_sampling_create_message", sampling), + ): + result = await GatewayOperations().execute( + GetPromptRequest(params=GetPromptRequestParams(name="explicit_prompt-prompt")), context + ) + assert result.messages == [] + await factory.call_args.kwargs["sampling_callback"](None, None) + captured = sampling.await_args.kwargs + assert captured["user_api_key_auth"] is not None + assert captured["user_api_key_auth"].user_id == "prompt-caller" + assert captured["raw_headers"] == {"x-caller": "prompt-caller"} + assert captured["client_ip"] == "192.0.2.41" + + +def _catalog_case(method): + from mcp import types + + cases = { + "prompts/list": ( + types.ListPromptsRequest(), + "list_prompts", + "get_prompts_from_server", + [types.Prompt(name="catalog-prompt")], + "prompts", + ), + "prompts/get": ( + types.GetPromptRequest( + params=types.GetPromptRequestParams(name="catalog-prompt", arguments={"topic": "test"}) + ), + "get_prompt", + "get_prompt_from_server", + types.GetPromptResult(messages=[]), + None, + ), + "resources/list": ( + types.ListResourcesRequest(), + "list_resources", + "get_resources_from_server", + [types.Resource(name="document", uri="https://example.com/document")], + "resources", + ), + "resources/templates/list": ( + types.ListResourceTemplatesRequest(), + "list_resource_templates", + "get_resource_templates_from_server", + [types.ResourceTemplate(name="document", uri_template="https://example.com/{name}")], + "resource_templates", + ), + "resources/read": ( + types.ReadResourceRequest(params=types.ReadResourceRequestParams(uri="https://example.com/document")), + "read_resource", + "read_resource_from_server", + types.ReadResourceResult( + contents=[types.TextResourceContents(uri="https://example.com/document", text="document body")] + ), + None, + ), + } + return cases[method] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method", ["prompts/list", "prompts/get", "resources/list", "resources/templates/list", "resources/read"] +) +@pytest.mark.parametrize("state", ["success", "denied", "upstream_failure", "scope_failure"]) +async def test_native_catalog_operations_preserve_context_results_and_failure_policy(method, state): + from types import SimpleNamespace + from fastapi import HTTPException + from mcp.server.context import ServerRequestContext + from mcp.types import PaginatedRequestParams + from litellm.proxy._experimental.mcp_server import operations, server + from litellm.types.mcp import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + operation, handler_name, manager_method, payload, collection = _catalog_case(method) + caller = UserAPIKeyAuth(user_id="catalog-caller") + headers = {"x-caller": "catalog-caller"} + upstream_server = MCPServer(server_id="catalog", name="catalog", transport=MCPTransport.http) + allowed = AsyncMock( + return_value=[] if state == "denied" else [upstream_server], + side_effect=HTTPException(status_code=403, detail="scope denied") if state == "scope_failure" else None, + ) + upstream = AsyncMock( + return_value=payload, side_effect=RuntimeError("upstream unavailable") if state == "upstream_failure" else None + ) + ctx = ServerRequestContext( + session=SimpleNamespace(), lifespan_context={}, protocol_version="2025-06-18", method=method + ) + auth = (caller, None, ["catalog"], None, None, headers, "192.0.2.41") + with ( + patch.object(server, "get_or_extract_auth_context", AsyncMock(return_value=auth)), + patch.object(operations, "_get_allowed_mcp_servers", allowed), + patch.object(operations.global_mcp_server_manager, manager_method, upstream), + ): + if collection is None and state != "success": + expected_error = RuntimeError if state == "upstream_failure" else HTTPException + with pytest.raises(expected_error): + await getattr(server, handler_name)(ctx, operation.params) + else: + result = await getattr(server, handler_name)(ctx, operation.params or PaginatedRequestParams()) + if collection: + assert getattr(result, collection) == (payload if state == "success" else []) + else: + assert result == payload + assert allowed.await_args.kwargs == { + "user_api_key_auth": caller, + "mcp_servers": ["catalog"], + "client_ip": "192.0.2.41", + } + if state in ("denied", "scope_failure"): + upstream.assert_not_awaited() + else: + upstream.assert_awaited_once() + forwarded = upstream.await_args.kwargs + assert forwarded["user_api_key_auth"] == caller + assert forwarded["raw_headers"] == headers + assert forwarded["client_ip"] == "192.0.2.41" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method", ["prompts/list", "prompts/get", "resources/list", "resources/templates/list", "resources/read"] +) +async def test_explicit_proxy_context_rejects_catalog_operations_before_upstream_access(method): + from mcp.shared.exceptions import MCPError + from mcp.types import METHOD_NOT_FOUND + from litellm.proxy._experimental.mcp_server import operations + + operation, _, manager_method, _, _ = _catalog_case(method) + upstream = AsyncMock() + with patch.object(operations.global_mcp_server_manager, manager_method, upstream): + with pytest.raises(MCPError) as rejected: + await GatewayOperations().execute(operation, prepare_context(mcp_proxy_mode=True)) + assert rejected.value.error.code == METHOD_NOT_FOUND + assert rejected.value.error.message == "Operation unavailable on /mcp/proxy" + upstream.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", ["missing_env", "pii", "guardrail", "unexpected"]) +async def test_tool_operation_preserves_failure_messages_and_request_trace(failure): + from mcp.types import CallToolRequest, CallToolRequestParams + from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException + from litellm.proxy._experimental.mcp_server import operations + from litellm.proxy._experimental.mcp_server.utils import MCPMissingUserEnvVarsError + + failures = { + "missing_env": ( + MCPMissingUserEnvVarsError( + server_id="server", server_name="server", missing=["TOKEN"], setup_url="https://example.com/setup" + ), + "https://example.com/setup", + ), + "pii": ( + BlockedPiiEntityError(entity_type="EMAIL_ADDRESS", guardrail_name="test"), + "Blocked PII entity detected", + ), + "guardrail": (GuardrailRaisedException(message="request denied"), "Guardrail violation"), + "unexpected": (RuntimeError("upstream unavailable"), "Error: upstream unavailable"), + } + error, expected = failures[failure] + dispatch = AsyncMock(side_effect=error) + context = prepare_context( + raw_headers={"x-litellm-trace-id": "operation-trace", "authorization": "private-test-header"} + ) + with patch.object(operations, "call_mcp_tool", dispatch): + result = await GatewayOperations().execute( + CallToolRequest(params=CallToolRequestParams(name="catalog-tool", arguments={})), context + ) + assert result.is_error is True + assert expected in result.content[0].text + assert "private-test-header" not in result.content[0].text + dispatch.assert_awaited_once() + assert dispatch.await_args.kwargs["litellm_trace_id"] == "operation-trace" + assert dispatch.await_args.kwargs["litellm_session_id"] == "operation-trace" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method,helper", + [ + ("prompts/list", "_list_mcp_prompts"), + ("resources/list", "_list_mcp_resources"), + ("resources/templates/list", "_list_mcp_resource_templates"), + ], +) +async def test_catalog_operation_preserves_empty_result_for_malformed_upstream_items(method, helper): + from litellm.proxy._experimental.mcp_server import operations + + operation, _, _, _, collection = _catalog_case(method) + with patch.object(operations, helper, AsyncMock(return_value=[{"unexpected": "item"}])): + result = await GatewayOperations().execute(operation, prepare_context()) + assert getattr(result, collection) == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("catalog_unavailable", [False, True]) +async def test_tool_listing_returns_empty_result_without_dispatch_for_unavailable_catalog(catalog_unavailable): + from mcp.types import ListToolsRequest + from litellm.proxy._experimental.mcp_server import operations + + allowed = AsyncMock( + return_value=[], side_effect=RuntimeError("catalog unavailable") if catalog_unavailable else None + ) + upstream = AsyncMock() + with ( + patch.object(operations, "_get_allowed_mcp_servers", allowed), + patch.object(operations.global_mcp_server_manager, "_get_tools_from_server", upstream), + ): + result = await GatewayOperations().execute(ListToolsRequest(), prepare_context()) + assert result.tools == [] + allowed.assert_awaited_once() + upstream.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_explicit_proxy_context_lists_builtin_tools_and_blocks_direct_tool_dispatch(): + from mcp.types import CallToolRequest, CallToolRequestParams, ListToolsRequest + from litellm.proxy._experimental.mcp_server import operations + + context = prepare_context(mcp_proxy_mode=True) + allowed = AsyncMock() + with patch.object(operations, "_get_allowed_mcp_servers", allowed): + listing = await GatewayOperations().execute(ListToolsRequest(), context) + denied = await GatewayOperations().execute( + CallToolRequest(params=CallToolRequestParams(name="catalog-tool", arguments={})), context + ) + assert {tool.name for tool in listing.tools} == {"search_tools", "get_tool_schema", "call_tool"} + assert denied.is_error is True + assert "unavailable on /mcp/proxy" in denied.content[0].text + allowed.assert_not_awaited() diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index 13af58c15c0..233a8cc96ba 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -1,3 +1,4 @@ +from litellm.proxy._experimental.mcp_server import operations as mcp_operations import asyncio import inspect import json @@ -1253,6 +1254,7 @@ class TestListToolsRestAPI: user_api_key_auth=None, extra_headers=None, apply_tool_filters=True, + client_ip=None, ): captured["called"] = True captured["server"] = server @@ -1338,6 +1340,7 @@ class TestListToolsRestAPI: user_api_key_auth=None, extra_headers=None, apply_tool_filters=True, + client_ip=None, ): captured["user_api_key_auth"] = user_api_key_auth return ["tool-1"] @@ -1891,6 +1894,7 @@ class TestListToolsRestAPI: user_api_key_auth=None, extra_headers=None, apply_tool_filters=True, + client_ip=None, ): captured["called"] = True captured["server_arg"] = server @@ -2027,6 +2031,7 @@ class TestListToolsRestAPI: user_api_key_auth=None, extra_headers=None, apply_tool_filters=True, + client_ip=None, ): captured["called"] = True captured["server_arg"] = server @@ -2112,6 +2117,7 @@ class TestListToolsRestAPI: user_api_key_auth=None, extra_headers=None, apply_tool_filters=True, + client_ip=None, ): return ["scoped-tool"] @@ -2319,6 +2325,7 @@ class TestListToolsRestAPI: user_api_key_auth=None, extra_headers=None, apply_tool_filters=True, + client_ip=None, ): captured["server"] = server captured["auth_header"] = server_auth_header @@ -3145,10 +3152,10 @@ async def test_request_selected_tool_specific_guardrail_applies_to_virtual_execu monkeypatch.setattr(litellm, "callbacks", [guardrail]) monkeypatch.setattr(tool_registry, "global_mcp_tool_registry", registry) monkeypatch.setattr(mcp_server_manager, "global_mcp_server_manager", manager) - monkeypatch.setattr(server, "global_mcp_tool_registry", registry) - monkeypatch.setattr(server, "global_mcp_server_manager", manager) + monkeypatch.setattr(mcp_operations, "global_mcp_tool_registry", registry) + monkeypatch.setattr(mcp_operations, "global_mcp_server_manager", manager) monkeypatch.setattr(rest_endpoints, "global_mcp_server_manager", manager) - monkeypatch.setattr(server, "_get_allowed_mcp_servers", AsyncMock(return_value=[managed_server])) + monkeypatch.setattr(mcp_operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[managed_server])) monkeypatch.setattr(proxy_server, "proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache())) monkeypatch.setattr(proxy_server, "add_litellm_data_to_request", passthrough_request_data) monkeypatch.setattr(proxy_server, "proxy_config", {}) diff --git a/tests/test_litellm/proxy/auth/test_login_utils.py b/tests/test_litellm/proxy/auth/test_login_utils.py index 55ece36252d..3786169c320 100644 --- a/tests/test_litellm/proxy/auth/test_login_utils.py +++ b/tests/test_litellm/proxy/auth/test_login_utils.py @@ -5,12 +5,15 @@ This module tests the refactored login logic that was moved from proxy_server.py to login_utils.py for better reusability. """ +import hashlib import os from collections.abc import Mapping from contextlib import ExitStack from typing import TYPE_CHECKING, Final +from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest if TYPE_CHECKING: @@ -34,6 +37,7 @@ def _unlimited_throttle(): from litellm.constants import LITELLM_PROXY_ADMIN_NAME +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy._types import ( LiteLLM_UserTable, LitellmUserRoles, @@ -46,8 +50,13 @@ from litellm.proxy.auth.login_utils import ( authenticate_user, get_ui_credentials, is_env_credential_login_enabled, + screen_login_password_for_breach, ) +# Successful DB-user logins schedule the background HIBP screen; disable it so +# no test ever does live network I/O to haveibeenpwned.com from CI. +_POLICY_NO_BREACH_CHECK = {"password_policy_check_breached_passwords": False} + def test_get_ui_credentials_prefers_explicit_password(): """The configured UI password should be returned when available.""" @@ -326,6 +335,7 @@ async def test_authenticate_user_email_case_insensitive_login(): master_key=master_key, prisma_client=mock_prisma_client, throttle=_unlimited_throttle(), + general_settings=_POLICY_NO_BREACH_CHECK, ) result_lower = await authenticate_user( username=stored_email, @@ -333,6 +343,7 @@ async def test_authenticate_user_email_case_insensitive_login(): master_key=master_key, prisma_client=mock_prisma_client, throttle=_unlimited_throttle(), + general_settings=_POLICY_NO_BREACH_CHECK, ) assert result_mixed.user_id == result_lower.user_id == "test-user-123" @@ -576,6 +587,7 @@ async def test_authenticate_user_database_login_with_non_ascii_password(): master_key=master_key, prisma_client=mock_prisma_client, throttle=_unlimited_throttle(), + general_settings=_POLICY_NO_BREACH_CHECK, ) assert isinstance(result, LoginResult) @@ -721,7 +733,12 @@ async def _db_login(throttle, username: str, password: str, *, correct: bool): ), ): return await authenticate_user( - username=username, password=password, master_key="sk-master", prisma_client=MagicMock(), throttle=throttle + username=username, + password=password, + master_key="sk-master", + prisma_client=MagicMock(), + throttle=throttle, + general_settings=_POLICY_NO_BREACH_CHECK, ) @@ -2064,3 +2081,265 @@ class TestIsEnvCredentialLoginEnabled: with ExitStack() as stack: _patch_sso_configured(stack, configured=False) assert is_env_credential_login_enabled({"disable_password_login_when_sso_enabled": True}) is True + + +def _db_user_row(*, password: str, password_reset_required: bool | None = None, last_breach_check_at=None): + hashed = hash_token(token=password) + row = MagicMock() + row.user_id = "reset-user-1" + row.user_email = "reset@example.com" + row.password = hashed + row.user_role = LitellmUserRoles.INTERNAL_USER + row.password_reset_required = password_reset_required + row.last_breach_check_at = last_breach_check_at + return row + + +def _prisma_with_user(row) -> MagicMock: + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=row) + mock_prisma_client.db.litellm_usertable.update = AsyncMock(return_value=row) + return mock_prisma_client + + +_DB_LOGIN_ENV = { + "DATABASE_URL": "postgresql://test:test@localhost/test", + "UI_USERNAME": "admin", + "UI_PASSWORD": "admin-password", +} + + +class TestPasswordResetRequiredSessionMinting: + """A user flagged `password_reset_required` must receive a UI session key + restricted to the change-password endpoint (server-side enforcement, so a + script driving the management API with the session key is blocked too); + an unflagged user must keep getting an unrestricted key.""" + + async def _login(self, mock_prisma_client) -> tuple[LoginResult, dict]: + with patch.dict(os.environ, _DB_LOGIN_ENV): + with patch( # test-quality-ok: asserting the minted key's restriction requires seeing its kwargs + "litellm.proxy.auth.login_utils.generate_key_helper_fn", + new_callable=AsyncMock, + return_value={"token": "session-token"}, + ) as mock_generate_key: + result = await authenticate_user( + username="reset@example.com", + password="Str0ng!Passw0rd", + master_key="sk-1234", + prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), + general_settings=_POLICY_NO_BREACH_CHECK, + ) + return result, mock_generate_key.call_args.kwargs + + @pytest.mark.asyncio + async def test_flagged_user_gets_key_restricted_to_change_password(self): + row = _db_user_row(password="Str0ng!Passw0rd", password_reset_required=True) + result, key_kwargs = await self._login(_prisma_with_user(row)) + + assert key_kwargs["allowed_routes"] == ["/user/password/change"] + assert key_kwargs["metadata"] == {"login_method": "username_password", "password_reset_required": True} + assert result.password_reset_required is True + + @pytest.mark.asyncio + async def test_unflagged_user_gets_unrestricted_key(self): + row = _db_user_row(password="Str0ng!Passw0rd", password_reset_required=None) + result, key_kwargs = await self._login(_prisma_with_user(row)) + + assert key_kwargs["allowed_routes"] is None + assert key_kwargs["metadata"] == {"login_method": "username_password"} + assert result.password_reset_required is False + + async def _login_with_screen_result(self, mock_prisma_client, breached: bool) -> tuple[LoginResult, dict, dict]: + with patch.dict(os.environ, _DB_LOGIN_ENV): + with patch( # test-quality-ok: asserting the minted key's restriction requires seeing its kwargs + "litellm.proxy.auth.login_utils.generate_key_helper_fn", + new_callable=AsyncMock, + return_value={"token": "session-token"}, + ) as mock_generate_key: + with ( + patch( # test-quality-ok: authenticate_user has no HIBP client seam; the screen itself is tested against MockTransport below + "litellm.proxy.auth.login_utils.screen_login_password_for_breach", + new_callable=AsyncMock, + return_value=breached, + ) as mock_screen + ): + result = await authenticate_user( + username="reset@example.com", + password="Str0ng!Passw0rd", + master_key="sk-1234", + prisma_client=mock_prisma_client, + throttle=_unlimited_throttle(), + general_settings=_POLICY_NO_BREACH_CHECK, + ) + return result, mock_generate_key.call_args.kwargs, mock_screen.call_args.kwargs + + @pytest.mark.asyncio + async def test_login_screens_with_row_state_before_minting(self): + """The login must hand the screen the row's recheck timestamp, or the + 24h throttle can never work.""" + checked_at = datetime.now(timezone.utc) - timedelta(hours=1) + row = _db_user_row(password="Str0ng!Passw0rd", last_breach_check_at=checked_at) + mock_prisma_client = _prisma_with_user(row) + + _, _, screen_kwargs = await self._login_with_screen_result(mock_prisma_client, breached=False) + + assert screen_kwargs["user_id"] == "reset-user-1" + assert screen_kwargs["password"] == "Str0ng!Passw0rd" + assert screen_kwargs["last_breach_check_at"] == checked_at + assert screen_kwargs["prisma_client"] is mock_prisma_client + + @pytest.mark.asyncio + async def test_fresh_breach_hit_restricts_the_current_session(self): + """A breach found during THIS login must restrict THIS session, not + just the next one.""" + row = _db_user_row(password="Str0ng!Passw0rd", password_reset_required=None) + mock_prisma_client = _prisma_with_user(row) + + result, key_kwargs, _ = await self._login_with_screen_result(mock_prisma_client, breached=True) + + assert key_kwargs["allowed_routes"] == ["/user/password/change"] + assert key_kwargs["metadata"] == {"login_method": "username_password", "password_reset_required": True} + assert result.password_reset_required is True + + +def _sha1_upper(password: str) -> str: + return hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper() + + +def _client_with_transport(handler) -> AsyncHTTPHandler: + http_handler = AsyncHTTPHandler() + http_handler.client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + return http_handler + + +def _client_returning_breach_hit(password: str) -> AsyncHTTPHandler: + body = f"{_sha1_upper(password)[5:]}:42" + return _client_with_transport(lambda request: httpx.Response(200, text=body)) + + +def _client_returning_no_hit() -> AsyncHTTPHandler: + return _client_with_transport(lambda request: httpx.Response(200, text="0000000000000000000000000000000000A:3")) + + +def _client_never_called() -> AsyncHTTPHandler: + def handler(request: httpx.Request) -> httpx.Response: + raise AssertionError(f"unexpected HTTP call to {request.url}") + + return _client_with_transport(handler) + + +class TestScreenLoginPasswordForBreach: + """The awaited login-time screen: flags a breached password for a forced + reset, stamps the recheck timestamp, rechecks at most every 24h, returns + the breach verdict so the login can restrict the session it is minting, + and never raises into the login.""" + + @pytest.mark.asyncio + async def test_breached_password_sets_reset_flag_and_timestamp(self): + password = "Password123!" + mock_prisma_client = _prisma_with_user(None) + + breached = await screen_login_password_for_breach( + user_id="reset-user-1", + password=password, + last_breach_check_at=None, + general_settings={}, + prisma_client=mock_prisma_client, + client=_client_returning_breach_hit(password), + ) + + assert breached is True + update_kwargs = mock_prisma_client.db.litellm_usertable.update.call_args.kwargs + assert update_kwargs["where"] == {"user_id": "reset-user-1"} + assert update_kwargs["data"]["password_reset_required"] is True + assert isinstance(update_kwargs["data"]["last_breach_check_at"], datetime) + + @pytest.mark.asyncio + async def test_clean_password_stamps_timestamp_without_flag(self): + mock_prisma_client = _prisma_with_user(None) + + breached = await screen_login_password_for_breach( + user_id="reset-user-1", + password="Str0ng!Passw0rd", + last_breach_check_at=None, + general_settings={}, + prisma_client=mock_prisma_client, + client=_client_returning_no_hit(), + ) + + assert breached is False + update_kwargs = mock_prisma_client.db.litellm_usertable.update.call_args.kwargs + assert "password_reset_required" not in update_kwargs["data"] + assert isinstance(update_kwargs["data"]["last_breach_check_at"], datetime) + + @pytest.mark.asyncio + async def test_skips_hibp_when_checked_within_24_hours(self): + mock_prisma_client = _prisma_with_user(None) + + breached = await screen_login_password_for_breach( + user_id="reset-user-1", + password="Password123!", + last_breach_check_at=datetime.now(timezone.utc) - timedelta(hours=23), + general_settings={}, + prisma_client=mock_prisma_client, + client=_client_never_called(), + ) + + assert breached is False + mock_prisma_client.db.litellm_usertable.update.assert_not_called() + + @pytest.mark.asyncio + async def test_rechecks_when_last_check_is_older_than_24_hours(self): + password = "Password123!" + mock_prisma_client = _prisma_with_user(None) + + breached = await screen_login_password_for_breach( + user_id="reset-user-1", + password=password, + last_breach_check_at=datetime.now(timezone.utc) - timedelta(hours=25), + general_settings={}, + prisma_client=mock_prisma_client, + client=_client_returning_breach_hit(password), + ) + + assert breached is True + assert ( + mock_prisma_client.db.litellm_usertable.update.call_args.kwargs["data"]["password_reset_required"] is True + ) + + @pytest.mark.asyncio + async def test_skips_hibp_when_check_disabled(self): + mock_prisma_client = _prisma_with_user(None) + + breached = await screen_login_password_for_breach( + user_id="reset-user-1", + password="Password123!", + last_breach_check_at=None, + general_settings=_POLICY_NO_BREACH_CHECK, + prisma_client=mock_prisma_client, + client=_client_never_called(), + ) + + assert breached is False + mock_prisma_client.db.litellm_usertable.update.assert_not_called() + + @pytest.mark.asyncio + async def test_db_failure_never_raises_but_still_reports_the_breach(self): + """A failed flag write must not fail the login, but the breach verdict + still has to restrict the session being minted right now.""" + password = "Password123!" + mock_prisma_client = _prisma_with_user(None) + mock_prisma_client.db.litellm_usertable.update = AsyncMock(side_effect=RuntimeError("db down")) + + assert ( + await screen_login_password_for_breach( + user_id="reset-user-1", + password=password, + last_breach_check_at=None, + general_settings={}, + prisma_client=mock_prisma_client, + client=_client_returning_breach_hit(password), + ) + is True + ) diff --git a/tests/test_litellm/proxy/auth/test_onboarding.py b/tests/test_litellm/proxy/auth/test_onboarding.py index 524b655b465..0454aea1239 100644 --- a/tests/test_litellm/proxy/auth/test_onboarding.py +++ b/tests/test_litellm/proxy/auth/test_onboarding.py @@ -8,15 +8,20 @@ Covers the security behavior of: session key only after the password is written """ +import hashlib from datetime import timedelta from unittest.mock import AsyncMock, MagicMock, patch +import httpx import jwt import pytest +import respx from fastapi import HTTPException import litellm -from litellm.proxy._types import InvitationClaim +from litellm.proxy._types import InvitationClaim, ProxyException + +_POLICY_NO_BREACH_CHECK = {"password_policy_check_breached_passwords": False} # --------------------------------------------------------------------------- # Helpers @@ -386,7 +391,9 @@ async def test_claim_token_rejects_concurrent_reuse_before_password_write(): with ( patch("litellm.proxy.proxy_server.prisma_client", prisma), patch("litellm.proxy.proxy_server.master_key", "sk-test"), - patch("litellm.proxy.proxy_server.general_settings", {}), + patch( # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), patch( "litellm.proxy.proxy_server.generate_key_helper_fn", new_callable=AsyncMock, @@ -426,7 +433,9 @@ async def test_claim_token_sets_accepted_at_after_password_written(): with ( patch("litellm.proxy.proxy_server.prisma_client", prisma), patch("litellm.proxy.proxy_server.master_key", "sk-test"), - patch("litellm.proxy.proxy_server.general_settings", {}), + patch( # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), patch("litellm.proxy.proxy_server.premium_user", False), patch( "litellm.proxy.proxy_server.generate_key_helper_fn", @@ -454,6 +463,10 @@ async def test_claim_token_sets_accepted_at_after_password_written(): call_kwargs = prisma.db.litellm_usertable.update.call_args assert call_kwargs.kwargs["where"] == {"user_id": "user-123"} assert "password" in call_kwargs.kwargs["data"] + # A freshly claimed, policy-screened password lifts any pending forced + # reset and re-arms the login-time breach screen. + assert call_kwargs.kwargs["data"]["password_reset_required"] is False + assert call_kwargs.kwargs["data"]["last_breach_check_at"] is None # is_accepted was flipped to True on the invitation link prisma.db.litellm_invitationlink.update.assert_called_once() @@ -483,7 +496,9 @@ async def test_claim_token_rolls_back_invite_when_session_key_mint_fails(): with ( patch("litellm.proxy.proxy_server.prisma_client", prisma), patch("litellm.proxy.proxy_server.master_key", "sk-test"), - patch("litellm.proxy.proxy_server.general_settings", {}), + patch( # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), patch( "litellm.proxy.proxy_server.generate_key_helper_fn", new_callable=AsyncMock, @@ -505,3 +520,124 @@ async def test_claim_token_rolls_back_invite_when_session_key_mint_fails(): } assert rollback_kwargs["data"]["accepted_at"] is None assert rollback_kwargs["data"]["is_accepted"] is False + + +# --------------------------------------------------------------------------- +# POST /onboarding/claim_token - password policy +# --------------------------------------------------------------------------- + + +def _hibp_url_for(password: str) -> str: + sha1 = hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper() + return f"https://api.pwnedpasswords.com/range/{sha1[:5]}" + + +def _hibp_suffix_for(password: str) -> str: + return hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper()[5:] + + +@pytest.mark.asyncio +async def test_claim_token_rejects_short_password_before_consuming_invite(): + """Default policy requires 12 characters; the invite must stay claimable.""" + from litellm.proxy.proxy_server import claim_onboarding_link + + invite = _make_invite(is_accepted=False) + prisma = _make_prisma(invite, _make_user()) + request = _make_claim_request(_make_onboarding_token()) + data = InvitationClaim( + invitation_link="invite-abc", + user_id="user-123", + password="Sh0rt!pw", + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam + patch("litellm.proxy.proxy_server.master_key", "sk-test"), # test-quality-ok: same as above + patch("litellm.proxy.proxy_server.general_settings", {}), # test-quality-ok: same as above + ): + with pytest.raises(ProxyException) as exc_info: + await claim_onboarding_link(data=data, request=request) + + assert exc_info.value.code == "400" + assert "at least 12 characters" in exc_info.value.message + prisma.db.litellm_invitationlink.update_many.assert_not_called() + prisma.db.litellm_usertable.update.assert_not_called() + + +@pytest.mark.asyncio +@respx.mock +async def test_claim_token_rejects_breached_password_before_consuming_invite(): + """A password found in the HIBP corpus must be rejected and never stored.""" + from litellm.proxy.proxy_server import claim_onboarding_link + + password = "P@ssword123456" + respx.get(_hibp_url_for(password)).mock( + return_value=httpx.Response(200, text=f"{_hibp_suffix_for(password)}:1387") + ) + + invite = _make_invite(is_accepted=False) + prisma = _make_prisma(invite, _make_user()) + request = _make_claim_request(_make_onboarding_token()) + data = InvitationClaim( + invitation_link="invite-abc", + user_id="user-123", + password=password, + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam + patch("litellm.proxy.proxy_server.master_key", "sk-test"), # test-quality-ok: same as above + patch("litellm.proxy.proxy_server.general_settings", {}), # test-quality-ok: same as above + ): + with pytest.raises(ProxyException) as exc_info: + await claim_onboarding_link(data=data, request=request) + + assert exc_info.value.code == "400" + assert "data breaches" in exc_info.value.message + prisma.db.litellm_invitationlink.update_many.assert_not_called() + prisma.db.litellm_usertable.update.assert_not_called() + + +@pytest.mark.asyncio +@respx.mock +async def test_claim_token_fails_open_when_hibp_unreachable(): + """An HIBP outage must never block onboarding: the claim proceeds.""" + from litellm.proxy.proxy_server import claim_onboarding_link + + password = "NewP@ssw0rd-2026" + respx.get(_hibp_url_for(password)).mock(side_effect=httpx.ConnectError("no route to host")) + + invite = _make_invite(is_accepted=False) + user = _make_user() + prisma = _make_prisma(invite, user) + request = _make_claim_request(_make_onboarding_token()) + data = InvitationClaim( + invitation_link="invite-abc", + user_id="user-123", + password=password, + ) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma), # test-quality-ok: claim_onboarding_link reads proxy_server module globals; no injection seam + patch("litellm.proxy.proxy_server.master_key", "sk-test"), # test-quality-ok: same as above + patch("litellm.proxy.proxy_server.general_settings", {}), # test-quality-ok: same as above + patch("litellm.proxy.proxy_server.premium_user", False), # test-quality-ok: same as above + patch( # test-quality-ok: same as above + "litellm.proxy.proxy_server.generate_key_helper_fn", + new_callable=AsyncMock, + return_value={"token": "sk-generated-key", "user_id": "user-123"}, + ), + patch( # test-quality-ok: same as above + "litellm.proxy.proxy_server.get_custom_url", + return_value="http://localhost:4000/", + ), + patch( # test-quality-ok: same as above + "litellm.proxy.proxy_server.get_disabled_non_admin_personal_key_creation", + return_value=False, + ), + patch("litellm.proxy.proxy_server.get_server_root_path", return_value=""), # test-quality-ok: same as above + ): + result = await claim_onboarding_link(data=data, request=request) + + assert "token" in result + prisma.db.litellm_usertable.update.assert_called_once() diff --git a/tests/test_litellm/proxy/auth/test_password_policy.py b/tests/test_litellm/proxy/auth/test_password_policy.py index f6e7d443907..f9f5025b57f 100644 --- a/tests/test_litellm/proxy/auth/test_password_policy.py +++ b/tests/test_litellm/proxy/auth/test_password_policy.py @@ -2,22 +2,56 @@ Tests for the configurable password-strength policy in `litellm.proxy.auth.password_policy`, enforced on every path that persists a new or changed password for a locally-managed user. + +The breach-check (HIBP) tests inject a real AsyncHTTPHandler wrapping an +httpx.MockTransport, so no network is touched and nothing is monkeypatched. """ +import asyncio +import hashlib + +import httpx import pytest +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy._types import ProxyErrorTypes, ProxyException from litellm.proxy.auth.password_policy import ( DEFAULT_MIN_LENGTH, MIN_ALLOWED_LENGTH, PasswordPolicy, get_password_policy, + validate_password_not_breached, validate_password_policy, + validate_passwords_bulk, ) STRONG_PASSWORD = "Str0ng!Passw0rd" +def _sha1_upper(password: str) -> str: + return hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper() + + +def _client_with_transport(handler) -> AsyncHTTPHandler: + http_handler = AsyncHTTPHandler() + http_handler.client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + return http_handler + + +def _client_never_called() -> AsyncHTTPHandler: + def handler(request: httpx.Request) -> httpx.Response: + raise AssertionError(f"unexpected HTTP call to {request.url}") + + return _client_with_transport(handler) + + +def _client_returning(body: str, status_code: int = 200) -> AsyncHTTPHandler: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(status_code, text=body) + + return _client_with_transport(handler) + + def test_get_password_policy_defaults_to_pif_baseline(): policy = get_password_policy({}) assert policy == PasswordPolicy( @@ -134,3 +168,178 @@ def test_validate_password_policy_rejects_unicode_letter_as_special_character(): def test_validate_password_policy_accepts_real_special_character_with_unicode_letters(): """Same base password as the rejection test above, plus an actual symbol.""" assert validate_password_policy("Passwörd1234!", {}) is None + + +@pytest.mark.asyncio +async def test_breach_check_skipped_when_disabled(): + result = await validate_password_not_breached( + password="password12345", # breached in reality, but the check is off + general_settings={"password_policy_check_breached_passwords": False}, + client=_client_never_called(), + ) + assert result is None + + +@pytest.mark.asyncio +async def test_rejects_breached_password(): + password = "correct horse battery staple" + sha1 = _sha1_upper(password) + body = f"AAAA000000000000000000000000000000A:0\r\n{sha1[5:]}:42\r\nBBBB000000000000000000000000000000B:7" + + with pytest.raises(ProxyException) as exc_info: + await validate_password_not_breached(password=password, general_settings={}, client=_client_returning(body)) + assert exc_info.value.code == "400" + assert exc_info.value.type == ProxyErrorTypes.validation_error + assert exc_info.value.param == "password" + assert "data breaches" in exc_info.value.message + + +@pytest.mark.asyncio +async def test_only_sha1_prefix_leaves_the_proxy(): + password = "a very secret password" + sha1 = _sha1_upper(password) + captured_requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + captured_requests.append(request) + return httpx.Response(200, text="0000000000000000000000000000000000A:1") + + result = await validate_password_not_breached( + password=password, general_settings={}, client=_client_with_transport(handler) + ) + assert result is None + + (request,) = captured_requests + assert request.url.path == f"/range/{sha1[:5]}" + assert sha1[5:] not in str(request.url) + assert request.headers["Add-Padding"] == "true" + assert "litellm" in request.headers["User-Agent"] + + +@pytest.mark.asyncio +async def test_ignores_padding_entries_with_zero_count(): + """HIBP padding entries (requested via Add-Padding) carry count 0 and must + not be treated as breaches when they collide with the password's suffix.""" + password = "a padded-away password" + sha1 = _sha1_upper(password) + + result = await validate_password_not_breached( + password=password, general_settings={}, client=_client_returning(f"{sha1[5:]}:0") + ) + assert result is None + + +@pytest.mark.asyncio +async def test_accepts_password_absent_from_breach_corpus(): + result = await validate_password_not_breached( + password="a genuinely novel password", + general_settings={}, + client=_client_returning("0018A45C4D1DEF81644B54AB7F969B88D65:1\r\n00D4F6E8FA6EECAD2A3AA415EEC418D38EC:2"), + ) + assert result is None + + +@pytest.mark.asyncio +async def test_breach_check_fails_open_on_network_error(): + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("no route to host") + + result = await validate_password_not_breached( + password="password12345", # breached, but HIBP is unreachable + general_settings={}, + client=_client_with_transport(handler), + ) + assert result is None + + +@pytest.mark.asyncio +async def test_breach_check_fails_open_on_http_error_status(): + result = await validate_password_not_breached( + password="password12345", + general_settings={}, + client=_client_returning("service unavailable", status_code=503), + ) + assert result is None + + +@pytest.mark.asyncio +async def test_breach_check_fails_open_on_malformed_response_body(): + result = await validate_password_not_breached( + password="password12345", + general_settings={}, + client=_client_returning(f"{_sha1_upper('password12345')[5:]}:not-a-number"), + ) + assert result is None + + +@pytest.mark.asyncio +async def test_validate_passwords_bulk_screens_concurrently(): + """All HIBP lookups for a batch must be in flight at once: each handler + call stalls until every expected request has arrived, and a handler that + gives up waiting reports the password as breached. Serial awaiting (the + old per-user behavior) leaves each earlier request waiting forever for the + later ones, so every verdict comes back as a breach and the test fails.""" + passwords = ("Uniqu3!Passw0rd-a", "Uniqu3!Passw0rd-b", "Uniqu3!Passw0rd-c") + suffix_by_prefix = {_sha1_upper(p)[:5]: _sha1_upper(p)[5:] for p in passwords} + all_arrived = asyncio.Event() + arrivals: list[str] = [] + + async def handler(request: httpx.Request) -> httpx.Response: + arrivals.append(request.url.path) + if len(arrivals) == len(passwords): + all_arrived.set() + try: + await asyncio.wait_for(all_arrived.wait(), timeout=5) + except TimeoutError: + return httpx.Response(200, text=f"{suffix_by_prefix[request.url.path.rsplit('/', 1)[-1]]}:1") + return httpx.Response(200, text="0000000000000000000000000000000000A:1") + + verdicts = await validate_passwords_bulk(passwords, {}, client=_client_with_transport(handler)) + assert set(arrivals) == {f"/range/{prefix}" for prefix in suffix_by_prefix} + assert all(verdicts[p] is None for p in passwords) + + +@pytest.mark.asyncio +async def test_validate_passwords_bulk_deduplicates_lookups(): + """500 users sharing one password must cost exactly one HIBP lookup.""" + password = "Sh@red-Passw0rd!" + request_count = 0 + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal request_count + request_count += 1 + return httpx.Response(200, text="0000000000000000000000000000000000A:1") + + verdicts = await validate_passwords_bulk((password,) * 500, {}, client=_client_with_transport(handler)) + assert request_count == 1 + assert verdicts == {password: None} + + +@pytest.mark.asyncio +async def test_validate_passwords_bulk_mixed_verdicts(): + """Weak passwords are rejected without an HIBP lookup; breached ones get + the breach error; acceptable ones map to None.""" + breached = "Br3ached!Passw0rd" + clean = "Cl3an!!Passw0rd42" + weak = "short1!" + breached_sha1 = _sha1_upper(breached) + looked_up_prefixes: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + looked_up_prefixes.append(request.url.path.rsplit("/", 1)[-1]) + if request.url.path == f"/range/{breached_sha1[:5]}": + return httpx.Response(200, text=f"{breached_sha1[5:]}:99") + return httpx.Response(200, text="0000000000000000000000000000000000A:1") + + verdicts = await validate_passwords_bulk((breached, clean, weak), {}, client=_client_with_transport(handler)) + assert _sha1_upper(weak)[:5] not in looked_up_prefixes + assert verdicts[clean] is None + assert "data breaches" in verdicts[breached].message + assert verdicts[breached].code == "400" + assert "12 characters" in verdicts[weak].message + + +@pytest.mark.asyncio +async def test_validate_passwords_bulk_empty_batch_makes_no_lookups(): + verdicts = await validate_passwords_bulk((), {}, client=_client_never_called()) + assert verdicts == {} diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 87bf4595af5..c9b8d6764c9 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -3,7 +3,6 @@ from datetime import datetime from typing import Final from unittest.mock import MagicMock, patch - import pytest from fastapi import HTTPException, Request @@ -39,7 +38,7 @@ def test_non_admin_config_update_route_rejected(): request.query_params = {} # Test that calling /config/update route raises HTTPException with 403 status - with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: + with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, @@ -50,9 +49,8 @@ def test_non_admin_config_update_route_rejected(): ) # Verify the exception is raised with the correct message - assert ( - "Only proxy admin can be used to generate, delete, update info for new keys/users/teams" - in str(exc_info.value) + assert "Only proxy admin can be used to generate, delete, update info for new keys/users/teams" in str( + exc_info.value ) assert "Route=/config/update" in str(exc_info.value) assert "Your role=internal_user" in str(exc_info.value) @@ -158,7 +156,7 @@ def test_user_banner_update_rejected_for_non_admin(): request = MagicMock(spec=Request) request.query_params = {} - with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: + with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, @@ -733,9 +731,7 @@ def test_virtual_key_llm_api_route_includes_passthrough_prefix(route): valid_token = UserAPIKeyAuth(user_id="test_user", allowed_routes=["llm_api_routes"]) - result = RouteChecks.is_virtual_key_allowed_to_call_route( - route=route, valid_token=valid_token - ) + result = RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=valid_token) assert result is True @@ -760,9 +756,7 @@ def test_virtual_key_llm_api_routes_allows_google_routes(route): valid_token = UserAPIKeyAuth(user_id="test_user", allowed_routes=["llm_api_routes"]) - result = RouteChecks.is_virtual_key_allowed_to_call_route( - route=route, valid_token=valid_token - ) + result = RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=valid_token) assert result is True @@ -832,18 +826,14 @@ def test_google_routes_with_dynamic_model_names_accessible_to_internal_users(): ) # If no exception is raised, the test passes except Exception as e: - pytest.fail( - f"Internal user should be able to access Google generateContent route. Got error: {str(e)}" - ) + pytest.fail(f"Internal user should be able to access Google generateContent route. Got error: {e!s}") def test_virtual_key_allowed_routes_with_multiple_litellm_routes_member_names(): """Test that virtual key works with multiple LiteLLMRoutes member names in allowed_routes""" # Create a UserAPIKeyAuth with multiple LiteLLMRoutes member names - valid_token = UserAPIKeyAuth( - user_id="test_user", allowed_routes=["openai_routes", "info_routes"] - ) + valid_token = UserAPIKeyAuth(user_id="test_user", allowed_routes=["openai_routes", "info_routes"]) # Test that routes from both groups are allowed result1 = RouteChecks.is_virtual_key_allowed_to_call_route( @@ -897,13 +887,9 @@ def test_virtual_key_allowed_routes_with_no_member_names_only_explicit(): ) # Test that explicit routes are allowed - result1 = RouteChecks.is_virtual_key_allowed_to_call_route( - route="/chat/completions", valid_token=valid_token - ) + result1 = RouteChecks.is_virtual_key_allowed_to_call_route(route="/chat/completions", valid_token=valid_token) - result2 = RouteChecks.is_virtual_key_allowed_to_call_route( - route="/custom/route", valid_token=valid_token - ) + result2 = RouteChecks.is_virtual_key_allowed_to_call_route(route="/custom/route", valid_token=valid_token) assert result1 is True assert result2 is True @@ -1301,9 +1287,7 @@ def test_virtual_key_without_llm_api_routes_cannot_access_pass_through(): ) assert exc_info.value.status_code == 403 - assert "Virtual key is not allowed to call this route" in str( - exc_info.value.detail - ) + assert "Virtual key is not allowed to call this route" in str(exc_info.value.detail) def test_check_passthrough_route_access_key_metadata_exact_match(): @@ -1762,9 +1746,7 @@ def test_videos_route_accessible_to_internal_users(): ) # If no exception is raised, the test passes except Exception as e: - pytest.fail( - f"Internal user should be able to access /v1/videos route. Got error: {str(e)}" - ) + pytest.fail(f"Internal user should be able to access /v1/videos route. Got error: {e!s}") def test_videos_route_with_virtual_key_llm_api_routes(): @@ -1786,12 +1768,8 @@ def test_videos_route_with_virtual_key_llm_api_routes(): ] for route in test_routes: - result = RouteChecks.is_virtual_key_allowed_to_call_route( - route=route, valid_token=valid_token - ) - assert ( - result is True - ), f"Virtual key with llm_api_routes should be able to access {route}" + result = RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=valid_token) + assert result is True, f"Virtual key with llm_api_routes should be able to access {route}" def test_non_proxy_admin_wildcard_allowed_routes(): @@ -1862,9 +1840,7 @@ def test_proxy_admin_viewer_can_access_global_spend_tags(): ) # If no exception is raised, the test passes except Exception as e: - pytest.fail( - f"proxy_admin_viewer should be able to access /global/spend/tags route. Got error: {str(e)}" - ) + pytest.fail(f"proxy_admin_viewer should be able to access /global/spend/tags route. Got error: {e!s}") # Routes returning proxy-wide spend across every team / customer / api_key. @@ -1892,7 +1868,7 @@ def test_internal_user_blocked_from_global_spend_routes(route): request = MagicMock(spec=Request) request.query_params = {} - with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: + with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, @@ -1921,7 +1897,7 @@ def test_internal_user_view_only_blocked_from_global_spend_routes(route): request = MagicMock(spec=Request) request.query_params = {} - with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: + with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value, @@ -2023,9 +1999,7 @@ def test_proxy_admin_viewer_can_access_audit_logs(route): request_data={}, ) except Exception as e: - pytest.fail( - f"proxy_admin_viewer should be able to access {route} route. Got error: {str(e)}" - ) + pytest.fail(f"proxy_admin_viewer should be able to access {route} route. Got error: {e!s}") # ── Admin Viewer parity: Logs page endpoints ────────────────────────────────── @@ -2088,9 +2062,7 @@ def test_proxy_admin_viewer_can_access_logs_page_endpoints(route): request_data={}, ) except Exception as e: - pytest.fail( - f"proxy_admin_viewer should be able to access {route}. Got error: {str(e)}" - ) + pytest.fail(f"proxy_admin_viewer should be able to access {route}. Got error: {e!s}") @pytest.mark.parametrize( @@ -2200,7 +2172,7 @@ def test_internal_user_blocked_from_admin_viewer_logs_routes(route): if route not in INTERNAL_USER_BLOCKED_SUBSET: return - with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: + with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, @@ -2276,9 +2248,7 @@ def test_proxy_admin_viewer_can_access_settings_read_endpoints(route): request_data={}, ) except Exception as e: - pytest.fail( - f"proxy_admin_viewer should be able to access {route}. Got error: {str(e)}" - ) + pytest.fail(f"proxy_admin_viewer should be able to access {route}. Got error: {e!s}") # ── Admin Viewer parity: default-allow GET semantics ───────────────────────── @@ -2477,9 +2447,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints: ) local_file = os.path.abspath(local_file) - spec = importlib.util.spec_from_file_location( - "local_enterprise_route_checks", local_file - ) + spec = importlib.util.spec_from_file_location("local_enterprise_route_checks", local_file) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) return mod.EnterpriseRouteChecks @@ -2490,9 +2458,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints: EnterpriseRouteChecks = self._get_enterprise_route_checks() with ( - patch.object( - EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True - ), + patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True), patch.object( EnterpriseRouteChecks, "is_management_routes_disabled", @@ -2508,9 +2474,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints: EnterpriseRouteChecks = self._get_enterprise_route_checks() with ( - patch.object( - EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True - ), + patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True), patch.object( EnterpriseRouteChecks, "is_management_routes_disabled", @@ -2526,9 +2490,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints: EnterpriseRouteChecks = self._get_enterprise_route_checks() with ( - patch.object( - EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True - ), + patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True), patch.object( EnterpriseRouteChecks, "is_management_routes_disabled", @@ -2539,9 +2501,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints: EnterpriseRouteChecks.should_call_route("/v1/chat/completions") assert exc_info.value.status_code == 403 - assert "LLM API routes are disabled for this instance." in str( - exc_info.value.detail - ) + assert "LLM API routes are disabled for this instance." in str(exc_info.value.detail) @patch("litellm.proxy.proxy_server.premium_user", True) def test_should_embeddings_still_blocked_when_llm_api_disabled(self): @@ -2549,9 +2509,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints: EnterpriseRouteChecks = self._get_enterprise_route_checks() with ( - patch.object( - EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True - ), + patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=True), patch.object( EnterpriseRouteChecks, "is_management_routes_disabled", @@ -2569,9 +2527,7 @@ class TestModelsRouteExemptFromDisableLLMEndpoints: EnterpriseRouteChecks = self._get_enterprise_route_checks() with ( - patch.object( - EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=False - ), + patch.object(EnterpriseRouteChecks, "is_llm_api_route_disabled", return_value=False), patch.object( EnterpriseRouteChecks, "is_management_routes_disabled", @@ -2590,9 +2546,7 @@ def test_route_in_additional_public_routes_wildcard_match(): from litellm.proxy.auth.auth_utils import route_in_additonal_public_routes with ( - patch( - "litellm.proxy.proxy_server.general_settings", {"public_routes": ["/api/*"]} - ), + patch("litellm.proxy.proxy_server.general_settings", {"public_routes": ["/api/*"]}), patch("litellm.proxy.proxy_server.premium_user", True), ): # Wildcard should match subpaths @@ -2684,7 +2638,7 @@ def test_non_admin_non_team_admin_cannot_access_config_update_but_can_attempt_re ) # /config/update is still blocked - with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: + with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, @@ -2772,8 +2726,6 @@ def test_available_roles_accessible_to_non_admin_users(user_role): # ── _user_is_org_admin tests ────────────────────────────────────────────────── - - def _make_org_admin_user(org_id: str) -> LiteLLM_UserTable: membership = LiteLLM_OrganizationMembershipTable( user_id="org-admin-user", @@ -2896,9 +2848,7 @@ async def test_add_team_org_context_noop_when_org_id_already_present(): raise AssertionError("must not resolve when organization_id is present") body = {"team_id": "team-1", "organization_id": "org-explicit"} - out = await add_team_org_context_to_request_body( - route="/team/update", request_body=body, fetch_team_org_id=fetch - ) + out = await add_team_org_context_to_request_body(route="/team/update", request_body=body, fetch_team_org_id=fetch) assert out == body @@ -2910,9 +2860,7 @@ async def test_add_team_org_context_noop_for_other_routes(): raise AssertionError("must not resolve for a non-opted-in route") body = {"team_id": "team-1"} - out = await add_team_org_context_to_request_body( - route="/team/delete", request_body=body, fetch_team_org_id=fetch - ) + out = await add_team_org_context_to_request_body(route="/team/delete", request_body=body, fetch_team_org_id=fetch) assert out == body @@ -2925,9 +2873,7 @@ async def test_add_team_org_context_noop_when_team_has_no_org(): return None body = {"team_id": "team-1"} - out = await add_team_org_context_to_request_body( - route="/team/update", request_body=body, fetch_team_org_id=fetch - ) + out = await add_team_org_context_to_request_body(route="/team/update", request_body=body, fetch_team_org_id=fetch) assert out == body @@ -3198,9 +3144,7 @@ async def test_initialize_pass_through_registers_wildcard_for_auth_subpath(): # Removing the endpoint should clean up openai_routes # remove_endpoint_routes takes endpoint_id (UUID portion of # the route key "{id}:exact:{path}:{methods}") - registered = ( - InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes() - ) + registered = InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes() endpoint_ids = {k.split(":")[0] for k in registered} for eid in endpoint_ids: InitPassThroughEndpointHelpers.remove_endpoint_routes(eid) @@ -3210,9 +3154,7 @@ async def test_initialize_pass_through_registers_wildcard_for_auth_subpath(): LiteLLMRoutes.openai_routes.value[:] = original_routes # Clean up any routes registered during this test to avoid # polluting the module-level _registered_pass_through_routes - registered = ( - InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes() - ) + registered = InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes() for k in registered: InitPassThroughEndpointHelpers.remove_endpoint_routes(k.split(":")[0]) @@ -3243,8 +3185,7 @@ def test_provider_name_substring_not_classified_as_llm_route(route): from litellm.proxy.auth.route_checks import RouteChecks assert RouteChecks.is_llm_api_route(route=route) is False, ( - f"{route!r} should NOT be classified as an LLM API route — " - "provider-name substring match bypass" + f"{route!r} should NOT be classified as an LLM API route — provider-name substring match bypass" ) @@ -3266,9 +3207,7 @@ def test_legitimate_passthrough_routes_still_classified_as_llm_route(route): """Legitimate passthrough routes must still pass is_llm_api_route.""" from litellm.proxy.auth.route_checks import RouteChecks - assert ( - RouteChecks.is_llm_api_route(route=route) is True - ), f"{route!r} should be classified as an LLM API route" + assert RouteChecks.is_llm_api_route(route=route) is True, f"{route!r} should be classified as an LLM API route" @pytest.mark.parametrize( @@ -3326,7 +3265,7 @@ def test_internal_user_blocked_from_search_tool_writes(route): request = MagicMock(spec=Request) request.query_params = {} - with pytest.raises(Exception, match='Only proxy admin can be used to generate, delete, update') as exc_info: + with pytest.raises(Exception, match="Only proxy admin can be used to generate, delete, update") as exc_info: RouteChecks.non_proxy_admin_allowed_routes_check( user_obj=user_obj, _user_role=LitellmUserRoles.INTERNAL_USER.value, @@ -3702,12 +3641,7 @@ def test_agent_inference_routes_stay_llm_api(route): def test_agent_routes_union_still_covers_both_halves(route): """Keys configured with allowed_routes=["agent_routes"] must keep both halves.""" - assert ( - RouteChecks.check_route_access( - route=route, allowed_routes=LiteLLMRoutes.agent_routes.value - ) - is True - ) + assert RouteChecks.check_route_access(route=route, allowed_routes=LiteLLMRoutes.agent_routes.value) is True @pytest.mark.parametrize("route", AGENT_MANAGEMENT_ROUTES) @@ -3761,6 +3695,136 @@ def test_agent_registry_route_gate_open_to_non_admin_roles(user_role, method, ro valid_token=valid_token, request_data={}, ) + + +def test_proxy_admin_viewer_user_update_password_param_rejected(): + """The self-service /user/update password carve-out is closed: non-admins + change their own password through /user/password/change, which verifies + the current password. Admin password sets don't pass through this check.""" + with pytest.raises(HTTPException) as exc_info: + RouteChecks._check_proxy_admin_viewer_access( + route="/user/update", + _user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value, + request_data={"password": "hunter2hunter2"}, + ) + assert exc_info.value.status_code == 403 + assert "password" in str(exc_info.value.detail) + + +def test_proxy_admin_viewer_user_update_user_email_still_allowed(): + request = MagicMock(spec=Request) + request.method = "POST" + + allowed = RouteChecks._check_proxy_admin_viewer_access( + route="/user/update", + _user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value, + request_data={"user_email": "viewer@example.com"}, + request=request, + ) + + assert allowed is None + + +def test_proxy_admin_viewer_can_change_own_password(): + request = MagicMock(spec=Request) + request.method = "POST" + + allowed = RouteChecks._check_proxy_admin_viewer_access( + route="/user/password/change", + _user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value, + request_data={"current_password": "a", "new_password": "b"}, + request=request, + ) + + assert allowed is None + + +@pytest.mark.parametrize( + "user_role", + [ + LitellmUserRoles.INTERNAL_USER.value, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value, + ], +) +def test_non_admin_roles_can_change_own_password(user_role): + valid_token = UserAPIKeyAuth(user_id="test_user", user_role=user_role) + request = MagicMock(spec=Request) + request.method = "POST" + request.query_params = {} + + allowed = RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=LiteLLM_UserTable(user_id="test_user", user_role=user_role), + _user_role=user_role, + route="/user/password/change", + request=request, + valid_token=valid_token, + request_data={"current_password": "a", "new_password": "b"}, + ) + + assert allowed is None + + +def _password_reset_session_token() -> UserAPIKeyAuth: + """The UI session key `authenticate_user` mints for a user flagged + `password_reset_required`.""" + return UserAPIKeyAuth( + user_id="flagged_user", + allowed_routes=["/user/password/change"], + metadata={"password_reset_required": True}, + ) + + +def test_password_reset_session_can_reach_change_password(): + result = RouteChecks.is_virtual_key_allowed_to_call_route( + route="/user/password/change", + valid_token=_password_reset_session_token(), + ) + + assert result is True + + +@pytest.mark.parametrize( + "route", + [ + "/user/info", + "/key/generate", + "/user/update", + "/chat/completions", + ], +) +def test_password_reset_session_is_blocked_everywhere_else_with_reset_message(route): + """Server-side enforcement of the forced reset: a script that logs in via + /v2/login and drives the management API with the session key must get a 403 + naming the remediation endpoint, on every route but the change-password one.""" + with pytest.raises(HTTPException) as exc_info: + RouteChecks.is_virtual_key_allowed_to_call_route( + route=route, + valid_token=_password_reset_session_token(), + ) + + assert exc_info.value.status_code == 403 + assert "password must be changed" in str(exc_info.value.detail) + assert "/user/password/change" in str(exc_info.value.detail) + + +def test_restricted_key_without_reset_marker_keeps_generic_message(): + """The reset-specific 403 must not leak onto ordinary allowed_routes keys.""" + valid_token = UserAPIKeyAuth( + user_id="test_user", + allowed_routes=["/chat/completions"], + ) + + with pytest.raises(HTTPException) as exc_info: + RouteChecks.is_virtual_key_allowed_to_call_route( + route="/user/info", + valid_token=valid_token, + ) + + assert exc_info.value.status_code == 403 + assert "password must be changed" not in str(exc_info.value.detail) + assert "not allowed to call this route" in str(exc_info.value.detail) + + TEAM_CALLBACK_ROUTES = ( "/team/06bda574-5ca9-43d3-beb8-3b23c2f17112/callback", "/team/06bda574-5ca9-43d3-beb8-3b23c2f17112/callback/langfuse", diff --git a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py index ede0ecb2790..75545a574e3 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py @@ -1,13 +1,18 @@ +import hashlib import json from datetime import datetime, timezone from types import SimpleNamespace from typing import Final +import httpx import pytest +import respx from fastapi import HTTPException from fastapi.testclient import TestClient from pytest_mock import MockerFixture +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.proxy._types import ( LiteLLM_UserTableFiltered, LitellmUserRoles, @@ -67,9 +72,7 @@ async def test_ui_view_users_with_null_email(mocker, caplog): # Proxy admin: no org filter, no get_user_object call response = await ui_view_users( - user_api_key_dict=UserAPIKeyAuth( - user_id="test_user", user_role=LitellmUserRoles.PROXY_ADMIN - ), + user_api_key_dict=UserAPIKeyAuth(user_id="test_user", user_role=LitellmUserRoles.PROXY_ADMIN), user_id="test_user", user_email=None, team_id=None, @@ -77,9 +80,7 @@ async def test_ui_view_users_with_null_email(mocker, caplog): page_size=50, ) - assert response == [ - LiteLLM_UserTableFiltered(user_id="test-user-null-email", user_email=None) - ] + assert response == [LiteLLM_UserTableFiltered(user_id="test-user-null-email", user_email=None)] @pytest.mark.asyncio @@ -103,9 +104,7 @@ async def test_ui_view_users_proxy_admin_no_org_filter(mocker): mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) await ui_view_users( - user_api_key_dict=UserAPIKeyAuth( - user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN - ), + user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), user_id=None, user_email="foo", team_id=None, @@ -128,9 +127,7 @@ async def test_ui_view_users_org_admin_filtered_by_org(mocker): async def mock_find_many(*args, **kwargs): where = kwargs.get("where") or {} assert "organization_memberships" in where - assert where["organization_memberships"] == { - "some": {"organization_id": {"in": [org_id]}} - } + assert where["organization_memberships"] == {"some": {"organization_id": {"in": [org_id]}}} return [] mock_prisma_client.db.litellm_usertable.find_many = mock_find_many @@ -268,9 +265,7 @@ async def test_ui_view_users_flag_on_team_admin_org_team(mocker): async def mock_find_many(*args, **kwargs): where = kwargs.get("where") or {} assert "organization_memberships" in where - assert where["organization_memberships"] == { - "some": {"organization_id": {"in": [org_id]}} - } + assert where["organization_memberships"] == {"some": {"organization_id": {"in": [org_id]}}} return [] mock_prisma_client.db.litellm_usertable.find_many = mock_find_many @@ -401,9 +396,7 @@ async def test_ui_view_users_flag_on_team_admin_org_member_no_team_id(mocker): async def mock_find_many(*args, **kwargs): where = kwargs.get("where") or {} assert "organization_memberships" in where - assert where["organization_memberships"] == { - "some": {"organization_id": {"in": [org_id]}} - } + assert where["organization_memberships"] == {"some": {"organization_id": {"in": [org_id]}}} return [] mock_prisma_client.db.litellm_usertable.find_many = mock_find_many @@ -462,9 +455,7 @@ async def test_ui_view_users_flag_on_team_admin_not_in_org_resolves_via_key_team async def mock_find_many(*args, **kwargs): where = kwargs.get("where") or {} assert "organization_memberships" in where - assert where["organization_memberships"] == { - "some": {"organization_id": {"in": [org_id]}} - } + assert where["organization_memberships"] == {"some": {"organization_id": {"in": [org_id]}}} return [] mock_prisma_client.db.litellm_usertable.find_many = mock_find_many @@ -507,9 +498,7 @@ async def test_ui_view_users_flag_on_team_admin_not_in_org_resolves_via_key_team # No team_id query param, but team_id on the API key response = await ui_view_users( - user_api_key_dict=UserAPIKeyAuth( - user_id="team-admin-no-org", user_role=None, team_id=tid - ), + user_api_key_dict=UserAPIKeyAuth(user_id="team-admin-no-org", user_role=None, team_id=tid), user_id=None, user_email="u", team_id=None, @@ -538,13 +527,9 @@ def test_user_daily_activity_types(): # Assert all fields in SpendMetrics are reported in DailySpendMetadata as "total_" for field in spend_metrics.__dict__: if field.startswith("total_"): - assert hasattr( - daily_spend_metadata, field - ), f"Field {field} is not reported in DailySpendMetadata" + assert hasattr(daily_spend_metadata, field), f"Field {field} is not reported in DailySpendMetadata" else: - assert not hasattr( - daily_spend_metadata, field - ), f"Field {field} is reported in DailySpendMetadata" + assert not hasattr(daily_spend_metadata, field), f"Field {field} is reported in DailySpendMetadata" @pytest.mark.asyncio @@ -591,9 +576,7 @@ async def test_get_users_includes_timestamps(mocker): # Call get_users function directly with proxy admin auth admin_key = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) - response = await get_users( - page=1, page_size=1, user_api_key_dict=admin_key, organization_ids=None - ) + response = await get_users(page=1, page_size=1, user_api_key_dict=admin_key, organization_ids=None) print("user /list response: ", response) @@ -654,14 +637,10 @@ async def test_get_users_redacts_scim_enterprise_metadata(mocker): ) admin_key = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) - response = await get_users( - page=1, page_size=1, user_api_key_dict=admin_key, organization_ids=None - ) + response = await get_users(page=1, page_size=1, user_api_key_dict=admin_key, organization_ids=None) listed = response["users"][0] - assert listed.metadata == { - "scim_metadata": {"givenName": "Jane", "familyName": "Doe"} - } + assert listed.metadata == {"scim_metadata": {"givenName": "Jane", "familyName": "Doe"}} assert "scim_enterprise" not in (listed.metadata or {}) @@ -853,9 +832,7 @@ async def test_new_user_license_over_limit(mocker): mocker.patch("litellm.proxy.proxy_server._license_check", mock_license_check) # Create test request data - user_request = NewUserRequest( - user_email="test@example.com", user_role="internal_user" - ) + user_request = NewUserRequest(user_email="test@example.com", user_role="internal_user") # Mock user_api_key_dict mock_user_api_key_dict = UserAPIKeyAuth(user_id="test_admin") @@ -916,9 +893,7 @@ async def test_new_user_license_gate_counts_only_billable_users(mocker): request = NewUserRequest(user_role="internal_user") # 2 active + 3 deactivated -> billable 2, not over max_users 2: gate passes - mocker.patch( - "litellm.proxy.proxy_server.prisma_client", _prisma(total=5, deactivated=3) - ) + mocker.patch("litellm.proxy.proxy_server.prisma_client", _prisma(total=5, deactivated=3)) with pytest.raises(ProxyException) as passed: await new_user(data=request, user_api_key_dict=admin) assert key_gen.call_count == 1 @@ -926,9 +901,7 @@ async def test_new_user_license_gate_counts_only_billable_users(mocker): # 3 active, 0 deactivated -> billable 3, over max_users 2: gate blocks key_gen.reset_mock() - mocker.patch( - "litellm.proxy.proxy_server.prisma_client", _prisma(total=3, deactivated=0) - ) + mocker.patch("litellm.proxy.proxy_server.prisma_client", _prisma(total=3, deactivated=0)) with pytest.raises(ProxyException) as blocked: await new_user(data=request, user_api_key_dict=admin) assert blocked.value.code == 403 or blocked.value.code == "403" @@ -978,14 +951,10 @@ async def test_new_user_non_admin_cannot_create_admin(mocker): mocker.patch("litellm.proxy.proxy_server._license_check", mock_license_check) # Test Case 1: INTERNAL_USER trying to create PROXY_ADMIN - user_request = NewUserRequest( - user_email="admin@example.com", user_role=LitellmUserRoles.PROXY_ADMIN - ) + user_request = NewUserRequest(user_email="admin@example.com", user_role=LitellmUserRoles.PROXY_ADMIN) # Mock user_api_key_dict with non-admin role - mock_user_api_key_dict = UserAPIKeyAuth( - user_id="test_internal_user", user_role=LitellmUserRoles.INTERNAL_USER - ) + mock_user_api_key_dict = UserAPIKeyAuth(user_id="test_internal_user", user_role=LitellmUserRoles.INTERNAL_USER) # Call new_user function and expect ProxyException with pytest.raises(ProxyException) as exc_info: @@ -993,9 +962,7 @@ async def test_new_user_non_admin_cannot_create_admin(mocker): # Verify the exception details assert exc_info.value.code == 403 or exc_info.value.code == "403" - assert "Only proxy admins can create administrative users" in str( - exc_info.value.message - ) + assert "Only proxy admins can create administrative users" in str(exc_info.value.message) assert "proxy_admin" in str(exc_info.value.message) assert "proxy_admin_viewer" in str(exc_info.value.message) assert str(LitellmUserRoles.PROXY_ADMIN) in str(exc_info.value.message) @@ -1008,15 +975,11 @@ async def test_new_user_non_admin_cannot_create_admin(mocker): ) with pytest.raises(ProxyException) as exc_info2: - await new_user( - data=user_request_viewer, user_api_key_dict=mock_user_api_key_dict - ) + await new_user(data=user_request_viewer, user_api_key_dict=mock_user_api_key_dict) # Verify the exception details assert exc_info2.value.code == 403 or exc_info2.value.code == "403" - assert "Only proxy admins can create administrative users" in str( - exc_info2.value.message - ) + assert "Only proxy admins can create administrative users" in str(exc_info2.value.message) assert str(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY) in str(exc_info2.value.message) @@ -1055,9 +1018,7 @@ async def test_new_user_non_admin_permissions_non_empty_rejected(mocker): user_role=LitellmUserRoles.INTERNAL_USER, permissions={"get_spend_routes": True}, ) - caller = UserAPIKeyAuth( - user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN - ) + caller = UserAPIKeyAuth(user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN) with pytest.raises(ProxyException) as exc_info: await new_user(data=data, user_api_key_dict=caller) @@ -1101,9 +1062,7 @@ async def test_new_user_non_admin_permissions_explicit_empty_rejected(mocker): permissions={}, ) assert "permissions" in data.model_fields_set - caller = UserAPIKeyAuth( - user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN - ) + caller = UserAPIKeyAuth(user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN) with pytest.raises(ProxyException) as exc_info: await new_user(data=data, user_api_key_dict=caller) @@ -1156,9 +1115,7 @@ async def test_new_user_non_admin_omits_permissions_succeeds(mocker): user_role=LitellmUserRoles.INTERNAL_USER, ) assert "permissions" not in data.model_fields_set - caller = UserAPIKeyAuth( - user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN - ) + caller = UserAPIKeyAuth(user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN) result = await new_user(data=data, user_api_key_dict=caller) assert result is not None @@ -1232,14 +1189,10 @@ async def test_update_single_user_non_admin_permissions_rejected(mocker): user_id="alice", permissions={"get_spend_routes": True}, ) - caller = UserAPIKeyAuth( - user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN - ) + caller = UserAPIKeyAuth(user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN) with pytest.raises(HTTPException) as exc_info: - await _update_single_user_helper( - user_request=data, user_api_key_dict=caller - ) + await _update_single_user_helper(user_request=data, user_api_key_dict=caller) assert exc_info.value.status_code == 403 assert "permissions" in str(exc_info.value.detail) @@ -1261,14 +1214,10 @@ async def test_update_single_user_non_admin_permissions_explicit_empty_rejected( data = UpdateUserRequest(user_id="alice", permissions={}) assert "permissions" in data.model_fields_set - caller = UserAPIKeyAuth( - user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN - ) + caller = UserAPIKeyAuth(user_id="org-admin", user_role=LitellmUserRoles.ORG_ADMIN) with pytest.raises(HTTPException) as exc_info: - await _update_single_user_helper( - user_request=data, user_api_key_dict=caller - ) + await _update_single_user_helper(user_request=data, user_api_key_dict=caller) assert exc_info.value.status_code == 403 assert "permissions" in str(exc_info.value.detail) @@ -1324,15 +1273,11 @@ async def test_user_info_url_encoding_plus_character(mocker): mock_request.url.query = "user_id=machine-user+alp-air-admin-b58-b@tempus.com" # Mock user_api_key_dict - mock_user_api_key_dict = UserAPIKeyAuth( - user_id="test_admin", user_role="proxy_admin" - ) + mock_user_api_key_dict = UserAPIKeyAuth(user_id="test_admin", user_role="proxy_admin") # Call user_info function with the URL-decoded user_id (as FastAPI would pass it) # FastAPI would normally convert + to space, but our fix should handle this - decoded_user_id = ( - "machine-user alp-air-admin-b58-b@tempus.com" # What FastAPI gives us - ) + decoded_user_id = "machine-user alp-air-admin-b58-b@tempus.com" # What FastAPI gives us expected_user_id = "machine-user+alp-air-admin-b58-b@tempus.com" response = await user_info( @@ -1383,9 +1328,7 @@ async def test_user_info_nonexistent_user(mocker): mock_request = mocker.MagicMock(spec=Request) # Mock user_api_key_dict - mock_user_api_key_dict = UserAPIKeyAuth( - user_id="test_admin", user_role="proxy_admin" - ) + mock_user_api_key_dict = UserAPIKeyAuth(user_id="test_admin", user_role="proxy_admin") # Call user_info function with a non-existent user_id nonexistent_user_id = "nonexistent-user@example.com" @@ -1423,14 +1366,10 @@ async def test_user_info_no_user_id_view_only_admin_gets_proxy_admin_payload(moc mock_get_user_info_for_proxy_admin, ) - viewer = UserAPIKeyAuth( - user_id="viewer", user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value - ) + viewer = UserAPIKeyAuth(user_id="viewer", user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value) mock_request = mocker.MagicMock(spec=Request) - response = await user_info( - user_id=None, user_api_key_dict=viewer, request=mock_request - ) + response = await user_info(user_id=None, user_api_key_dict=viewer, request=mock_request) mock_get_user_info_for_proxy_admin.assert_awaited_once_with(user_api_key_dict=viewer) assert response is admin_payload @@ -1457,9 +1396,7 @@ async def test_new_user_default_teams_flow(mocker): mock_prisma_client.db.litellm_usertable.count = mock_count persisted_user_row = mocker.MagicMock() persisted_user_row.teams = ["96fed65b-0182-4ff4-8429-2721cd7d42af"] - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - return_value=persisted_user_row - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(return_value=persisted_user_row) # Mock duplicate checks to pass async def mock_check_duplicate_user_email(*args, **kwargs): @@ -1527,26 +1464,20 @@ async def test_new_user_default_teams_flow(mocker): ) # Create test request data WITHOUT teams (teams should come from defaults) - user_request = NewUserRequest( - user_email="test@example.com", user_role="internal_user" - ) + user_request = NewUserRequest(user_email="test@example.com", user_role="internal_user") # Mock user_api_key_dict mock_user_api_key_dict = UserAPIKeyAuth(user_id="test_admin") # Call new_user function - response = await new_user( - data=user_request, user_api_key_dict=mock_user_api_key_dict - ) + response = await new_user(data=user_request, user_api_key_dict=mock_user_api_key_dict) # Verify generate_key_helper_fn was called WITHOUT teams mock_generate_key_helper_fn.assert_called_once() call_kwargs = mock_generate_key_helper_fn.call_args.kwargs # Teams should be removed from the data passed to generate_key_helper_fn - assert ( - "teams" not in call_kwargs - ), "Teams should not be passed to generate_key_helper_fn" + assert "teams" not in call_kwargs, "Teams should not be passed to generate_key_helper_fn" assert call_kwargs["request_type"] == "user" assert call_kwargs["user_email"] == "test@example.com" assert call_kwargs["user_role"] == "internal_user" @@ -1591,24 +1522,16 @@ def test_update_internal_new_user_params_proxy_admin_role(): try: # Create test data with PROXY_ADMIN role - data = NewUserRequest( - user_email="admin@example.com", user_role=LitellmUserRoles.PROXY_ADMIN.value - ) + data = NewUserRequest(user_email="admin@example.com", user_role=LitellmUserRoles.PROXY_ADMIN.value) data_json = data.model_dump(exclude_unset=True) # Call the function result = _update_internal_new_user_params(data_json=data_json, data=data) # Assertions - default params should NOT be applied for PROXY_ADMIN - assert ( - "max_budget" not in result - ), "Default max_budget should NOT be applied to PROXY_ADMIN" - assert ( - "models" not in result - ), "Default models should NOT be applied to PROXY_ADMIN" - assert ( - "tpm_limit" not in result - ), "Default tpm_limit should NOT be applied to PROXY_ADMIN" + assert "max_budget" not in result, "Default max_budget should NOT be applied to PROXY_ADMIN" + assert "models" not in result, "Default models should NOT be applied to PROXY_ADMIN" + assert "tpm_limit" not in result, "Default tpm_limit should NOT be applied to PROXY_ADMIN" # These should still work assert result["user_email"] == "admin@example.com" @@ -1722,15 +1645,9 @@ async def test_check_duplicate_user_email_case_insensitive(mocker): user_email_clause = where_clause.get("user_email", {}) # Check that the query structure is correct for case insensitive search - assert ( - "equals" in user_email_clause - ), "Query should use 'equals' for case insensitive search" - assert ( - user_email_clause.get("mode") == "insensitive" - ), "Query should use 'insensitive' mode" - assert ( - user_email_clause.get("equals") == "user@example.com" - ), "Query should search for the provided email" + assert "equals" in user_email_clause, "Query should use 'equals' for case insensitive search" + assert user_email_clause.get("mode") == "insensitive", "Query should use 'insensitive' mode" + assert user_email_clause.get("equals") == "user@example.com", "Query should search for the provided email" return mock_existing_user # Return existing user to simulate duplicate @@ -1741,9 +1658,7 @@ async def test_check_duplicate_user_email_case_insensitive(mocker): await _check_duplicate_user_email("user@example.com", mock_prisma_client) assert exc_info.value.status_code == 409 - assert "User with email User@Example.com already exists" in str( - exc_info.value.detail - ) + assert "User with email User@Example.com already exists" in str(exc_info.value.detail) # Test Case 2: No duplicate found async def mock_find_first_no_duplicate(*args, **kwargs): @@ -1768,9 +1683,7 @@ async def test_check_duplicate_user_email_case_insensitive(mocker): pytest.fail(f"Should not raise exception when no duplicate found, but got: {e}") # Test Case 3: None email should not cause issues - await _check_duplicate_user_email( - None, mock_prisma_client - ) # Should not raise exception + await _check_duplicate_user_email(None, mock_prisma_client) # Should not raise exception @pytest.mark.asyncio @@ -1880,9 +1793,7 @@ def test_process_keys_for_user_info_filters_dashboard_keys(monkeypatch): # Verify dashboard key is not in results result_team_ids = [key.get("team_id") for key in result] - assert ( - UI_SESSION_TOKEN_TEAM_ID not in result_team_ids - ), "Dashboard key should be filtered out" + assert UI_SESSION_TOKEN_TEAM_ID not in result_team_ids, "Dashboard key should be filtered out" # Verify regular keys are included assert "regular-team" in result_team_ids, "Regular team key should be included" @@ -1892,9 +1803,7 @@ def test_process_keys_for_user_info_filters_dashboard_keys(monkeypatch): result_tokens = [key.get("token") for key in result] assert "sk-regular-token" in result_tokens, "Regular key should be included" assert "sk-no-team-token" in result_tokens, "No-team key should be included" - assert ( - "sk-dashboard-token" not in result_tokens - ), "Dashboard key should not be included" + assert "sk-dashboard-token" not in result_tokens, "Dashboard key should not be included" def test_process_keys_for_user_info_handles_none_keys(monkeypatch): @@ -2419,9 +2328,7 @@ async def test_get_user_daily_activity_non_admin_cannot_view_other_users(monkeyp ) assert exc_info.value.status_code == 403 - assert "Non-admin users can only view their own spend data" in str( - exc_info.value.detail - ) + assert "Non-admin users can only view their own spend data" in str(exc_info.value.detail) # Case 2: Non-admin omits user_id — should default to their own user_id mock_response = MagicMock() @@ -2708,14 +2615,10 @@ async def test_delete_user_cleans_up_created_by_invitation_links(mocker): async def mock_find_unique(*args, **kwargs): return mock_user_row - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) # Mock find_many for teams (no teams) - mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock( - return_value=[] - ) + mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock(return_value=[]) # Mock all delete_many calls mock_prisma_client.db.litellm_verificationtoken.find_many = mocker.AsyncMock( @@ -2741,9 +2644,7 @@ async def test_delete_user_cleans_up_created_by_invitation_links(mocker): # Call delete_user data = DeleteUserRequest(user_ids=["admin-creator"]) - user_api_key_dict = UserAPIKeyAuth( - user_id="proxy-admin", user_role=LitellmUserRoles.PROXY_ADMIN - ) + user_api_key_dict = UserAPIKeyAuth(user_id="proxy-admin", user_role=LitellmUserRoles.PROXY_ADMIN) await delete_user(data=data, user_api_key_dict=user_api_key_dict) @@ -2752,9 +2653,7 @@ async def test_delete_user_cleans_up_created_by_invitation_links(mocker): call_kwargs = mock_prisma_client.db.litellm_invitationlink.delete_many.call_args where_clause = call_kwargs.kwargs.get("where") or call_kwargs[1].get("where") - assert ( - "OR" in where_clause - ), "Should use OR to match user_id, created_by, and updated_by" + assert "OR" in where_clause, "Should use OR to match user_id, created_by, and updated_by" or_conditions = where_clause["OR"] assert len(or_conditions) == 3, "Should have 3 OR conditions" @@ -2875,9 +2774,7 @@ async def test_delete_user_rejects_org_admin_deleting_outside_scope(mocker): async def mock_find_unique(*args, **kwargs): return mock_target_user - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) # Caller (org_admin_user) administers org-A. caller_membership = mocker.MagicMock() @@ -2903,16 +2800,12 @@ async def test_delete_user_rejects_org_admin_deleting_outside_scope(mocker): return [caller_membership] return [] - mock_prisma_client.db.litellm_organizationmembership.find_many = mocker.AsyncMock( - side_effect=mock_find_memberships - ) + mock_prisma_client.db.litellm_organizationmembership.find_many = mocker.AsyncMock(side_effect=mock_find_memberships) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) data = DeleteUserRequest(user_ids=["victim"]) - user_api_key_dict = UserAPIKeyAuth( - user_id="org_admin_user", user_role=LitellmUserRoles.ORG_ADMIN - ) + user_api_key_dict = UserAPIKeyAuth(user_id="org_admin_user", user_role=LitellmUserRoles.ORG_ADMIN) with pytest.raises(HTTPException) as exc: await delete_user(data=data, user_api_key_dict=user_api_key_dict) @@ -2920,11 +2813,8 @@ async def test_delete_user_rejects_org_admin_deleting_outside_scope(mocker): # Critical: no delete_many calls should have executed. assert ( - not hasattr( - mock_prisma_client.db.litellm_verificationtoken.delete_many, "mock_calls" - ) - or len(mock_prisma_client.db.litellm_verificationtoken.delete_many.mock_calls) - == 0 + not hasattr(mock_prisma_client.db.litellm_verificationtoken.delete_many, "mock_calls") + or len(mock_prisma_client.db.litellm_verificationtoken.delete_many.mock_calls) == 0 ) @@ -2943,9 +2833,7 @@ async def test_user_update_rejects_silent_create_for_non_proxy_admin(mocker): mock_prisma_client = mocker.MagicMock() # user_email lookup yields None → would silently create pre-fix. - mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock( - return_value=None - ) + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=None) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) user_request = UpdateUserRequest( @@ -2959,9 +2847,7 @@ async def test_user_update_rejects_silent_create_for_non_proxy_admin(mocker): ) with pytest.raises(HTTPException) as exc: - await _update_single_user_helper( - user_request=user_request, user_api_key_dict=org_admin - ) + await _update_single_user_helper(user_request=user_request, user_api_key_dict=org_admin) assert exc.value.status_code == 404 @@ -3005,17 +2891,13 @@ async def test_user_info_v2_proxy_admin_can_query_any_user(mocker): return mock_user_row return None - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_request = mocker.MagicMock(spec=Request) - admin_key = UserAPIKeyAuth( - user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN - ) + admin_key = UserAPIKeyAuth(user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN) response = await user_info_v2( request=mock_request, @@ -3069,17 +2951,13 @@ async def test_user_info_v2_redacts_scim_enterprise_metadata(mocker): return mock_user_row return None - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_request = mocker.MagicMock(spec=Request) - admin_key = UserAPIKeyAuth( - user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN - ) + admin_key = UserAPIKeyAuth(user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN) response = await user_info_v2( request=mock_request, @@ -3088,9 +2966,7 @@ async def test_user_info_v2_redacts_scim_enterprise_metadata(mocker): ) assert isinstance(response, UserInfoV2Response) - assert response.metadata == { - "scim_metadata": {"givenName": "Jane", "familyName": "Doe"} - } + assert response.metadata == {"scim_metadata": {"givenName": "Jane", "familyName": "Doe"}} assert "scim_enterprise" not in (response.metadata or {}) @@ -3159,17 +3035,13 @@ async def test_user_info_v2_internal_user_can_query_self(mocker): return mock_user_row return None - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_request = mocker.MagicMock(spec=Request) - user_key = UserAPIKeyAuth( - user_id="self-user", user_role=LitellmUserRoles.INTERNAL_USER - ) + user_key = UserAPIKeyAuth(user_id="self-user", user_role=LitellmUserRoles.INTERNAL_USER) response = await user_info_v2( request=mock_request, @@ -3204,17 +3076,13 @@ async def test_user_info_v2_internal_user_cannot_query_other(mocker): return mock_caller_row return None - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_request = mocker.MagicMock(spec=Request) - user_key = UserAPIKeyAuth( - user_id="caller-user", user_role=LitellmUserRoles.INTERNAL_USER - ) + user_key = UserAPIKeyAuth(user_id="caller-user", user_role=LitellmUserRoles.INTERNAL_USER) with pytest.raises(ProxyException) as exc_info: await user_info_v2( @@ -3261,17 +3129,13 @@ async def test_user_info_v2_no_user_id_defaults_to_self(mocker): return mock_user_row return None - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_request = mocker.MagicMock(spec=Request) - user_key = UserAPIKeyAuth( - user_id="my-user-id", user_role=LitellmUserRoles.INTERNAL_USER - ) + user_key = UserAPIKeyAuth(user_id="my-user-id", user_role=LitellmUserRoles.INTERNAL_USER) # Call without user_id response = await user_info_v2( @@ -3299,17 +3163,13 @@ async def test_user_info_v2_nonexistent_user_returns_404(mocker): async def mock_find_unique(*args, **kwargs): return None - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_request = mocker.MagicMock(spec=Request) - admin_key = UserAPIKeyAuth( - user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN - ) + admin_key = UserAPIKeyAuth(user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN) with pytest.raises(ProxyException) as exc_info: await user_info_v2( @@ -3357,17 +3217,13 @@ async def test_user_info_v2_response_shape(mocker): async def mock_find_unique(*args, **kwargs): return mock_user_row - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_request = mocker.MagicMock(spec=Request) - admin_key = UserAPIKeyAuth( - user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN - ) + admin_key = UserAPIKeyAuth(user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN) response = await user_info_v2( request=mock_request, @@ -3402,9 +3258,7 @@ async def test_user_info_v2_response_shape(mocker): # The dashboard's user edit form hydrates its per-model budget rows from # these two, so dropping them makes a save replace the user's budgets. - assert response_dict["model_max_budget"] == { - "gpt-3.5-turbo": {"budget_limit": 5.0, "time_period": "30d"} - } + assert response_dict["model_max_budget"] == {"gpt-3.5-turbo": {"budget_limit": 5.0, "time_period": "30d"}} assert response_dict["model_max_budget_usage"] == { "gpt-3.5-turbo": {"current_spend": 0.0, "budget_limit": 5.0, "time_period": "30d"} } @@ -3463,9 +3317,7 @@ async def test_user_info_v2_team_admin_can_query_team_member(mocker): return mock_target return None - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) # Mock team with caller as admin mock_team = mocker.MagicMock() @@ -3482,17 +3334,13 @@ async def test_user_info_v2_team_admin_can_query_team_member(mocker): async def mock_find_many_teams(*args, **kwargs): return [mock_team] - mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock( - side_effect=mock_find_many_teams - ) + mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock(side_effect=mock_find_many_teams) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_request = mocker.MagicMock(spec=Request) - team_admin_key = UserAPIKeyAuth( - user_id="team-admin-user", user_role=LitellmUserRoles.INTERNAL_USER - ) + team_admin_key = UserAPIKeyAuth(user_id="team-admin-user", user_role=LitellmUserRoles.INTERNAL_USER) response = await user_info_v2( request=mock_request, @@ -3532,9 +3380,7 @@ async def test_user_info_v2_team_admin_cannot_query_non_team_member(mocker): return mock_target return None - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) # Mock team where caller is admin mock_team = mocker.MagicMock() @@ -3550,17 +3396,13 @@ async def test_user_info_v2_team_admin_cannot_query_non_team_member(mocker): async def mock_find_many_teams(*args, **kwargs): return [mock_team] - mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock( - side_effect=mock_find_many_teams - ) + mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock(side_effect=mock_find_many_teams) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_request = mocker.MagicMock(spec=Request) - team_admin_key = UserAPIKeyAuth( - user_id="team-admin-user", user_role=LitellmUserRoles.INTERNAL_USER - ) + team_admin_key = UserAPIKeyAuth(user_id="team-admin-user", user_role=LitellmUserRoles.INTERNAL_USER) with pytest.raises(ProxyException) as exc_info: await user_info_v2( @@ -3610,18 +3452,14 @@ async def test_user_info_v2_url_encoding_plus_character(mocker): return mock_user_row return None - mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock( - side_effect=mock_find_unique - ) + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(side_effect=mock_find_unique) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mock_request = mocker.MagicMock(spec=Request) mock_request.url.query = f"user_id={expected_user_id}" - admin_key = UserAPIKeyAuth( - user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN - ) + admin_key = UserAPIKeyAuth(user_id="admin-user", user_role=LitellmUserRoles.PROXY_ADMIN) # Simulate FastAPI converting + to space decoded_user_id = "machine-user admin@example.com" @@ -3718,9 +3556,7 @@ def test_enforce_user_info_access_admin_bypass(): _enforce_user_info_access, ) - admin = UserAPIKeyAuth( - user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN.value - ) + admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN.value) # Should not raise even when querying a different user _enforce_user_info_access(user_id="someone_else", user_api_key_dict=admin) @@ -3759,9 +3595,7 @@ def test_enforce_user_info_access_owner_allowed(): _enforce_user_info_access, ) - user = UserAPIKeyAuth( - user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER.value - ) + user = UserAPIKeyAuth(user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER.value) _enforce_user_info_access(user_id="alice", user_api_key_dict=user) @@ -3773,9 +3607,7 @@ def test_enforce_user_info_access_no_user_id_allowed(): _enforce_user_info_access, ) - user = UserAPIKeyAuth( - user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER.value - ) + user = UserAPIKeyAuth(user_id="alice", user_role=LitellmUserRoles.INTERNAL_USER.value) _enforce_user_info_access(user_id=None, user_api_key_dict=user) @@ -3832,9 +3664,7 @@ async def test_ghsa_wvg4_non_admin_cannot_self_escalate_max_budget(mocker, budge "max_budget": 100, } existing_user.user_id = "user-1" - mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock( - return_value=existing_user - ) + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) user_request = UpdateUserRequest.model_validate({"user_id": "user-1", budget_field: budget_value}) @@ -3844,9 +3674,7 @@ async def test_ghsa_wvg4_non_admin_cannot_self_escalate_max_budget(mocker, budge ) with pytest.raises(HTTPException) as exc: - await _update_single_user_helper( - user_request=user_request, user_api_key_dict=caller - ) + await _update_single_user_helper(user_request=user_request, user_api_key_dict=caller) assert exc.value.status_code == 403 assert budget_field in str(exc.value.detail) mock_prisma_client.update_data.assert_not_called() @@ -3868,9 +3696,7 @@ async def test_ghsa_wvg4_non_admin_cannot_self_escalate_spend(mocker): "spend": 50.0, } existing_user.user_id = "user-1" - mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock( - return_value=existing_user - ) + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) user_request = UpdateUserRequest( @@ -3883,9 +3709,7 @@ async def test_ghsa_wvg4_non_admin_cannot_self_escalate_spend(mocker): ) with pytest.raises(HTTPException) as exc: - await _update_single_user_helper( - user_request=user_request, user_api_key_dict=caller - ) + await _update_single_user_helper(user_request=user_request, user_api_key_dict=caller) assert exc.value.status_code == 403 assert "spend" in str(exc.value.detail) @@ -3904,12 +3728,8 @@ async def test_ghsa_wvg4_proxy_admin_can_update_user_budget(mocker): "max_budget": 100, } existing_user.user_id = "target-user" - mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock( - return_value=existing_user - ) - mock_prisma_client.update_data = mocker.AsyncMock( - return_value={"user_id": "target-user", "max_budget": 500} - ) + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user) + mock_prisma_client.update_data = mocker.AsyncMock(return_value={"user_id": "target-user", "max_budget": 500}) mock_prisma_client.jsonify_object = mocker.MagicMock(side_effect=lambda x: x) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mocker.patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin") @@ -3923,9 +3743,7 @@ async def test_ghsa_wvg4_proxy_admin_can_update_user_budget(mocker): user_role=LitellmUserRoles.PROXY_ADMIN, ) - result = await _update_single_user_helper( - user_request=user_request, user_api_key_dict=admin_caller - ) + result = await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller) assert result is not None @@ -3941,12 +3759,8 @@ async def test_admin_user_update_spend_invalidates_counter(mocker): existing_user = mocker.MagicMock() existing_user.model_dump.return_value = {"user_id": "target-user", "spend": 50.0} existing_user.user_id = "target-user" - mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock( - return_value=existing_user - ) - mock_prisma_client.update_data = mocker.AsyncMock( - return_value={"user_id": "target-user", "spend": -25.0} - ) + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user) + mock_prisma_client.update_data = mocker.AsyncMock(return_value={"user_id": "target-user", "spend": -25.0}) mock_prisma_client.jsonify_object = mocker.MagicMock(side_effect=lambda x: x) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mocker.patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin") @@ -3961,13 +3775,9 @@ async def test_admin_user_update_spend_invalidates_counter(mocker): # without raising the recurring budget ceiling. Future changes should # continue allowing negative spend counters. user_request = UpdateUserRequest(user_id="target-user", spend=-25) - admin_caller = UserAPIKeyAuth( - user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN - ) + admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) - await _update_single_user_helper( - user_request=user_request, user_api_key_dict=admin_caller - ) + await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller) mock_invalidate.assert_awaited_once_with(counter_key="spend:user:target-user") @@ -3984,9 +3794,7 @@ async def test_user_update_rejects_non_finite_spend(mocker): existing_user = mocker.MagicMock() existing_user.model_dump.return_value = {"user_id": "target-user", "spend": 50.0} existing_user.user_id = "target-user" - mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock( - return_value=existing_user - ) + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user) mock_prisma_client.update_data = mocker.AsyncMock() mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mocker.patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin") @@ -3996,14 +3804,10 @@ async def test_user_update_rejects_non_finite_spend(mocker): ) user_request = UpdateUserRequest(user_id="target-user", spend=float("nan")) - admin_caller = UserAPIKeyAuth( - user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN - ) + admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) with pytest.raises(HTTPException) as exc: - await _update_single_user_helper( - user_request=user_request, user_api_key_dict=admin_caller - ) + await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller) assert exc.value.status_code == 400 mock_prisma_client.update_data.assert_not_called() mock_invalidate.assert_not_awaited() @@ -4023,9 +3827,7 @@ async def test_resolve_user_email_metadata_maps_page_user_ids_to_email(mocker): mock_prisma_client = mocker.MagicMock() find_many = mocker.AsyncMock( return_value=[ - SimpleNamespace( - user_id="u1", user_email="alice@example.com", user_alias="Alice" - ), + SimpleNamespace(user_id="u1", user_email="alice@example.com", user_alias="Alice"), SimpleNamespace(user_id="u2", user_email=None, user_alias="bob-alias"), ] ) @@ -4224,19 +4026,13 @@ def _object_permission_mocks(mocker, existing_object_permission_id=None): } existing_user.user_id = "target-user" existing_user.object_permission_id = existing_object_permission_id - mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock( - return_value=existing_user - ) - mock_prisma_client.db.litellm_objectpermissiontable.find_unique = mocker.AsyncMock( - return_value=None - ) + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user) + mock_prisma_client.db.litellm_objectpermissiontable.find_unique = mocker.AsyncMock(return_value=None) mock_prisma_client.db.litellm_objectpermissiontable.upsert = mocker.AsyncMock( return_value=SimpleNamespace(object_permission_id="perm-new") ) mock_prisma_client.db.litellm_mcpservertable.find_many = mocker.AsyncMock(return_value=[]) - mock_prisma_client.update_data = mocker.AsyncMock( - return_value={"user_id": "target-user"} - ) + mock_prisma_client.update_data = mocker.AsyncMock(return_value={"user_id": "target-user"}) mock_prisma_client.jsonify_object = mocker.MagicMock(side_effect=lambda x: x) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mocker.patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin") @@ -4272,9 +4068,7 @@ async def test_user_update_persists_mcp_entitlement_and_links_it(mocker): "mcp_tool_permissions": {"github": ["list_issues"]}, }, ), - user_api_key_dict=UserAPIKeyAuth( - user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN - ), + user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN), ) upsert_kwargs = mock_prisma_client.db.litellm_objectpermissiontable.upsert.call_args.kwargs @@ -4308,9 +4102,7 @@ async def test_user_update_invalidates_the_cached_entitlement(mocker): user_id="target-user", object_permission={"mcp_tool_permissions": {"github": []}}, ), - user_api_key_dict=UserAPIKeyAuth( - user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN - ), + user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN), ) deleted = {call.kwargs["key"] for call in cache.async_delete_cache.call_args_list} @@ -4343,9 +4135,7 @@ async def test_admin_can_clear_a_users_mcp_entitlement(mocker): await _update_single_user_helper( user_request=UpdateUserRequest(user_id="target-user", object_permission={}), - user_api_key_dict=UserAPIKeyAuth( - user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN - ), + user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN), ) written = mock_prisma_client.update_data.call_args.kwargs["data"] @@ -4382,9 +4172,7 @@ async def test_user_update_invalidates_both_the_old_and_new_permission_rows(mock user_id="target-user", object_permission={"mcp_tool_permissions": {"github": ["list_issues"]}}, ), - user_api_key_dict=UserAPIKeyAuth( - user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN - ), + user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN), ) deleted = {call.kwargs["key"] for call in cache.async_delete_cache.call_args_list} @@ -4415,9 +4203,7 @@ async def test_non_admin_cannot_clear_their_own_mcp_entitlement(mocker): with pytest.raises(HTTPException) as exc: await _update_single_user_helper( user_request=UpdateUserRequest(user_id="target-user", object_permission={}), - user_api_key_dict=UserAPIKeyAuth( - user_id="target-user", user_role=LitellmUserRoles.INTERNAL_USER - ), + user_api_key_dict=UserAPIKeyAuth(user_id="target-user", user_role=LitellmUserRoles.INTERNAL_USER), ) assert exc.value.status_code == 403 @@ -4445,9 +4231,7 @@ async def test_non_admin_cannot_rewrite_their_own_mcp_entitlement(mocker): user_id="target-user", object_permission={"mcp_servers": [], "mcp_tool_permissions": {}}, ), - user_api_key_dict=UserAPIKeyAuth( - user_id="target-user", user_role=LitellmUserRoles.INTERNAL_USER - ), + user_api_key_dict=UserAPIKeyAuth(user_id="target-user", user_role=LitellmUserRoles.INTERNAL_USER), ) assert exc.value.status_code == 403 @@ -4464,9 +4248,7 @@ async def test_new_user_persists_the_requested_mcp_entitlement(mocker): return_value=SimpleNamespace(object_permission_id="perm-created") ) mock_prisma_client.db.litellm_mcpservertable.find_many = mocker.AsyncMock(return_value=[]) - mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock( - return_value=None - ) + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=None) mock_prisma_client.db.litellm_usertable.count = mocker.AsyncMock(return_value=0) mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) mocker.patch( @@ -4475,9 +4257,7 @@ async def test_new_user_persists_the_requested_mcp_entitlement(mocker): ) mock_generate = mocker.patch( "litellm.proxy.management_endpoints.internal_user_endpoints.generate_key_helper_fn", - new=mocker.AsyncMock( - return_value={"user_id": "new-human", "token": "sk-x", "expires": None} - ), + new=mocker.AsyncMock(return_value={"user_id": "new-human", "token": "sk-x", "expires": None}), ) mocker.patch( "litellm.proxy.hooks.user_management_event_hooks.UserManagementEventHooks.async_user_created_hook", @@ -4489,9 +4269,7 @@ async def test_new_user_persists_the_requested_mcp_entitlement(mocker): user_id="new-human", object_permission={"mcp_tool_permissions": {"github": ["list_issues"]}}, ), - user_api_key_dict=UserAPIKeyAuth( - user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN - ), + user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN), ) created = mock_prisma_client.db.litellm_objectpermissiontable.create.call_args.kwargs["data"] @@ -4533,16 +4311,12 @@ async def test_user_info_v2_returns_the_mcp_entitlement(mocker): response = await user_info_v2( request=SimpleNamespace(query_params={}), user_id="human-1", - user_api_key_dict=UserAPIKeyAuth( - user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN - ), + user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN), ) assert response.object_permission is not None assert response.object_permission.mcp_servers == ["github"] - assert response.object_permission.mcp_tool_permissions == { - "github": ["list_issues"] - } + assert response.object_permission.mcp_tool_permissions == {"github": ["list_issues"]} @pytest.mark.asyncio @@ -4558,9 +4332,7 @@ async def test_user_info_v2_returns_the_mcp_entitlement(mocker): ], ids=["supplied", "omitted", "empty"], ) -async def test_user_new_persists_model_max_budget( - monkeypatch, model_max_budget, expected_written -): +async def test_user_new_persists_model_max_budget(monkeypatch, model_max_budget, expected_written): """ /user/new used to echo model_max_budget back while writing {} to the user row, so a per-model budget looked configured and was read by nothing. @@ -4674,6 +4446,11 @@ async def test_user_update_hashes_and_persists_strong_password(_admin_prisma, mo _update_single_user_helper, ) + mocker.patch( # test-quality-ok: same module-global mocking every test in this file already uses + "litellm.proxy.proxy_server.general_settings", + {"password_policy_check_breached_passwords": False}, + ) + mock_prisma_client = _admin_prisma existing_user = mocker.MagicMock() existing_user.model_dump.return_value = {"user_id": "target-user"} @@ -4691,6 +4468,152 @@ async def test_user_update_hashes_and_persists_strong_password(_admin_prisma, mo written_data = mock_prisma_client.update_data.call_args.kwargs["data"] assert written_data.get("password") is not None assert written_data["password"] != strong_password + # An admin-set password is known to the admin, so the user must be forced + # to change it at next login and the breach screen re-armed. + assert written_data["password_reset_required"] is True + assert written_data["last_breach_check_at"] is None + + +@pytest.mark.asyncio +@respx.mock +async def test_user_update_rejects_breached_password(_admin_prisma): + """A strength-passing password found in the HIBP corpus must be rejected + before it ever reaches the DB write.""" + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + _update_single_user_helper, + ) + + password = "Str0ng!Passw0rd" + sha1 = hashlib.sha1(password.encode("utf-8"), usedforsecurity=False).hexdigest().upper() + respx.get(f"https://api.pwnedpasswords.com/range/{sha1[:5]}").mock( + return_value=httpx.Response(200, text=f"{sha1[5:]}:1387") + ) + + user_request = UpdateUserRequest(user_id="target-user", password=password) + admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + + with pytest.raises(ProxyException) as exc_info: + await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller) + + assert exc_info.value.code == "400" + assert "data breaches" in exc_info.value.message + _admin_prisma.db.litellm_usertable.find_first.assert_not_called() + + +@pytest.mark.asyncio +async def test_bulk_update_all_users_rejects_a_password(_admin_prisma): + """The all_users fast path writes user_updates straight to update_many, + bypassing _update_single_user_helper. A password riding along would be + stored as unvalidated plaintext on every row, so it must be rejected + before any DB access.""" + from fastapi import HTTPException + + from litellm.proxy._types import UpdateUserRequestNoUserIDorEmail + from litellm.proxy.management_endpoints.internal_user_endpoints import bulk_user_update + from litellm.types.proxy.management_endpoints.internal_user_endpoints import ( + BulkUpdateUserRequest, + ) + + data = BulkUpdateUserRequest( + all_users=True, + user_updates=UpdateUserRequestNoUserIDorEmail(password="Str0ng!Passw0rd"), + ) + admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + + with pytest.raises(HTTPException) as exc_info: + await bulk_user_update(data=data, user_api_key_dict=admin_caller) + + assert exc_info.value.status_code == 400 + assert "not supported" in str(exc_info.value.detail) + _admin_prisma.db.litellm_usertable.find_many.assert_not_called() + _admin_prisma.db.litellm_usertable.update_many.assert_not_called() + + +def _hibp_client_with_handler(handler) -> AsyncHTTPHandler: + """A real AsyncHTTPHandler over httpx.MockTransport (the DI seam used + throughout test_password_policy.py), so no network is touched.""" + http_handler = AsyncHTTPHandler() + http_handler.client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + return http_handler + + +@pytest.mark.asyncio +async def test_bulk_update_breached_password_fails_only_that_user(_admin_prisma, mocker): + """In a bulk batch, a breached password fails only its own entry, before + any DB write for it; sibling entries with acceptable passwords persist.""" + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + bulk_update_processed_users, + ) + + breached = "Br3ached!Passw0rd" + clean = "NewP@ssw0rd123" + breached_sha1 = hashlib.sha1(breached.encode("utf-8"), usedforsecurity=False).hexdigest().upper() + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == f"/range/{breached_sha1[:5]}": + return httpx.Response(200, text=f"{breached_sha1[5:]}:1387") + return httpx.Response(200, text="0000000000000000000000000000000000A:1") + + mock_prisma_client = _admin_prisma + existing_user = mocker.MagicMock() + existing_user.model_dump.return_value = {"user_id": "user-clean"} + existing_user.user_id = "user-clean" + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user) + mock_prisma_client.update_data = mocker.AsyncMock(return_value={"user_id": "user-clean"}) + mock_prisma_client.jsonify_object = mocker.MagicMock(side_effect=lambda x: x) + + admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + response = await bulk_update_processed_users( + users_to_update=[ + UpdateUserRequest(user_id="user-breached", password=breached), + UpdateUserRequest(user_id="user-clean", password=clean), + ], + user_api_key_dict=admin_caller, + hibp_client=_hibp_client_with_handler(handler), + ) + + assert response.successful_updates == 1 + assert response.failed_updates == 1 + by_user = {r.user_id: r for r in response.results} + assert by_user["user-breached"].success is False + assert "data breaches" in by_user["user-breached"].error + assert by_user["user-clean"].success is True + (write_call,) = mock_prisma_client.update_data.call_args_list + assert write_call.kwargs["user_id"] == "user-clean" + + +@pytest.mark.asyncio +async def test_bulk_update_screens_shared_password_with_single_lookup(_admin_prisma, mocker): + """A batch where every user gets the same password costs one HIBP lookup, + not one per user (the serial per-user checks this regresses against).""" + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + bulk_update_processed_users, + ) + + lookup_count = 0 + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal lookup_count + lookup_count += 1 + return httpx.Response(200, text="0000000000000000000000000000000000A:1") + + mock_prisma_client = _admin_prisma + existing_user = mocker.MagicMock() + existing_user.model_dump.return_value = {"user_id": "user-0"} + existing_user.user_id = "user-0" + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock(return_value=existing_user) + mock_prisma_client.update_data = mocker.AsyncMock(return_value={"user_id": "user-0"}) + mock_prisma_client.jsonify_object = mocker.MagicMock(side_effect=lambda x: x) + + admin_caller = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + response = await bulk_update_processed_users( + users_to_update=[UpdateUserRequest(user_id=f"user-{i}", password="NewP@ssw0rd123") for i in range(5)], + user_api_key_dict=admin_caller, + hibp_client=_hibp_client_with_handler(handler), + ) + + assert response.successful_updates == 5 + assert lookup_count == 1 @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py new file mode 100644 index 00000000000..c04353fec99 --- /dev/null +++ b/tests/test_litellm/proxy/management_endpoints/test_password_endpoints.py @@ -0,0 +1,401 @@ +""" +Tests for POST /user/password/change (litellm/proxy/management_endpoints/password_endpoints.py). + +HIBP traffic is intercepted with respx; no test here touches the network. +""" + +import hashlib +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest +import respx +from fastapi import HTTPException + +from litellm.proxy._types import UI_TEAM_ID, LitellmTableNames, ProxyErrorTypes, ProxyException, UserAPIKeyAuth +from litellm.proxy.auth.login_utils import PASSWORD_SESSION_METADATA +from litellm.proxy.management_endpoints.password_endpoints import change_password +from litellm.proxy.utils import hash_password, verify_password + +CURRENT_PASSWORD = "OldP@ssw0rd-2026" +NEW_PASSWORD = "NewP@ssw0rd-2026" + +_POLICY_NO_BREACH_CHECK = {"password_policy_check_breached_passwords": False} + + +def _make_user_row(password: str | None) -> MagicMock: + user = MagicMock() + user.user_id = "user-123" + user.password = password + return user + + +def _make_prisma(user: MagicMock | None) -> MagicMock: + prisma = MagicMock() + prisma.db.litellm_usertable.find_first = AsyncMock(return_value=user) + prisma.db.litellm_usertable.update = AsyncMock(return_value=user) + return prisma + + +def _caller(user_id: str | None = "user-123") -> UserAPIKeyAuth: + return UserAPIKeyAuth(user_id=user_id, team_id=UI_TEAM_ID, metadata=dict(PASSWORD_SESSION_METADATA)) + + +def _sso_session_caller() -> UserAPIKeyAuth: + return UserAPIKeyAuth(user_id="user-123", team_id=UI_TEAM_ID, metadata={}) + + +def _virtual_key_caller() -> UserAPIKeyAuth: + return UserAPIKeyAuth(user_id="user-123", team_id="team-abc", metadata=dict(PASSWORD_SESSION_METADATA)) + + +def _hibp_url_for(password: str) -> str: + sha1 = hashlib.sha1(password.encode(), usedforsecurity=False).hexdigest().upper() + return f"https://api.pwnedpasswords.com/range/{sha1[:5]}" + + +def _hibp_suffix_for(password: str) -> str: + return hashlib.sha1(password.encode(), usedforsecurity=False).hexdigest().upper()[5:] + + +@pytest.mark.asyncio +async def test_change_password_success_writes_new_scrypt_hash(): + from litellm.proxy._types import ChangePasswordRequest + + prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD))) + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), + ): + response = await change_password( + data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD), + user_api_key_dict=_caller(), + ) + + assert response.user_id == "user-123" + update_kwargs = prisma.db.litellm_usertable.update.call_args.kwargs + assert update_kwargs["where"] == {"user_id": "user-123"} + stored = update_kwargs["data"]["password"] + assert stored != NEW_PASSWORD + assert verify_password(NEW_PASSWORD, stored) + # A successful change lifts any pending forced reset and re-arms the + # login-time breach screen for the new password. + assert update_kwargs["data"]["password_reset_required"] is False + assert update_kwargs["data"]["last_breach_check_at"] is None + + +@pytest.mark.asyncio +async def test_change_password_rejects_wrong_current_password(): + from litellm.proxy._types import ChangePasswordRequest + + prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD))) + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), + ): + with pytest.raises(HTTPException) as exc_info: + await change_password( + data=ChangePasswordRequest(current_password="not-the-password", new_password=NEW_PASSWORD), + user_api_key_dict=_caller(), + ) + + assert exc_info.value.status_code == 400 + assert "Current password is incorrect" in exc_info.value.detail["error"] + prisma.db.litellm_usertable.update.assert_not_called() + + +@pytest.mark.asyncio +async def test_change_password_rejects_unchanged_password(): + from litellm.proxy._types import ChangePasswordRequest + + prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD))) + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), + ): + with pytest.raises(HTTPException) as exc_info: + await change_password( + data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=CURRENT_PASSWORD), + user_api_key_dict=_caller(), + ) + + assert exc_info.value.status_code == 400 + assert "must be different from the current password" in exc_info.value.detail["error"] + prisma.db.litellm_usertable.update.assert_not_called() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "caller", + [ + pytest.param(_sso_session_caller(), id="sso_dashboard_session"), + pytest.param(_virtual_key_caller(), id="virtual_key_with_forged_metadata"), + ], +) +async def test_change_password_rejects_non_password_login_session(caller: UserAPIKeyAuth): + """Only the session minted by a password login may change the password, so a + stolen virtual key or an SSO session cannot use the endpoint as a + current_password guessing oracle.""" + from litellm.proxy._types import ChangePasswordRequest + + prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD))) + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), + ): + with pytest.raises(HTTPException) as exc_info: + await change_password( + data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD), + user_api_key_dict=caller, + ) + + assert exc_info.value.status_code == 403 + assert "logging in with a password" in exc_info.value.detail["error"] + prisma.db.litellm_usertable.find_first.assert_not_called() + prisma.db.litellm_usertable.update.assert_not_called() + + +@pytest.mark.asyncio +async def test_change_password_rejects_session_without_user(): + from litellm.proxy._types import ChangePasswordRequest + + prisma = _make_prisma(user=None) + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), + ): + with pytest.raises(HTTPException) as exc_info: + await change_password( + data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD), + user_api_key_dict=_caller(user_id=None), + ) + + assert exc_info.value.status_code == 400 + prisma.db.litellm_usertable.find_first.assert_not_called() + prisma.db.litellm_usertable.update.assert_not_called() + + +@pytest.mark.asyncio +async def test_change_password_rejects_account_without_password(): + """SSO users and the env-credential admin have no DB password row to change.""" + from litellm.proxy._types import ChangePasswordRequest + + prisma = _make_prisma(_make_user_row(password=None)) + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), + ): + with pytest.raises(HTTPException) as exc_info: + await change_password( + data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD), + user_api_key_dict=_caller(), + ) + + assert exc_info.value.status_code == 400 + assert "no password set" in exc_info.value.detail["error"] + prisma.db.litellm_usertable.update.assert_not_called() + + +@pytest.mark.asyncio +async def test_change_password_enforces_min_length(): + from litellm.proxy._types import ChangePasswordRequest + + prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD))) + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), + ): + with pytest.raises(ProxyException) as exc_info: + await change_password( + data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password="Short1!"), + user_api_key_dict=_caller(), + ) + + assert exc_info.value.code == "400" + assert exc_info.value.type == ProxyErrorTypes.validation_error + assert exc_info.value.param == "password" + assert "at least 12 characters" in exc_info.value.message + prisma.db.litellm_usertable.update.assert_not_called() + + +@pytest.mark.asyncio +@respx.mock +async def test_change_password_rejects_breached_password(): + """With the default policy, the new password is screened against HIBP.""" + from litellm.proxy._types import ChangePasswordRequest + + breached_password = "Password123!" + respx.get(_hibp_url_for(breached_password)).mock( + return_value=httpx.Response(200, text=f"{_hibp_suffix_for(breached_password)}:1") + ) + prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD))) + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", {} + ), + ): + with pytest.raises(ProxyException) as exc_info: + await change_password( + data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=breached_password), + user_api_key_dict=_caller(), + ) + + assert exc_info.value.code == "400" + assert exc_info.value.type == ProxyErrorTypes.validation_error + assert exc_info.value.param == "password" + assert "data breaches" in exc_info.value.message + prisma.db.litellm_usertable.update.assert_not_called() + + +@pytest.mark.asyncio +@respx.mock +async def test_change_password_verifies_current_password_before_hibp_lookup(): + """A caller who fails current-password verification must not trigger any + HIBP traffic. The HIBP check fails open on errors, so an unmocked lookup + could not prove ordering; instead the route is registered and asserted + uncalled.""" + from litellm.proxy._types import ChangePasswordRequest + + hibp_route = respx.get(_hibp_url_for(NEW_PASSWORD)).mock(return_value=httpx.Response(200, text="")) + prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD))) + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", {} + ), + ): + with pytest.raises(HTTPException) as exc_info: + await change_password( + data=ChangePasswordRequest(current_password="not-the-password", new_password=NEW_PASSWORD), + user_api_key_dict=_caller(), + ) + + assert exc_info.value.status_code == 400 + assert "Current password is incorrect" in exc_info.value.detail["error"] + assert not hibp_route.called + prisma.db.litellm_usertable.update.assert_not_called() + + +@pytest.mark.asyncio +async def test_change_password_success_emits_redacted_audit_log(): + """A successful change must land in the audit trail as field names only; + the plaintext passwords must never reach the audit call.""" + from litellm.proxy._types import ChangePasswordRequest + + prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD))) + audit_mock = AsyncMock() + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), + patch( # test-quality-ok: audit sink is a module-level import; no injection seam + "litellm.proxy.management_endpoints.password_endpoints.create_object_audit_log", audit_mock + ), + ): + await change_password( + data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD), + user_api_key_dict=_caller(), + ) + + audit_mock.assert_awaited_once() + audit_kwargs = audit_mock.await_args.kwargs + assert audit_kwargs["object_id"] == "user-123" + assert audit_kwargs["action"] == "updated" + assert audit_kwargs["table_name"] == LitellmTableNames.USER_TABLE_NAME + assert audit_kwargs["after_value"] == '{"fields_changed": ["password"]}' + assert CURRENT_PASSWORD not in str(audit_kwargs) + assert NEW_PASSWORD not in str(audit_kwargs) + + +@pytest.mark.asyncio +async def test_change_password_failure_emits_no_audit_log(): + from litellm.proxy._types import ChangePasswordRequest + + prisma = _make_prisma(_make_user_row(hash_password(CURRENT_PASSWORD))) + audit_mock = AsyncMock() + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", prisma + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), + patch( # test-quality-ok: audit sink is a module-level import; no injection seam + "litellm.proxy.management_endpoints.password_endpoints.create_object_audit_log", audit_mock + ), + ): + with pytest.raises(HTTPException): + await change_password( + data=ChangePasswordRequest(current_password="not-the-password", new_password=NEW_PASSWORD), + user_api_key_dict=_caller(), + ) + + audit_mock.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_change_password_requires_db(): + from litellm.proxy._types import ChangePasswordRequest + + with ( + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.prisma_client", None + ), + patch( # test-quality-ok: change_password reads proxy_server module globals; no injection seam + "litellm.proxy.proxy_server.general_settings", _POLICY_NO_BREACH_CHECK + ), + ): + with pytest.raises(HTTPException) as exc_info: + await change_password( + data=ChangePasswordRequest(current_password=CURRENT_PASSWORD, new_password=NEW_PASSWORD), + user_api_key_dict=_caller(), + ) + + assert exc_info.value.status_code == 500 diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py b/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py index 88f8be4e49a..8daea8d0ad2 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_login_sso.py @@ -10,6 +10,7 @@ Routes covered: from __future__ import annotations +from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock from .conftest import normalize @@ -510,6 +511,8 @@ def _db_user(monkeypatch, email: str): user.user_email = email user.user_role = "internal_user" user.password = "scrypt:stored" + user.password_reset_required = None + user.last_breach_check_at = datetime.now(timezone.utc) repo = MagicMock() repo.return_value.table.find_first = AsyncMock(return_value=user) monkeypatch.setattr(ps, "prisma_client", MagicMock()) diff --git a/tests/test_litellm/proxy/test__types.py b/tests/test_litellm/proxy/test__types.py index 9e1486ce90f..f5abe0561db 100644 --- a/tests/test_litellm/proxy/test__types.py +++ b/tests/test_litellm/proxy/test__types.py @@ -5,11 +5,13 @@ from pydantic import ValidationError from litellm.proxy._types import ( ROLES_WITHIN_ORG, + ChangePasswordRequest, GenerateKeyRequest, KeyRequest, LiteLLM_AuditLogs, LiteLLM_TeamMembership, LitellmUserRoles, + NewUserRequest, OrganizationMemberUpdateRequest, ResetSpendRequest, UpdateKeyRequest, @@ -335,3 +337,43 @@ def test_virtual_key_mapping_counts_as_configured_when_any_issuer_sets_the_claim ) assert jwt_auth.is_virtual_key_mapping_configured() is is_configured + + +def test_new_user_request_loudly_rejects_a_password(): + """ + /user/new has never persisted a password (the field used to be silently + dropped). Sending one must now fail visibly so the dead path cannot be + revived without going through the password policy. + """ + with pytest.raises(ValidationError, match="invitation link"): + NewUserRequest(user_email="alice@example.com", password="hunter2hunter2") + + +def test_new_user_request_without_password_still_works(): + request = NewUserRequest(user_email="alice@example.com") + assert request.password is None + + +def test_update_user_request_accepts_a_password(): + """Admins set user passwords through /user/update; the value must survive + model validation so the endpoint can policy-check and hash it.""" + request = UpdateUserRequest(user_id="user-123", password="hunter2hunter2") + assert request.password == "hunter2hunter2" + + +def test_update_user_request_password_hidden_from_repr(): + """management_endpoint_wrapper string-formats endpoint kwargs into Slack + alerts, so the model's repr/str must never contain the plaintext password.""" + request = UpdateUserRequest(user_id="user-123", password="hunter2hunter2") + assert "hunter2hunter2" not in repr(request) + assert "hunter2hunter2" not in str(request) + + +def test_change_password_request_passwords_hidden_from_repr(): + """Any accidental str()/repr() of the request model (debug logs, exception + handlers, a future management_endpoint_wrapper) must never contain either + plaintext password.""" + request = ChangePasswordRequest(current_password="hunter2hunter2", new_password="NewP@ssw0rd-2026") + for rendered in (repr(request), str(request)): + assert "hunter2hunter2" not in rendered + assert "NewP@ssw0rd-2026" not in rendered diff --git a/tests/test_litellm/test_check_mcp_operation_boundary.py b/tests/test_litellm/test_check_mcp_operation_boundary.py new file mode 100644 index 00000000000..d7ac72de9f0 --- /dev/null +++ b/tests/test_litellm/test_check_mcp_operation_boundary.py @@ -0,0 +1,52 @@ +from pathlib import Path + +import pytest + +from scripts.check_mcp_operation_boundary import main, violations + + +@pytest.mark.parametrize( + "source", + ( + "from mcp.server.auth.middleware.auth_context import auth_context_var as hidden", + "from litellm.proxy._experimental.mcp_server.mcp_context import _mcp_proxy_mode as mode", + "caller = legacy.get_active_auth_context()", + "owners = transport._stateful_session_owners", + "from weakref import WeakKeyDictionary", + "from litellm.proxy._experimental.mcp_server.server import get_auth_context", + ), +) +def test_shared_operation_boundary_rejects_ambient_state(source): + assert violations(Path("operations.py"), source) + + +def test_legacy_adapter_may_resolve_context_but_policy_must_receive_it(): + source = "from litellm.proxy._experimental.mcp_server.mcp_context import _mcp_proxy_mode" + assert violations(Path("server.py"), source) == () + assert violations(Path("legacy_callbacks.py"), source) == () + assert violations(Path("operations.py"), "def execute(context):\n return context.client_ip") == () + assert violations(Path("mcp_server_manager.py"), "def _mcp_registry_key(server):\n return server.name") == () + + +def test_boundary_command_rejects_shared_state_and_accepts_explicit_context(tmp_path, monkeypatch, capsys): + import subprocess + import sys + + package = tmp_path / "litellm/proxy/_experimental/mcp_server" + package.mkdir(parents=True) + module = package / "operations.py" + module.write_text("from mcp.server.auth.middleware.auth_context import auth_context_var as hidden\n") + command = [sys.executable, str(Path(__file__).resolve().parents[2] / "scripts/check_mcp_operation_boundary.py")] + monkeypatch.chdir(tmp_path) + assert main() == 1 + assert "operations.py:1:" in capsys.readouterr().err + rejected = subprocess.run(command, cwd=tmp_path, capture_output=True, text=True, check=False) + assert rejected.returncode == 1 + assert "operations.py:1: MCP request/session state belongs in a legacy adapter" in rejected.stderr + + module.write_text("def execute(context):\n return context.client_ip\n") + assert main() == 0 + assert "MCP operation boundary: passed" in capsys.readouterr().out + accepted = subprocess.run(command, cwd=tmp_path, capture_output=True, text=True, check=False) + assert accepted.returncode == 0 + assert "MCP operation boundary: passed" in accepted.stdout diff --git a/tests/test_litellm_rust/support/fake_gcs.py b/tests/test_litellm_rust/support/fake_gcs.py new file mode 100644 index 00000000000..67eb61798b9 --- /dev/null +++ b/tests/test_litellm_rust/support/fake_gcs.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +import json +import threading +from collections.abc import Mapping +from dataclasses import dataclass +from functools import partial +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from socket import socket +from types import MappingProxyType +from typing import Final, cast +from urllib.parse import unquote, urlsplit + + +@dataclass(frozen=True, slots=True) +class RecordedRequest: + method: str + path: str + query: str + headers: Mapping[str, str] + body: bytes + + +class _FakeGcsHandler(BaseHTTPRequestHandler): + def __init__( + self, + request: socket | tuple[bytes, socket], + client_address: tuple[str, int], + server: ThreadingHTTPServer, + *, + fake: FakeGcs, + ) -> None: + self._fake: Final = fake + super().__init__(request, client_address, server) + + def _handle(self) -> None: + parsed: Final = urlsplit(self.path) + content_length: Final = int(self.headers.get("Content-Length", "0")) + body: Final = self.rfile.read(content_length) if content_length else b"" + headers: Final = MappingProxyType( + {name.title(): value for name, value in self.headers.items()} + ) + self._fake.record( + RecordedRequest( + method=self.command, + path=parsed.path, + query=parsed.query, + headers=headers, + body=body, + ) + ) + if self.headers.get("Authorization") != f"Bearer {self._fake.token}": + self._send_json(401, {"error": "unauthorized"}) + return + + upload_prefix: Final = "/upload/storage/v1/b/" + download_prefix: Final = "/storage/v1/b/" + if parsed.path.startswith(upload_prefix) and parsed.path.endswith("/o"): + self._upload(parsed.path[len(upload_prefix) : -2], parsed.query, body) + return + if parsed.path.startswith(download_prefix): + self._download(parsed.path[len(download_prefix) :], parsed.query) + return + self._send_json(404, {"error": "not found"}) + + def _upload(self, path: str, query: str, body: bytes) -> None: + values: Final = { + unquote(pair.partition("=")[0]): unquote(pair.partition("=")[2]) + for pair in query.split("&") + if pair + } + if not path or values.get("uploadType") != "media" or "name" not in values: + self._send_json(404, {"error": "not found"}) + return + self._fake.put_object(path, values["name"], body) + self._send_json(200, {"name": values["name"], "bucket": path}) + + def _download(self, path: str, query: str) -> None: + bucket, separator, encoded_name = path.partition("/o/") + if not separator or query != "alt=media": + self._send_json(404, {"error": "not found"}) + return + name: Final = unquote(encoded_name) + if name.endswith("/server-error") or name == "server-error": + self._send_json(500, {"error": "server error"}) + return + body: Final = self._fake.get_object(bucket, name) + if body is None: + self._send_json(404, {"error": "not found"}) + return + self._send(200, body, "application/octet-stream") + + def _send_json(self, status: int, value: object) -> None: + payload: Final = json.dumps(value).encode() + self._send(status, payload, "application/json") + + def _send(self, status: int, body: bytes, content_type: str) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + pass + + do_GET = _handle + do_POST = _handle + + +class FakeGcs: + def __init__(self) -> None: + self._objects: dict[tuple[str, str], bytes] = {} # mutable-ok: fake object store + self._requests: list[RecordedRequest] = [] # mutable-ok: recorded request history + self._server = ThreadingHTTPServer( + ("127.0.0.1", 0), + partial(_FakeGcsHandler, fake=self), + ) + self._worker = threading.Thread(target=self._server.serve_forever, daemon=True) + self._worker.start() + self.token: Final = "test-token" + + @property + def url(self) -> str: + address: Final = cast(tuple[str, int], self._server.server_address) + host, port = address + return f"http://{host}:{port}" + + @property + def objects(self) -> Mapping[tuple[str, str], bytes]: + return MappingProxyType(self._objects) + + @property + def requests(self) -> tuple[RecordedRequest, ...]: + return tuple(self._requests) + + def put(self, bucket: str, name: str, body: bytes) -> None: + self.put_object(bucket, name, body) + + def close(self) -> None: + self._server.shutdown() + self._server.server_close() + self._worker.join(timeout=5) + + def record(self, request: RecordedRequest) -> None: + self._requests.append(request) + + def put_object(self, bucket: str, name: str, body: bytes) -> None: + self._objects[(bucket, name)] = body + + def get_object(self, bucket: str, name: str) -> bytes | None: + return self._objects.get((bucket, name)) diff --git a/tests/test_litellm_rust/test_cache.py b/tests/test_litellm_rust/test_cache.py index a0f2396a25d..ba95b21d530 100644 --- a/tests/test_litellm_rust/test_cache.py +++ b/tests/test_litellm_rust/test_cache.py @@ -8,10 +8,12 @@ import time import uuid import weakref from collections.abc import Generator +from pathlib import Path from types import SimpleNamespace from typing import Final, Protocol, cast from urllib.parse import urlparse +import diskcache import fakeredis import pytest import redis @@ -20,10 +22,13 @@ from azure.storage.blob import ContainerClient import litellm from litellm.caching.azure_blob_cache import AzureBlobCache from litellm.caching.caching import Cache, disable_cache, enable_cache, update_cache +from litellm.caching.gcs_cache import GCSCache +from litellm.caching.disk_cache import DiskCache from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_cluster_cache import RedisClusterCache from litellm.rust_bridge import _native from litellm.types.caching import LiteLLMCacheType +from tests.test_litellm_rust.support.fake_gcs import FakeGcs from tests.test_litellm_rust.support.isolation import rebound pytestmark: Final = pytest.mark.requires_rust_extension @@ -31,6 +36,7 @@ pytestmark: Final = pytest.mark.requires_rust_extension class CacheLookup(Protocol): def get_cache(self, **kwargs: object) -> object: ... + def flush_cache(self) -> object: ... def request(key: str = "key") -> dict[str, object]: @@ -50,6 +56,15 @@ def redis_url() -> Generator[str]: worker.join(timeout=5) +@pytest.fixture +def fake_gcs() -> Generator[FakeGcs]: + server: Final = FakeGcs() + try: + yield server + finally: + server.close() + + @pytest.fixture def azure_blob_facade() -> Generator[Cache]: account_url: Final = os.environ.get("AZURE_BLOB_CACHE_ACCOUNT_URL") @@ -519,6 +534,328 @@ async def test_redis_facade_buffers_native_async_writes(redis_url: str) -> None: assert client.get("second") is not None await facade.cache.disconnect() client.close() +async def test_disk_reads_python_entries_and_python_reads_native_entries(tmp_path: Path) -> None: + disk_cache: Final = DiskCache(disk_cache_dir=str(tmp_path)) + response: Final = {"choices": [{"text": "cached"}], "usage": {"total_tokens": 3}} + disk_cache.disk_cache.set( + "sync", + {"timestamp": time.time(), "response": json.dumps(response)}, + ) + disk_cache.disk_cache.set("async", json.dumps({"timestamp": time.time(), "response": response})) + disk_cache.disk_cache.set("raw", json.dumps(response)) + disk_cache.disk_cache.set("invalid", "not a cache entry") + disk_cache.disk_cache.set( + "large", + {"timestamp": time.time(), "response": {"text": "x" * 70_000}}, + ) + binding: Final = _native._CacheTestResolver( + SimpleNamespace(cache=_native._CacheTestHandle.disk(str(tmp_path))) + ).resolve() + + assert binding.lookup(request("sync")) == response + assert await binding.async_lookup(request("async")) == response + assert binding.lookup(request("raw")) == response + assert await binding.async_lookup(request("invalid")) is None + assert binding.lookup(request("large")) == {"text": "x" * 70_000} + + await binding.async_store({**request("native"), "ttl_seconds": 12.0}, response) + stored_response: Final = disk_cache.get_cache("native") + assert isinstance(stored_response, dict) + assert stored_response["response"] == response + stored, expire_time = disk_cache.disk_cache.get("native", expire_time=True) + assert stored is not None + assert time.time() < expire_time <= time.time() + 12.0 + await binding.async_store(request("no-ttl"), response) + _, no_expiry = disk_cache.disk_cache.get("no-ttl", expire_time=True) + assert no_expiry is None + + +async def test_disk_entries_survive_a_fresh_handle_and_expire_on_time(tmp_path: Path) -> None: + first: Final = _native._CacheTestResolver( + SimpleNamespace(cache=_native._CacheTestHandle.disk(str(tmp_path))) + ).resolve() + await first.async_store(request("persistent"), {"value": "persistent"}) + await first.async_store({**request("expiring"), "ttl_seconds": 0.3}, {"value": "expiring"}) + fresh: Final = _native._CacheTestResolver( + SimpleNamespace(cache=_native._CacheTestHandle.disk(str(tmp_path))) + ).resolve() + assert fresh.lookup(request("persistent")) == {"value": "persistent"} + assert fresh.lookup(request("expiring")) == {"value": "expiring"} + await asyncio.sleep(0.4) + assert fresh.lookup(request("expiring")) is None + assert fresh.lookup(request("persistent")) == {"value": "persistent"} + + +def test_disk_facade_registers_and_store_changes_fall_back(tmp_path: Path) -> None: + facade: Final = Cache(type=LiteLLMCacheType.DISK, disk_cache_dir=str(tmp_path)) + with pytest.raises(TypeError, match="directories must match"): + _native._CacheTestHandle.disk(str(tmp_path / "other"))._bind_facade(facade) + handle: Final = _native._CacheTestHandle.disk(str(tmp_path)) + handle._bind_facade(facade) + resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade)) + binding: Final = resolver.resolve() + assert binding.kind == "native" + binding.store(request("native"), {"value": "native"}) + assert facade.get_cache(cache_key="native") == {"value": "native"} + + with rebound(facade.cache, "disk_cache", diskcache.Cache(str(tmp_path))): + assert resolver.resolve().kind == "python_callback" + assert resolver.resolve().kind == "native" + + class CustomDiskCache(DiskCache): + pass + + with rebound(facade, "cache", CustomDiskCache(disk_cache_dir=str(tmp_path))): + assert resolver.resolve().kind == "python_callback" + + class CustomStore(diskcache.Cache): + pass + + custom_facade: Final = Cache(type=LiteLLMCacheType.DISK, disk_cache_dir=str(tmp_path)) + custom_facade.cache.disk_cache = CustomStore(str(tmp_path)) + with pytest.raises(TypeError, match="built-in diskcache store"): + _native._CacheTestHandle.disk(str(tmp_path))._bind_facade(custom_facade) + + +async def test_disk_native_batch_lookup_and_store_report_partial_hits(tmp_path: Path) -> None: + binding: Final = _native._CacheTestResolver( + SimpleNamespace(cache=_native._CacheTestHandle.disk(str(tmp_path))) + ).resolve() + requests: Final = [request("hit"), request("miss"), request("disabled")] + requests[2]["controls"] = { + "supported_call_type": True, + "configured": True, + "native_backend": True, + "default_on": True, + "caching": False, + "no_cache": False, + "no_store": False, + "use_cache": False, + } + await binding.async_store_batch(requests, [{"value": 1}, {"value": 2}, {"value": 3}]) + + partial: Final = await binding.async_lookup_batch(requests) + + assert partial == { + "values": [{"value": 1}, {"value": 2}, None], + "missing_indices": [2], + } + + +async def test_gcs_reads_python_entries_and_writes_python_compatible_objects( + fake_gcs: FakeGcs, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("GCS_PATH_SERVICE_ACCOUNT", raising=False) + monkeypatch.delenv("GCS_BUCKET_NAME", raising=False) + response: Final = {"choices": [{"text": "cached"}], "usage": {"total_tokens": 3}, "flag": True, "empty": None} + fake_gcs.put( + "bucket", + "cache/sync", + json.dumps({"timestamp": time.time(), "response": json.dumps(response)}).encode(), + ) + fake_gcs.put("bucket", "cache/async", json.dumps({"timestamp": time.time(), "response": response}).encode()) + fake_gcs.put("bucket", "cache/raw", json.dumps(response).encode()) + fake_gcs.put("bucket", "cache/invalid", b"not a cache entry") + binding: Final = _native._CacheTestResolver( + SimpleNamespace( + cache=_native._CacheTestHandle.gcs( + "bucket", + gcs_path="cache", + endpoint=fake_gcs.url, + token=fake_gcs.token, + ) + ) + ).resolve() + + assert binding.lookup(request("sync")) == response + assert await binding.async_lookup(request("async")) == response + assert binding.lookup(request("raw")) == response + assert await binding.async_lookup(request("invalid")) is None + assert binding.lookup(request("missing")) is None + + await binding.async_store({**request("native"), "ttl_seconds": 12.0}, response) + stored: Final = fake_gcs.objects[("bucket", "cache/native")] + stored_value: Final = cast(dict[str, object], json.loads(stored)) + assert stored_value["response"] == response + assert isinstance(stored_value["timestamp"], float) + upload: Final = next(item for item in fake_gcs.requests if item.method == "POST") + assert upload.path == "/upload/storage/v1/b/bucket/o" + assert upload.query == "uploadType=media&name=cache%2Fnative" + assert upload.headers["Authorization"] == f"Bearer {fake_gcs.token}" + assert upload.headers["Content-Type"] == "application/json" + upload_text: Final = f"{upload.path}?{upload.query}{upload.headers}" + assert "ttl" not in upload_text.lower() + assert "expiry" not in upload_text.lower() + download: Final = next(item for item in fake_gcs.requests if item.path.endswith("/cache%2Fsync")) + assert download.path == "/storage/v1/b/bucket/o/cache%2Fsync" + assert download.query == "alt=media" + + binding.store(request("sync2"), response) + assert binding.lookup(request("sync2")) == response + assert GCSCache(bucket_name="bucket", gcs_path="cache").key_prefix == "cache/" + assert GCSCache(bucket_name="bucket", gcs_path="cache/").key_prefix == "cache/" + assert GCSCache(bucket_name="bucket").key_prefix == "" + + +async def test_gcs_batch_lookup_preserves_order_and_treats_malformed_entries_as_misses(fake_gcs: FakeGcs) -> None: + fake_gcs.put("bucket", "cache/hit", json.dumps({"timestamp": time.time(), "response": {"value": 1}}).encode()) + fake_gcs.put("bucket", "cache/invalid", b"not a cache entry") + binding: Final = _native._CacheTestResolver( + SimpleNamespace( + cache=_native._CacheTestHandle.gcs( + "bucket", + gcs_path="cache", + endpoint=fake_gcs.url, + token=fake_gcs.token, + ) + ) + ).resolve() + requests: Final = [request("hit"), request("missing"), request("invalid")] + expected: Final = {"values": [{"value": 1}, None, None], "missing_indices": [1, 2]} + + assert await binding.async_lookup_batch(requests) == expected + assert binding.lookup_batch(requests) == expected + await binding.async_store_batch([request("first"), request("second")], [{"value": 1}, {"value": 2}]) + assert ("bucket", "cache/first") in fake_gcs.objects + assert ("bucket", "cache/second") in fake_gcs.objects + + +async def test_gcs_facade_binds_only_exact_matching_configuration( + fake_gcs: FakeGcs, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("GCS_PATH_SERVICE_ACCOUNT", raising=False) + monkeypatch.delenv("GCS_BUCKET_NAME", raising=False) + monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", "/nonexistent") + facade: Final = Cache(type=LiteLLMCacheType.GCS, gcs_bucket_name="bucket", gcs_path="cache/") + assert type(facade.cache) is GCSCache + + mismatched_bucket: Final = _native._CacheTestHandle.gcs( + "other", + gcs_path="cache", + endpoint=fake_gcs.url, + token=fake_gcs.token, + ) + with pytest.raises(TypeError, match="buckets must match"): + mismatched_bucket._bind_facade(facade) + mismatched_prefix: Final = _native._CacheTestHandle.gcs( + "bucket", + gcs_path="x", + endpoint=fake_gcs.url, + token=fake_gcs.token, + ) + with pytest.raises(TypeError, match="key prefixes must match"): + mismatched_prefix._bind_facade(facade) + mismatched_credentials: Final = _native._CacheTestHandle.gcs( + "bucket", + gcs_path="cache", + path_service_account="sa.json", + endpoint=fake_gcs.url, + token=fake_gcs.token, + ) + with pytest.raises(TypeError, match="credentials must match"): + mismatched_credentials._bind_facade(facade) + with pytest.raises(TypeError, match="types must match"): + _native._CacheTestHandle.memory()._bind_facade(facade) + + matching: Final = _native._CacheTestHandle.gcs( + "bucket", + gcs_path="cache", + endpoint=fake_gcs.url, + token=fake_gcs.token, + ) + matching._bind_facade(facade) + resolver: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade)) + binding: Final = resolver.resolve() + assert binding.kind == "native" + await binding.async_store(request("native"), {"value": "native"}) + assert await binding.async_lookup(request("native")) == {"value": "native"} + assert cast(CacheLookup, facade).get_cache(cache_key="native") is None + + with rebound(facade.cache, "bucket_name", "other"): + assert resolver.resolve().kind == "python_callback" + with rebound(facade.cache, "key_prefix", "x/"): + assert resolver.resolve().kind == "python_callback" + with rebound(facade.cache, "path_service_account", "sa.json"): + assert resolver.resolve().kind == "python_callback" + def no_get_cache(*args: object, **kwargs: object) -> None: + return None + + with rebound(facade.cache, "get_cache", no_get_cache): + assert resolver.resolve().kind == "python_callback" + with rebound(facade, "ttl", 12): + assert resolver.resolve().kind == "python_callback" + + class CustomGcs(GCSCache): + pass + + with rebound(facade, "cache", CustomGcs(bucket_name="bucket", gcs_path="cache/")): + assert resolver.resolve().kind == "python_callback" + custom_facade: Final = Cache(type=LiteLLMCacheType.GCS, gcs_bucket_name="bucket", gcs_path="cache/") + with rebound(custom_facade, "cache", CustomGcs(bucket_name="bucket", gcs_path="cache/")): + with pytest.raises(TypeError, match="types must match"): + matching._bind_facade(custom_facade) + + missing_bucket: Final = Cache(type=LiteLLMCacheType.GCS) + with pytest.raises(TypeError, match="requires a configured bucket name"): + matching._bind_facade(missing_bucket) + + +async def test_gcs_flush_is_a_no_op_and_ping_is_not_implemented( + fake_gcs: FakeGcs, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("GCS_PATH_SERVICE_ACCOUNT", raising=False) + monkeypatch.delenv("GCS_BUCKET_NAME", raising=False) + binding: Final = _native._CacheTestResolver( + SimpleNamespace( + cache=_native._CacheTestHandle.gcs( + "bucket", + gcs_path="cache", + endpoint=fake_gcs.url, + token=fake_gcs.token, + ) + ) + ).resolve() + await binding.async_store(request("key"), {"value": "stored"}) + await binding.async_flush() + assert ("bucket", "cache/key") in fake_gcs.objects + assert await binding.async_lookup(request("key")) == {"value": "stored"} + with pytest.raises(NotImplementedError): + await binding.ping() + + facade: Final = Cache(type=LiteLLMCacheType.GCS, gcs_bucket_name="bucket", gcs_path="cache/") + with pytest.raises(AttributeError): + await facade.ping() + assert cast(CacheLookup, facade.cache).flush_cache() is None + + +async def test_gcs_unauthorized_and_server_errors_surface_as_runtime_errors(fake_gcs: FakeGcs) -> None: + wrong_token: Final = _native._CacheTestResolver( + SimpleNamespace( + cache=_native._CacheTestHandle.gcs( + "bucket", + gcs_path="cache", + endpoint=fake_gcs.url, + token="wrong-token", + ) + ) + ).resolve() + with pytest.raises(RuntimeError): + wrong_token.lookup(request("missing")) + assert not fake_gcs.objects + + binding: Final = _native._CacheTestResolver( + SimpleNamespace( + cache=_native._CacheTestHandle.gcs( + "bucket", + gcs_path="cache", + endpoint=fake_gcs.url, + token=fake_gcs.token, + ) + ) + ).resolve() + with pytest.raises(RuntimeError): + binding.lookup(request("server-error")) + assert binding.lookup(request("missing")) is None async def test_redis_cluster_facade_serves_multi_slot_batches_and_scoped_flush_natively( diff --git a/tests/unit/models/test_models.py b/tests/unit/models/test_models.py index b8bf55f1b4a..ab456bb1624 100644 --- a/tests/unit/models/test_models.py +++ b/tests/unit/models/test_models.py @@ -324,6 +324,8 @@ class TestUser: assert user_no_models.has_model_access("any-model") def test_password_hash_excluded_from_serialization(self): + import json + from litellm.proxy._types import LiteLLM_UserTableWithKeyCount secret = "$2b$12$abcdefghijklmnopqrstuv" @@ -331,12 +333,12 @@ class TestUser: assert user.password == secret assert "password" not in user.model_dump() - assert "password" not in user.model_dump_json() + assert "password" not in json.loads(user.model_dump_json()) with_keys = LiteLLM_UserTableWithKeyCount(user_id="u1", user_email="a@b.c", password=secret, key_count=2) assert with_keys.password == secret assert "password" not in with_keys.model_dump() - assert "password" not in with_keys.model_dump_json() + assert "password" not in json.loads(with_keys.model_dump_json()) class TestVerificationToken: diff --git a/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.integration.test.tsx new file mode 100644 index 00000000000..c4cf8ebcf9d --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.integration.test.tsx @@ -0,0 +1,110 @@ +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import ChangePasswordForm from "./ChangePasswordForm"; + +const mockChangePasswordCall = vi.fn(); +const mockToastSuccess = vi.fn(); +const mockClearTokenCookies = vi.fn(); +let mockPasswordResetRequired = false; + +vi.mock("@/components/networking", () => ({ + changePasswordCall: (...args: unknown[]) => mockChangePasswordCall(...args), + getProxyBaseUrl: () => "", +})); + +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => ({ accessToken: "sk-session-token", passwordResetRequired: mockPasswordResetRequired }), +})); + +vi.mock("@/lib/toast", () => ({ + toast: { + success: (...args: unknown[]) => mockToastSuccess(...args), + fromError: vi.fn(), + }, +})); + +vi.mock("@/utils/cookieUtils", () => ({ + clearTokenCookies: (...args: unknown[]) => mockClearTokenCookies(...args), +})); + +const fillForm = (values: { current: string; next: string; confirm: string }) => { + fireEvent.change(screen.getByLabelText("Current Password"), { target: { value: values.current } }); + fireEvent.change(screen.getByLabelText("New Password"), { target: { value: values.next } }); + fireEvent.change(screen.getByLabelText("Confirm New Password"), { target: { value: values.confirm } }); +}; + +const submit = () => fireEvent.click(screen.getByRole("button", { name: "Change Password" })); + +describe("ChangePasswordForm", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockPasswordResetRequired = false; + }); + + it("sends the current and new password to the change endpoint and resets on success", async () => { + mockChangePasswordCall.mockResolvedValue({ user_id: "user-123", message: "Password updated successfully." }); + render(); + + fillForm({ current: "OldP@ssw0rd-2026", next: "NewP@ssw0rd-2026", confirm: "NewP@ssw0rd-2026" }); + submit(); + + expect(await screen.findByLabelText("Current Password")).toHaveValue(""); + expect(mockChangePasswordCall).toHaveBeenCalledWith("sk-session-token", "OldP@ssw0rd-2026", "NewP@ssw0rd-2026"); + expect(mockToastSuccess).toHaveBeenCalled(); + }); + + it("blocks submission when the confirmation does not match", async () => { + render(); + + fillForm({ current: "OldP@ssw0rd-2026", next: "NewP@ssw0rd-2026", confirm: "Different-2026" }); + submit(); + + expect(await screen.findByText("New passwords do not match")).toBeInTheDocument(); + expect(mockChangePasswordCall).not.toHaveBeenCalled(); + }); + + it("shows the proxy's rejection message unwrapped", async () => { + mockChangePasswordCall.mockRejectedValue(new Error("{'error': 'Current password is incorrect.'}")); + render(); + + fillForm({ current: "wrong-password", next: "NewP@ssw0rd-2026", confirm: "NewP@ssw0rd-2026" }); + submit(); + + expect(await screen.findByText("Current password is incorrect.")).toBeInTheDocument(); + expect(mockToastSuccess).not.toHaveBeenCalled(); + }); + + describe("forced password reset", () => { + it("shows the forced-reset warning only when the session is flagged", () => { + mockPasswordResetRequired = true; + render(); + + expect(screen.getByText(/must be changed before you can use the dashboard/)).toBeInTheDocument(); + }); + + it("hides the forced-reset warning for a normal session", () => { + render(); + + expect(screen.queryByText(/must be changed before you can use the dashboard/)).not.toBeInTheDocument(); + }); + + it("signs the user out to re-login after a successful forced change", async () => { + mockPasswordResetRequired = true; + mockChangePasswordCall.mockResolvedValue({ user_id: "user-123", message: "Password updated successfully." }); + const replaceMock = vi.fn(); + const realLocation = window.location; + Object.defineProperty(window, "location", { configurable: true, value: { replace: replaceMock } }); + + try { + render(); + fillForm({ current: "OldP@ssw0rd-2026", next: "NewP@ssw0rd-2026", confirm: "NewP@ssw0rd-2026" }); + submit(); + + await waitFor(() => expect(replaceMock).toHaveBeenCalledWith("/ui/login/")); + expect(mockClearTokenCookies).toHaveBeenCalled(); + } finally { + Object.defineProperty(window, "location", { configurable: true, value: realLocation }); + } + }); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.tsx new file mode 100644 index 00000000000..05a6bf3ae94 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/change-password/ChangePasswordForm.tsx @@ -0,0 +1,120 @@ +"use client"; + +import React, { useState } from "react"; +import { CircleAlert } from "lucide-react"; +import { z } from "zod/v4"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { Alert, AlertTitle } from "@/components/shared/Alert"; +import { PasswordInput } from "@/components/shared/PasswordInput"; +import { FormField } from "@/components/shared/form/FormField"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent } from "@/components/ui/card"; +import { FieldGroup } from "@/components/ui/field"; +import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; +import { changePasswordCall, getProxyBaseUrl } from "@/components/networking"; +import { extractProxyErrorMessage } from "@/lib/http/client"; +import { useZodForm } from "@/lib/forms/useZodForm"; +import { toast } from "@/lib/toast"; +import { clearTokenCookies } from "@/utils/cookieUtils"; +import { getLoginUrl } from "@/utils/returnUrlUtils"; + +const changePasswordSchema = z + .object({ + currentPassword: z.string().min(1, "Current password is required"), + newPassword: z.string().min(1, "New password is required"), + confirmNewPassword: z.string().min(1, "Confirm your new password"), + }) + .refine((values) => values.newPassword === values.confirmNewPassword, { + message: "New passwords do not match", + path: ["confirmNewPassword"], + }); + +type ChangePasswordValues = z.infer; + +export function ChangePasswordForm() { + const { accessToken, passwordResetRequired } = useAuthorized(); + const form = useZodForm(changePasswordSchema, { + defaultValues: { currentPassword: "", newPassword: "", confirmNewPassword: "" }, + }); + const [isPending, setIsPending] = useState(false); + const [submitError, setSubmitError] = useState(null); + + const handleSubmit = async (values: ChangePasswordValues) => { + if (!accessToken) return; + setSubmitError(null); + setIsPending(true); + try { + await changePasswordCall(accessToken, values.currentPassword, values.newPassword); + if (passwordResetRequired) { + // The session key was minted restricted; only a fresh login lifts it. + toast.success("Password updated. Please log in with your new password."); + clearTokenCookies(); + window.location.replace(getLoginUrl(getProxyBaseUrl())); + return; + } + toast.success("Password updated"); + form.reset(); + } catch (error) { + setSubmitError(extractProxyErrorMessage(error)); + } finally { + setIsPending(false); + } + }; + + return ( +
+ + +

Change Password

+

+ Enter your current password and choose a new one. The new password must meet this proxy's password + policy. +

+ + {passwordResetRequired && ( + + + + Your password must be changed before you can use the dashboard: it was either found in a known data + breach or set by an administrator as a temporary password. After updating it, you will be signed out to + log in again. + + + )} + +
+ + + {({ ref, ...field }) => } + + + + {({ ref, ...field }) => } + + + + {({ ref, ...field }) => } + + + + {submitError && ( + + + {submitError} + + )} + +
+ +
+
+ + +
+ ); +} + +export default ChangePasswordForm; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/change-password/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/change-password/page.tsx new file mode 100644 index 00000000000..0a6ae926ceb --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/change-password/page.tsx @@ -0,0 +1,7 @@ +"use client"; + +import ChangePasswordForm from "./ChangePasswordForm"; + +export default function ChangePasswordPage() { + return ; +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts index 40d1ec09d1f..581ee8b2580 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts @@ -50,7 +50,9 @@ const useAuthorized = () => { isViewOnly: isViewOnlySessionRole(decoded?.user_role), premiumUser: decoded?.premium_user ?? null, disabledPersonalKeyCreation: decoded?.disabled_non_admin_personal_key_creation ?? null, + loginMethod: decoded?.login_method ?? null, showSSOBanner: decoded?.login_method === "username_password", + passwordResetRequired: decoded?.password_reset_required === true, }; }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx index d7e1bb82564..d854befa197 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx @@ -1,4 +1,4 @@ -import { describe, it, expect, vi, beforeEach } from "vitest"; +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import { render, screen, waitFor } from "@testing-library/react"; import { AuthProvider } from "@/contexts/AuthContext"; import Layout from "./layout"; @@ -121,4 +121,60 @@ describe("(dashboard) Layout", () => { expect(screen.queryByTestId("dashboard-header")).not.toBeInTheDocument(); expect(screen.queryByTestId("sidebar")).not.toBeInTheDocument(); }); + + describe("forced password reset routing", () => { + const sessionCookie = (claims: Record) => { + const encode = (part: Record) => + btoa(JSON.stringify(part)).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); + const exp = Math.floor(Date.now() / 1000) + 3600; + return `${encode({ alg: "HS256", typ: "JWT" })}.${encode({ ...claims, exp })}.sig`; + }; + + afterEach(() => { + document.cookie = "token=; Max-Age=0; Path=/"; + }); + + it("routes a session flagged password_reset_required to the change-password page", async () => { + const flaggedClaims = { + user_id: "flagged-user", + key: "sk-session", + login_method: "username_password", + password_reset_required: true, + }; + document.cookie = `token=${sessionCookie(flaggedClaims)}; Path=/`; + + render( + + +
+ + , + ); + + pendingUiConfig.resolve(); + + await waitFor(() => expect(replaceMock).toHaveBeenCalledWith(expect.stringContaining("/change-password"))); + }); + + it("does not reroute an unflagged session", async () => { + document.cookie = `token=${sessionCookie({ + user_id: "normal-user", + key: "sk-session", + login_method: "username_password", + })}; Path=/`; + + render( + + +
+ + , + ); + + pendingUiConfig.resolve(); + + expect(await screen.findByTestId("page-content")).toBeInTheDocument(); + expect(replaceMock).not.toHaveBeenCalled(); + }); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx index 6d326f5280e..2e903c7b150 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx @@ -7,7 +7,7 @@ import LoadingScreen from "@/components/common_components/LoadingScreen"; import { ThemeProvider } from "@/contexts/ThemeContext"; import { useAuth } from "@/contexts/AuthContext"; import SidebarProvider from "@/app/(dashboard)/components/SidebarProvider"; -import { useRouter, useSearchParams } from "next/navigation"; +import { usePathname, useRouter, useSearchParams } from "next/navigation"; import { DebugWarningBanner } from "@/components/DebugWarningBanner"; import { NoRedisWarningBanner } from "@/components/NoRedisWarningBanner"; import { EnvCredentialLoginWarningBanner } from "@/components/EnvCredentialLoginWarningBanner"; @@ -149,7 +149,8 @@ function DashboardShell({ children }: { children: React.ReactNode }) { function LayoutContent({ children }: { children: React.ReactNode }) { const router = useRouter(); const searchParams = useSearchParams(); - const { accessToken, authLoading } = useAuth(); + const pathname = usePathname(); + const { accessToken, authLoading, passwordResetRequired } = useAuth(); const isInvitationFlow = Boolean(searchParams.get("invitation_id")); // Legacy invitation links point at /ui/?invitation_id=; the onboarding form now lives at its own @@ -160,6 +161,14 @@ function LayoutContent({ children }: { children: React.ReactNode }) { } }, [authLoading, isInvitationFlow, router, searchParams]); + // A session flagged for a forced password reset can only reach the change-password + // endpoint server-side; keep the UI on the matching page. + useEffect(() => { + if (!authLoading && passwordResetRequired && !pathname?.endsWith("/change-password")) { + router.replace(uiHref("change-password")); + } + }, [authLoading, passwordResetRequired, pathname, router]); + if (authLoading || isInvitationFlow) { return ; } diff --git a/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.test.tsx b/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.test.tsx index cad5ced340e..4bdf0da3407 100644 --- a/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.test.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.test.tsx @@ -3,13 +3,25 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { renderWithProviders, screen, waitFor } from "../../../../tests/test-utils"; import UserDropdown from "./UserDropdown"; -let mockUseAuthorizedImpl = () => ({ +let mockUseAuthorizedImpl: () => { + userId: string | null; + userEmail: string | null; + userRoleLabel: string; + premiumUser: boolean; + loginMethod?: string | null; +} = () => ({ userId: "test-user-id", userEmail: "test@example.com", userRoleLabel: "Admin", premiumUser: false, }); +const mockRouterPush = vi.fn(); + +vi.mock("next/navigation", () => ({ + useRouter: () => ({ push: mockRouterPush }), +})); + let mockUseDisableShowPromptsImpl = () => false; let mockGetLocalStorageItemImpl = (key: string): string | null => { @@ -143,6 +155,44 @@ describe("UserDropdown", () => { expect(mockOnLogout).toHaveBeenCalledTimes(1); }); + it("should navigate to the change-password page for username/password sessions", async () => { + mockUseAuthorizedImpl = () => ({ + userId: "test-user-id", + userEmail: "test@example.com", + userRoleLabel: "Admin", + premiumUser: false, + loginMethod: "username_password", + }); + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(getAccountTrigger()); + + await user.click(await screen.findByText("Change Password")); + + expect(mockRouterPush).toHaveBeenCalledWith(expect.stringContaining("change-password")); + }); + + it("should hide the change-password entry for SSO sessions", async () => { + mockUseAuthorizedImpl = () => ({ + userId: "test-user-id", + userEmail: "test@example.com", + userRoleLabel: "Admin", + premiumUser: false, + loginMethod: "sso", + }); + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(getAccountTrigger()); + + await waitFor(() => { + expect(screen.getAllByText("test@example.com").length).toBeGreaterThan(0); + }); + + expect(screen.queryByText("Change Password")).not.toBeInTheDocument(); + }); + it("should toggle hide new feature indicators switch", async () => { const user = userEvent.setup(); renderWithProviders(); diff --git a/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.tsx b/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.tsx index 95f76dbb2cc..9c02defc778 100644 --- a/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/UserDropdown/UserDropdown.tsx @@ -9,7 +9,9 @@ import { setLocalStorageItem, } from "@/utils/localStorageUtils"; import { navAccountDisplayName } from "@/components/Navbar/navDisplayName"; -import { ChevronDown, ChevronsUpDown, Crown, LogOut, Mail, ShieldCheck, User } from "lucide-react"; +import { uiHref } from "@/utils/uiHref"; +import { ChevronDown, ChevronsUpDown, Crown, KeyRound, LogOut, Mail, ShieldCheck, User } from "lucide-react"; +import { useRouter } from "next/navigation"; import { Avatar, AvatarFallback } from "@/components/ui/avatar"; import { Badge } from "@/components/ui/badge"; import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; @@ -63,7 +65,9 @@ interface UserDropdownProps { } const UserDropdown: React.FC = ({ onLogout, variant = "navbar", collapsed = false }) => { - const { userId, userEmail, userRoleLabel: userRole, premiumUser } = useAuthorized(); + const { userId, userEmail, userRoleLabel: userRole, premiumUser, loginMethod } = useAuthorized(); + const router = useRouter(); + const [open, setOpen] = useState(false); const disableShowPrompts = useDisableShowPrompts(); const disableBlogPosts = useDisableBlogPosts(); const disableBouncingIcon = useDisableBouncingIcon(); @@ -197,7 +201,7 @@ const UserDropdown: React.FC = ({ onLogout, variant = "navbar const displayName = navAccountDisplayName(userEmail, userId); return ( - + {variant === "sidebar" ? ( = ({ onLogout, variant = "navbar > {renderUserInfoSection()} + {loginMethod === "username_password" && ( + + )} + )} +