scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-19 00:01:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 88

Merge remote-tracking branch 'origin/main' into litellm_auto_merge_price_sync

Browse source
This commit is contained in:
kerry 2026-09-16 23:56:48 +00:00
commit 42bf9f0ece
189 changed files with 10268 additions and 2013 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
ARCHITECTURE.md
View file

@ -45,7 +45,7 @@ sequenceDiagram
ProxyServer->>Auth: user_api_key_auth()
Auth->>Redis: Check API key cache
Redis-->>Auth: Key info + spend limits
ProxyServer->>Hooks: max_budget_limiter, parallel_request_limiter
ProxyServer->>Hooks: parallel_request_limiter, cache_control_check
Hooks->>Redis: Check/increment rate limit counters
ProxyServer->>Router: route_request()
Router->>Main: litellm.acompletion()
@ -145,7 +145,6 @@ graph TD
| Hook | File | Purpose |
|------|------|---------|
| `max_budget_limiter` | `proxy/hooks/max_budget_limiter.py` | Enforce budget limits |
| `parallel_request_limiter` | `proxy/hooks/parallel_request_limiter_v3.py` | Rate limiting per key/user |
| `cache_control_check` | `proxy/hooks/cache_control_check.py` | Cache validation |
| `responses_id_security` | `proxy/hooks/responses_id_security.py` | Response ID validation |

5
litellm-proxy-extras/litellm_proxy_extras/migrations/20260916000000_add_key_total_spend/migration.sql Normal file
View file

@ -0,0 +1,5 @@
-- AlterTable
ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "total_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN IF NOT EXISTS "total_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

2
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -426,6 +426,7 @@ model LiteLLM_VerificationToken {
key_alias String?
soft_budget_cooldown Boolean @default(false) // key-level state on if budget alerts need to be cooled down
spend Float @default(0.0)
total_spend Float @default(0.0)
expires DateTime?
models String[]
aliases Json @default("{}")
@ -528,6 +529,7 @@ model LiteLLM_DeletedVerificationToken {
key_alias String?
soft_budget_cooldown Boolean @default(false)
spend Float @default(0.0)
total_spend Float @default(0.0)
expires DateTime?
models String[]
aliases Json @default("{}")

102
litellm-rust/Cargo.lock generated
View file

@ -70,6 +70,12 @@ dependencies = [
"rustversion",
]
[[package]]
name = "arcstr"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d"
[[package]]
name = "async-compression"
version = "0.4.46"
@ -1837,6 +1843,12 @@ version = "0.2.186"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
[[package]]
name = "linux-raw-sys"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
[[package]]
name = "litellm-auth"
version = "0.1.0"
@ -1915,6 +1927,17 @@ dependencies = [
"tokio",
]
[[package]]
name = "litellm-cache-redis"
version = "0.1.0"
dependencies = [
"litellm-cache",
"redis",
"redis-test",
"serde_json",
"tokio",
]
[[package]]
name = "litellm-core"
version = "0.1.0"
@ -1951,6 +1974,7 @@ dependencies = [
name = "litellm-python-bridge"
version = "0.1.0"
dependencies = [
"bytes",
"criterion",
"futures-util",
"litellm-auth",
@ -2139,6 +2163,16 @@ dependencies = [
"minimal-lexical",
]
[[package]]
name = "num-bigint"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0"
dependencies = [
"num-integer",
"num-traits",
]
[[package]]
name = "num-conv"
version = "0.2.2"
@ -2655,6 +2689,36 @@ dependencies = [
"crossbeam-utils",
]
[[package]]
name = "redis"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2acbc41a996f7652b2ddd9dfd98cc4ff602cfd742ae35382f07f608405ab50ed"
dependencies = [
"arcstr",
"combine",
"itoa",
"num-bigint",
"percent-encoding",
"ryu",
"sha1_smol",
"socket2 0.6.5",
"url",
"xxhash-rust",
]
[[package]]
name = "redis-test"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "804d36862e4323b69f96440cbb13c9894fc90176abdeaf91264e21d5d77f6aca"
dependencies = [
"rand 0.9.5",
"redis",
"socket2 0.6.5",
"tempfile",
]
[[package]]
name = "redox_syscall"
version = "0.5.18"
@ -2845,6 +2909,19 @@ dependencies = [
"semver",
]
[[package]]
name = "rustix"
version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d"
dependencies = [
"bitflags",
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.61.2",
]
[[package]]
name = "rustls"
version = "0.21.12"
@ -3095,6 +3172,12 @@ dependencies = [
"digest 0.10.7",
]
[[package]]
name = "sha1_smol"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d"
[[package]]
name = "sha2"
version = "0.10.9"
@ -3298,6 +3381,19 @@ version = "0.13.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
[[package]]
name = "tempfile"
version = "3.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
"getrandom 0.4.3",
"once_cell",
"rustix",
"windows-sys 0.61.2",
]
[[package]]
name = "thiserror"
version = "1.0.69"
@ -4181,6 +4277,12 @@ version = "0.13.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
[[package]]
name = "xxhash-rust"
version = "0.8.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6"
[[package]]
name = "yoke"
version = "0.8.3"

15
litellm-rust/crates/cache-redis/Cargo.toml Normal file
View file

@ -0,0 +1,15 @@
[package]
name = "litellm-cache-redis"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
litellm-cache.workspace = true
redis = "1.7.0"
serde_json.workspace = true
tokio.workspace = true
[dev-dependencies]
redis-test = "1.0.4"

315
litellm-rust/crates/cache-redis/src/cache.rs Normal file
View file

@ -0,0 +1,315 @@
use std::sync::{Arc, Mutex, MutexGuard};
use std::time::Duration;
use litellm_cache::{
BaseCache, CacheConnectionResult, CacheConnectionStatus, CacheEntry, CacheFuture, CacheKwargs,
Error,
};
use redis::Commands;
const DEFAULT_TTL: Duration = Duration::from_secs(600);
const KEY_PREFIX: &str = "litellm-cache:";
pub struct RedisCache<C = redis::Connection> {
connection: Arc<Mutex<C>>,
default_ttl: Duration,
}
impl RedisCache<redis::Connection> {
pub fn new(url: &str, default_ttl: Option<Duration>) -> Result<Self, Error> {
let client = redis::Client::open(url).map_err(|_| Error::Unavailable)?;
let connection = client.get_connection().map_err(|_| Error::Unavailable)?;
Ok(Self::with_connection(connection, default_ttl))
}
}
impl<C> RedisCache<C>
where
C: redis::ConnectionLike + Send + 'static,
{
fn with_connection(connection: C, default_ttl: Option<Duration>) -> Self {
Self {
connection: Arc::new(Mutex::new(connection)),
default_ttl: default_ttl.unwrap_or(DEFAULT_TTL),
}
}
fn connection(&self) -> Result<MutexGuard<'_, C>, Error> {
self.connection.lock().map_err(|_| Error::Unavailable)
}
fn namespaced_key(key: &str) -> String {
format!("{KEY_PREFIX}{key}")
}
fn namespaced_pattern() -> &'static str {
const PATTERN: &str = "litellm-cache:*";
PATTERN
}
fn encode(value: &CacheEntry) -> Result<Vec<u8>, Error> {
serde_json::to_vec(value).map_err(|_| Error::InvalidEntry)
}
fn decode(value: Vec<u8>) -> Result<CacheEntry, Error> {
serde_json::from_slice(&value).map_err(|_| Error::InvalidEntry)
}
fn ttl_seconds(ttl: Duration) -> u64 {
ttl.as_secs()
.saturating_add(u64::from(ttl.subsec_nanos() > 0))
.max(1)
}
fn run_blocking<T, F>(connection: Arc<Mutex<C>>, operation: F) -> CacheFuture<'static, T>
where
T: Send + 'static,
F: FnOnce(&mut C) -> Result<T, Error> + Send + 'static,
{
Box::pin(async move {
tokio::task::spawn_blocking(move || {
let mut connection = connection.lock().map_err(|_| Error::Unavailable)?;
operation(&mut connection)
})
.await
.map_err(|_| Error::Unavailable)?
})
}
}
impl<C> BaseCache for RedisCache<C>
where
C: redis::ConnectionLike + Send + 'static,
{
type Value = CacheEntry;
fn default_ttl(&self) -> Duration {
self.default_ttl
}
fn set_cache(&self, key: &str, value: Self::Value, kwargs: CacheKwargs) -> Result<(), Error> {
let payload = Self::encode(&value)?;
let ttl = Self::ttl_seconds(self.get_ttl(&kwargs));
self.connection()?
.set_ex::<_, _, ()>(Self::namespaced_key(key), payload, ttl)
.map_err(|_| Error::Unavailable)
}
fn get_cache(&self, key: &str, _: &CacheKwargs) -> Result<Option<Self::Value>, Error> {
self.connection()?
.get::<_, Option<Vec<u8>>>(Self::namespaced_key(key))
.map_err(|_| Error::Unavailable)?
.map(Self::decode)
.transpose()
}
fn delete_cache(&self, key: &str) -> Result<(), Error> {
self.connection()?
.del::<_, ()>(Self::namespaced_key(key))
.map_err(|_| Error::Unavailable)
}
fn flush_cache(&self) -> Result<(), Error> {
let mut connection = self.connection()?;
let keys = connection
.scan_match(Self::namespaced_pattern())
.map_err(|_| Error::Unavailable)?
.collect::<redis::RedisResult<Vec<String>>>()
.map_err(|_| Error::Unavailable)?;
if keys.is_empty() {
return Ok(());
}
connection
.del::<_, usize>(keys)
.map(|_| ())
.map_err(|_| Error::Unavailable)
}
fn async_set_cache<'a>(
&'a self,
key: &'a str,
value: Self::Value,
kwargs: CacheKwargs,
) -> CacheFuture<'a, ()> {
let payload = Self::encode(&value);
let key = Self::namespaced_key(key);
let ttl = Self::ttl_seconds(self.get_ttl(&kwargs));
Self::run_blocking(Arc::clone(&self.connection), move |connection| {
connection
.set_ex::<_, _, ()>(key, payload?, ttl)
.map_err(|_| Error::Unavailable)
})
}
fn async_get_cache<'a>(
&'a self,
key: &'a str,
_: &'a CacheKwargs,
) -> CacheFuture<'a, Option<Self::Value>> {
let key = Self::namespaced_key(key);
Box::pin(async move {
Self::run_blocking(Arc::clone(&self.connection), move |connection| {
connection
.get::<_, Option<Vec<u8>>>(key)
.map_err(|_| Error::Unavailable)
})
.await?
.map(Self::decode)
.transpose()
})
}
fn async_set_cache_pipeline<'a>(
&'a self,
cache_list: Vec<(String, Self::Value)>,
kwargs: CacheKwargs,
) -> CacheFuture<'a, ()> {
let entries = cache_list
.into_iter()
.map(|(key, value)| {
Self::encode(&value).map(|payload| (Self::namespaced_key(&key), payload))
})
.collect::<Result<Vec<_>, _>>();
let ttl = Self::ttl_seconds(self.get_ttl(&kwargs));
Self::run_blocking(Arc::clone(&self.connection), move |connection| {
for (key, payload) in entries? {
connection
.set_ex::<_, _, ()>(key, payload, ttl)
.map_err(|_| Error::Unavailable)?;
}
Ok(())
})
}
fn async_delete_cache<'a>(&'a self, key: &'a str) -> CacheFuture<'a, ()> {
let key = Self::namespaced_key(key);
Self::run_blocking(Arc::clone(&self.connection), move |connection| {
connection.del::<_, ()>(key).map_err(|_| Error::Unavailable)
})
}
fn disconnect(&self) -> CacheFuture<'_, ()> {
Box::pin(async { Ok(()) })
}
fn test_connection(&self) -> CacheFuture<'_, CacheConnectionResult> {
Box::pin(async move {
Self::run_blocking(Arc::clone(&self.connection), |connection| {
redis::cmd("PING")
.query::<String>(connection)
.map_err(|_| Error::Unavailable)
})
.await?;
Ok(CacheConnectionResult {
status: CacheConnectionStatus::Success,
message: "Redis cache connection test successful".into(),
error: None,
})
})
}
}
#[cfg(test)]
mod tests {
use super::RedisCache;
use litellm_cache::{BaseCache, CacheEntry, CacheKwargs};
use redis_test::{MockCmd, MockRedisConnection};
use serde_json::json;
use std::time::Duration;
fn entry() -> CacheEntry {
CacheEntry {
timestamp: 123.0,
response: json!({"choices": [{"text": "cached"}]}),
}
}
#[test]
fn cache_entries_round_trip_through_json() {
let entry = entry();
let encoded = RedisCache::<redis::Connection>::encode(&entry).unwrap();
assert_eq!(
RedisCache::<redis::Connection>::decode(encoded).unwrap(),
entry
);
}
#[test]
fn invalid_json_is_rejected() {
assert!(RedisCache::<redis::Connection>::decode(b"not json".to_vec()).is_err());
}
#[test]
fn ttl_seconds_rounds_up_and_keeps_expiration_positive() {
assert_eq!(
RedisCache::<redis::Connection>::ttl_seconds(Duration::ZERO),
1
);
assert_eq!(
RedisCache::<redis::Connection>::ttl_seconds(Duration::from_millis(1500)),
2
);
assert_eq!(
RedisCache::<redis::Connection>::ttl_seconds(Duration::from_secs(15)),
15
);
}
#[test]
fn redis_commands_round_trip_entries_and_delete_only_namespaced_keys() {
let value = entry();
let payload = RedisCache::<redis::Connection>::encode(&value).unwrap();
let connection = MockRedisConnection::new([
MockCmd::new(
redis::cmd("SETEX")
.arg("litellm-cache:key")
.arg(600)
.arg(payload.clone()),
Ok("OK"),
),
MockCmd::new(redis::cmd("GET").arg("litellm-cache:key"), Ok(payload)),
MockCmd::new(redis::cmd("DEL").arg("litellm-cache:key"), Ok(1u32)),
])
.assert_all_commands_consumed();
let cache = RedisCache::with_connection(connection, None);
cache
.set_cache("key", value.clone(), CacheKwargs::default())
.unwrap();
assert_eq!(
cache.get_cache("key", &CacheKwargs::default()).unwrap(),
Some(value)
);
cache.delete_cache("key").unwrap();
}
#[test]
fn flush_scans_and_deletes_only_cache_keys() {
let connection = MockRedisConnection::new([
MockCmd::new(
redis::cmd("SCAN")
.cursor_arg(0)
.arg("MATCH")
.arg("litellm-cache:*"),
Ok(redis_test::redis_value!(["0", ["litellm-cache:key"]])),
),
MockCmd::new(redis::cmd("DEL").arg("litellm-cache:key"), Ok(1u32)),
])
.assert_all_commands_consumed();
let cache = RedisCache::with_connection(connection, None);
cache.flush_cache().unwrap();
}
#[tokio::test]
async fn test_connection_runs_ping_off_executor() {
let connection = MockRedisConnection::new([MockCmd::new(redis::cmd("PING"), Ok("PONG"))])
.assert_all_commands_consumed();
let cache = RedisCache::with_connection(connection, None);
assert_eq!(
cache.test_connection().await.unwrap().status,
litellm_cache::CacheConnectionStatus::Success
);
}
}

3
litellm-rust/crates/cache-redis/src/lib.rs Normal file
View file

@ -0,0 +1,3 @@
mod cache;
pub use cache::RedisCache;

6
litellm-rust/crates/cache-redis/tests/cache.rs Normal file
View file

@ -0,0 +1,6 @@
use litellm_cache_redis::RedisCache;
#[test]
fn constructor_rejects_invalid_urls() {
assert!(RedisCache::new("not a redis url", None).is_err());
}

13
litellm-rust/crates/core/src/ocr/client.rs
View file

@ -54,9 +54,16 @@ impl OcrClient {
match call.resume(result.take()).await? {
OcrCallStep::Host(OcrHostOperation::ProjectRequest) => {
result = Some(OcrHostResult::Request(Ok((
Box::new(request.take().ok_or_else(|| {
Error::InvalidRequest("OCR request was already projected".into())
})?),
Box::new(
request
.take()
.ok_or_else(|| {
Error::InvalidRequest(
"OCR request was already projected".into(),
)
})?
.into(),
),
false,
))))
}

126
litellm-rust/crates/core/src/ocr/document.rs
View file

@ -1,3 +1,6 @@
use std::io::Read;
use std::path::Path;
use base64::{Engine, engine::general_purpose::STANDARD};
use data_url::mime::Mime;
use data_url::{DataUrl, DataUrlError, forgiving_base64::DecodeError};
@ -5,12 +8,52 @@ use reqwest::Url;
use serde_json::Map;
use super::error::{OcrError, OcrRequestError, OcrResponseError};
use super::types::{OcrConnection, OcrDocument};
use super::types::{OcrConnection, OcrDocument, OcrDocumentInput};
use crate::constants::{OCR_INLINE_MAX_BYTES, OCR_MAX_FETCH_REDIRECTS};
use crate::media::Error as MediaError;
use crate::media::{DownloadPolicy, MediaFetcher};
use crate::transport::Error as TransportError;
pub fn prepare_document(input: OcrDocumentInput) -> Result<OcrDocument, super::Error> {
match input {
OcrDocumentInput::Document(document) => Ok(document),
OcrDocumentInput::Path { path, mime_type } => {
read_path_document(&path, mime_type.as_deref())
}
OcrDocumentInput::Bytes {
bytes,
file_name,
mime_type,
} => Ok(encode_file_document(
&bytes,
file_name.as_deref(),
mime_type.as_deref(),
)?),
OcrDocumentInput::HostReader { .. } => Err(super::Error::InvalidRequest(
"OCR file reader was not read by the host".into(),
)),
}
}
pub fn read_path_document(
path: &Path,
mime_type: Option<&str>,
) -> Result<OcrDocument, super::Error> {
let mut bytes = Vec::new();
std::fs::File::open(path)
.and_then(|file| {
file.take(OCR_INLINE_MAX_BYTES as u64 + 1)
.read_to_end(&mut bytes)
})
.map_err(|source| super::Error::FileRead {
path: path.to_owned(),
kind: source.kind(),
message: source.to_string(),
})?;
let name = path.file_name().map(|name| name.to_string_lossy());
Ok(encode_file_document(&bytes, name.as_deref(), mime_type)?)
}
pub fn encode_file_document(
bytes: &[u8],
file_name: Option<&str>,
@ -75,18 +118,6 @@ pub fn mime_type_for_name(name: &str) -> &'static str {
}
}
pub fn upload_mime_type<'a>(file_name: Option<&str>, content_type: Option<&'a str>) -> &'a str {
match content_type
.and_then(|value| value.split(';').next())
.map(str::trim)
{
Some(value) if !value.is_empty() && value != "application/octet-stream" => value,
_ => file_name
.map(mime_type_for_name)
.unwrap_or("application/octet-stream"),
}
}
pub(crate) struct InlineDocument<'a>(DataUrl<'a>);
impl<'a> InlineDocument<'a> {
@ -230,24 +261,65 @@ mod tests {
}
#[test]
fn upload_mime_mapping_matches_python() {
fn path_documents_are_read_and_named_by_core() {
let dir = std::env::temp_dir().join(format!("litellm-ocr-{}", rand::random::<u64>()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("scan.png");
std::fs::write(&path, b"abc").unwrap();
assert_eq!(
upload_mime_type(Some("report.pdf"), Some("application/octet-stream")),
"application/pdf"
);
assert_eq!(upload_mime_type(Some("image.png"), None), "image/png");
assert_eq!(upload_mime_type(None, None), "application/octet-stream");
assert_eq!(
upload_mime_type(Some("doc.pdf"), Some("application/pdf; charset=utf-8")),
"application/pdf"
prepare_document(OcrDocumentInput::Path {
path: path.clone(),
mime_type: None,
})
.unwrap(),
OcrDocument::ImageUrl {
image_url: "data:image/png;base64,YWJj".into(),
extra_fields: Map::new(),
}
);
assert_eq!(
upload_mime_type(
Some("img.png"),
Some("image/png; charset=utf-8; boundary=something")
),
"image/png"
prepare_document(OcrDocumentInput::Path {
path: path.clone(),
mime_type: Some("application/pdf".into()),
})
.unwrap(),
document("data:application/pdf;base64,YWJj")
);
std::fs::write(&path, vec![b'a'; OCR_INLINE_MAX_BYTES + 1]).unwrap();
assert_eq!(
prepare_document(OcrDocumentInput::Path {
path: path.clone(),
mime_type: None,
}),
Err(OcrRequestError::InlineDocumentTooLarge.into())
);
std::fs::remove_dir_all(&dir).unwrap();
let missing = dir.join("missing.pdf");
let Err(super::super::Error::FileRead { path, kind, .. }) =
prepare_document(OcrDocumentInput::Path {
path: missing.clone(),
mime_type: None,
})
else {
panic!("missing paths must surface a file read error");
};
assert_eq!(path, missing);
assert_eq!(kind, std::io::ErrorKind::NotFound);
}
#[test]
fn byte_documents_are_encoded_and_host_readers_must_be_read_first() {
assert_eq!(
prepare_document(OcrDocumentInput::Bytes {
bytes: b"abc".as_slice().into(),
file_name: Some("scan.pdf".into()),
mime_type: None,
})
.unwrap(),
document("data:application/pdf;base64,YWJj")
);
assert!(prepare_document(OcrDocumentInput::HostReader { mime_type: None }).is_err());
}
#[test]

6
litellm-rust/crates/core/src/ocr/error.rs
View file

@ -50,6 +50,12 @@ pub enum Error {
Connect(String),
#[error("routing error: {0}")]
Routing(String),
#[error("Failed to read OCR file {}: {message}", path.display())]
FileRead {
path: std::path::PathBuf,
kind: std::io::ErrorKind,
message: String,
},
/// The request is outside the surface this route covers in Rust. Hosts that
/// keep a reference implementation treat this as "fall back", not "fail".
#[error("unsupported by the rust path: {0}")]

50
litellm-rust/crates/core/src/ocr/lifecycle.rs
View file

@ -9,6 +9,7 @@ use super::hooks::{
OcrDuringCallRequest, OcrHookFuture, OcrHooks, OcrLogFuture, OcrPostCallRequest,
OcrPreCallRequest,
};
use super::types::{OcrDocumentInput, OcrFileContent};
use super::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrClient};
use crate::call_lifecycle::host::{
HostCall, HostCallFuture, HostCallStep, HostFailure, HostLifecycle, HostPhase,
@ -52,6 +53,7 @@ impl OcrAdmission {
#[derive(Clone, Debug)]
pub enum OcrHostOperation {
ProjectRequest,
ReadDocument,
Lifecycle(HostPhase),
ConstructResponse(Arc<LiteLLMOcrResponse>),
MapFailure(Error),
@ -83,7 +85,8 @@ impl OcrHostOperation {
}
pub enum OcrHostResult {
Request(Result<(Box<LiteLLMOcrRequest>, bool), Error>),
Request(Result<(Box<LiteLLMOcrRequest<OcrDocumentInput>>, bool), Error>),
Document(Result<OcrFileContent, Error>),
Lifecycle(Result<(), HostFailure<Error>>),
AzureAdToken(Result<ResolvedCredential, AuthError>),
PreCall(Result<OcrPreCallRequest, Error>),
@ -313,7 +316,7 @@ struct PendingOperation {
struct OcrExecution {
client: Option<OcrClient>,
request: Option<LiteLLMOcrRequest>,
request: Option<LiteLLMOcrRequest<OcrDocumentInput>>,
operations_tx: mpsc::UnboundedSender<PendingOperation>,
operations_rx: mpsc::UnboundedReceiver<PendingOperation>,
pending_result: Option<oneshot::Sender<OcrHostResult>>,
@ -397,12 +400,14 @@ impl OcrExecution {
},
)));
}
request.hooks = Arc::new(ProtocolHooks {
let hooks = Arc::new(ProtocolHooks {
operations: self.operations_tx.clone(),
intercepts_requests,
terminal: self.terminal.clone(),
});
request.hooks = hooks.clone();
self.execution = Some(tokio::spawn(async move {
let request = prepare_request_document(request, &hooks).await?;
perform_ocr_request(&client, request).await
}));
}
@ -423,6 +428,39 @@ impl OcrExecution {
}
}
async fn prepare_request_document(
request: LiteLLMOcrRequest<OcrDocumentInput>,
hooks: &ProtocolHooks,
) -> Result<LiteLLMOcrRequest, Error> {
let request = match &request.document {
OcrDocumentInput::HostReader { mime_type } => {
let mime_type = mime_type.clone();
let content = match hooks.invoke(OcrHostOperation::ReadDocument).await? {
OcrHostResult::Document(result) => result?,
_ => {
return Err(Error::InvalidRequest(
"invalid OCR document read host result".into(),
));
}
};
request.with_document(OcrDocumentInput::Bytes {
bytes: content.bytes,
file_name: content.file_name,
mime_type,
})
}
_ => request,
};
if let OcrDocumentInput::Document(_) = &request.document {
return request.map_document(super::document::prepare_document);
}
tokio::task::spawn_blocking(move || request.map_document(super::document::prepare_document))
.await
.map_err(|error| {
Error::InvalidRequest(format!("OCR document preparation task failed: {error}"))
})?
}
impl Drop for OcrExecution {
fn drop(&mut self) {
if let Some(execution) = &self.execution {
@ -567,6 +605,9 @@ impl OcrHost for NoopOcrHost {
OcrHostOperation::ProjectRequest => OcrHostResult::Request(Err(
Error::InvalidRequest("OCR host has no request projection".into()),
)),
OcrHostOperation::ReadDocument => OcrHostResult::Document(Err(
Error::InvalidRequest("OCR host has no document reader".into()),
)),
OcrHostOperation::Lifecycle(_)
| OcrHostOperation::ConstructResponse(_)
| OcrHostOperation::MapFailure(_)
@ -602,6 +643,9 @@ impl OcrHost for OcrHookHost {
OcrHostOperation::ProjectRequest => OcrHostResult::Request(Err(
Error::InvalidRequest("OCR hook host has no request projection".into()),
)),
OcrHostOperation::ReadDocument => OcrHostResult::Document(Err(
Error::InvalidRequest("OCR hook host has no document reader".into()),
)),
OcrHostOperation::Success {
context,
response,

7
litellm-rust/crates/core/src/ocr/mod.rs
View file

@ -13,12 +13,15 @@ pub mod types;
pub mod wire;
pub use client::{OcrClient, ocr};
pub use document::{encode_file_document, mime_type_for_name, upload_mime_type};
pub use document::{encode_file_document, mime_type_for_name, read_path_document};
pub use lifecycle::{
NativeOutcome, NativeResult, NoopOcrHost, OcrAdmission, OcrCall, OcrCallStep, OcrDecline,
OcrHookHost, OcrHost, OcrHostOperation, OcrHostResult,
};
pub use types::{LiteLLMOcrRequest, LiteLLMOcrResponse, OcrConnection, OcrDocument};
pub use types::{
LiteLLMOcrRequest, LiteLLMOcrResponse, OcrConnection, OcrDocument, OcrDocumentInput,
OcrFileContent,
};
#[cfg(test)]
#[path = "../../tests/azure_ai_ocr.rs"]

70
litellm-rust/crates/core/src/ocr/types.rs
View file

@ -1,7 +1,10 @@
use std::collections::BTreeMap;
use std::convert::Infallible;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use bytes::Bytes;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
@ -50,6 +53,35 @@ impl OcrDocument {
}
}
#[derive(Clone, Debug, PartialEq)]
pub enum OcrDocumentInput {
Document(OcrDocument),
Path {
path: PathBuf,
mime_type: Option<String>,
},
Bytes {
bytes: Bytes,
file_name: Option<String>,
mime_type: Option<String>,
},
HostReader {
mime_type: Option<String>,
},
}
impl From<OcrDocument> for OcrDocumentInput {
fn from(document: OcrDocument) -> Self {
Self::Document(document)
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct OcrFileContent {
pub bytes: Bytes,
pub file_name: Option<String>,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum OcrResponseFormat {
@ -89,9 +121,9 @@ impl Default for OcrConnection {
}
}
pub struct LiteLLMOcrRequest {
pub struct LiteLLMOcrRequest<D = OcrDocument> {
pub model: String,
pub document: OcrDocument,
pub document: D,
pub connection: OcrConnection,
pub hooks: Arc<dyn OcrHooks>,
pub litellm_call_id: Option<String>,
@ -101,10 +133,10 @@ pub struct LiteLLMOcrRequest {
pub(crate) adapter: OcrAdapterKind,
}
impl LiteLLMOcrRequest {
impl<D> LiteLLMOcrRequest<D> {
pub fn new(
model: String,
document: OcrDocument,
document: D,
custom_llm_provider: Option<&str>,
optional_params: Map<String, Value>,
) -> Result<Self, Error> {
@ -151,6 +183,36 @@ impl LiteLLMOcrRequest {
..self
}
}
pub fn map_document<T, E>(
self,
map: impl FnOnce(D) -> Result<T, E>,
) -> Result<LiteLLMOcrRequest<T>, E> {
Ok(LiteLLMOcrRequest {
model: self.model,
document: map(self.document)?,
connection: self.connection,
hooks: self.hooks,
litellm_call_id: self.litellm_call_id,
optional_params: self.optional_params,
input_sources: self.input_sources,
azure_ad_token_provider: self.azure_ad_token_provider,
adapter: self.adapter,
})
}
pub fn with_document<T>(self, document: T) -> LiteLLMOcrRequest<T> {
let Ok(request) = self.map_document(|_| Ok::<T, Infallible>(document));
request
}
}
impl From<LiteLLMOcrRequest> for LiteLLMOcrRequest<OcrDocumentInput> {
fn from(request: LiteLLMOcrRequest) -> Self {
let Ok(request) = request
.map_document(|document| Ok::<_, Infallible>(OcrDocumentInput::Document(document)));
request
}
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]

43
litellm-rust/crates/core/src/ocr/wire.rs
View file

@ -68,9 +68,9 @@ pub struct DecodedOcrResponse<T> {
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
pub struct OcrWireRequest {
pub struct OcrWireRequest<D = Value> {
pub model: String,
pub document: Value,
pub document: D,
pub api_key: Option<String>,
pub api_base: Option<String>,
pub custom_llm_provider: Option<String>,
@ -141,10 +141,34 @@ pub fn consumed_optional_params(
}
pub fn decode_request(wire: OcrWireRequest) -> Result<LiteLLMOcrRequest, Error> {
let OcrWireRequest {
model,
document,
api_key,
api_base,
custom_llm_provider,
extra_headers,
optional_params,
input_sources,
timeout_seconds,
} = wire;
decode_request_input(OcrWireRequest {
model,
document: decode_document(document)?,
api_key,
api_base,
custom_llm_provider,
extra_headers,
optional_params,
input_sources,
timeout_seconds,
})
}
pub fn decode_request_input<D>(wire: OcrWireRequest<D>) -> Result<LiteLLMOcrRequest<D>, Error> {
let api_key_source = source_for(&wire.input_sources, "api_key");
let api_base_source = source_for(&wire.input_sources, "api_base");
let extra_headers_source = source_for(&wire.input_sources, "extra_headers");
let document = decode_document(wire.document)?;
let headers = wire
.extra_headers
.unwrap_or_default()
@ -183,7 +207,7 @@ pub fn decode_request(wire: OcrWireRequest) -> Result<LiteLLMOcrRequest, Error>
.unwrap_or(defaults.max_response_bytes);
let request = LiteLLMOcrRequest::new(
wire.model,
document,
wire.document,
wire.custom_llm_provider.as_deref(),
wire.optional_params
.into_iter()
@ -209,14 +233,14 @@ pub fn decode_request(wire: OcrWireRequest) -> Result<LiteLLMOcrRequest, Error>
})
}
fn decode_document(value: Value) -> Result<OcrDocument, OcrRequestError> {
pub fn decode_document(value: Value) -> Result<OcrDocument, Error> {
let kind = value.get("type").and_then(Value::as_str);
let missing_url = matches!(kind, Some("document_url")) && value.get("document_url").is_none()
|| matches!(kind, Some("image_url")) && value.get("image_url").is_none();
if missing_url {
return Err(OcrRequestError::MissingDocumentUrl);
return Err(OcrRequestError::MissingDocumentUrl.into());
}
decode_request_value(value, "document")
Ok(decode_request_value(value, "document")?)
}
fn source_for(sources: &BTreeMap<String, InputSource>, name: &str) -> InputSource {
@ -334,10 +358,7 @@ mod tests {
serde_json::json!({"type": "document_url"}),
serde_json::json!({"type": "image_url"}),
] {
assert_eq!(
decode_document(document),
Err(OcrRequestError::MissingDocumentUrl)
);
assert_eq!(decode_document(document), Err(Error::MissingDocumentUrl));
}
}
}

154
litellm-rust/crates/core/tests/ocr.rs
View file

@ -348,13 +348,14 @@ async fn fallible_host_phases_do_not_replay_or_reach_transport() {
}
OcrHostOperation::ProjectRequest => {
result = Some(OcrHostResult::Request(Ok((
Box::new(request.take().unwrap()),
Box::new(request.take().unwrap().into()),
false,
))))
}
OcrHostOperation::AcquireAzureAdToken => {
panic!("test request has no token provider")
}
OcrHostOperation::ReadDocument => panic!("test request has no file reader"),
OcrHostOperation::PreCall(request) => {
phases.push("pre");
result = Some(OcrHostResult::PreCall(if failure_phase == "pre" {
@ -405,7 +406,7 @@ async fn invalid_provider_response_runs_post_call_before_normalization_failure()
match call.resume(result.take()).await {
Ok(OcrCallStep::Host(OcrHostOperation::ProjectRequest)) => {
result = Some(OcrHostResult::Request(Ok((
Box::new(request.take().unwrap()),
Box::new(request.take().unwrap().into()),
false,
))));
}
@ -467,9 +468,10 @@ async fn direct_native_host_drives_the_same_state_machine() {
_ => panic!("unexpected OCR operation"),
});
result = Some(match operation {
OcrHostOperation::ProjectRequest => {
OcrHostResult::Request(Ok((Box::new(request.take().unwrap()), false)))
}
OcrHostOperation::ProjectRequest => OcrHostResult::Request(Ok((
Box::new(request.take().unwrap().into()),
false,
))),
operation => host.invoke(operation).await,
});
}
@ -501,6 +503,137 @@ async fn direct_native_host_drives_the_same_state_machine() {
));
}
async fn drive_native_file_call(
request: super::LiteLLMOcrRequest<super::OcrDocumentInput>,
content: Result<super::OcrFileContent, crate::ocr::Error>,
) -> (Result<super::LiteLLMOcrResponse, crate::ocr::Error>, usize) {
let NativeOutcome::Completed(mut call) =
OcrCall::admit(super::test_support::ocr_client(), OcrAdmission::all())
else {
panic!("supported call declined")
};
let mut request = Some(request);
let mut content = Some(content);
let mut result = None;
let mut reads = 0;
let outcome = loop {
match call.resume(result.take()).await {
Ok(OcrCallStep::Host(OcrHostOperation::ProjectRequest)) => {
result = Some(OcrHostResult::Request(Ok((
Box::new(request.take().unwrap()),
false,
))));
}
Ok(OcrCallStep::Host(OcrHostOperation::ReadDocument)) => {
reads += 1;
result = Some(OcrHostResult::Document(content.take().unwrap()));
}
Ok(OcrCallStep::Host(operation)) => result = Some(NoopOcrHost.invoke(operation).await),
Ok(OcrCallStep::Complete(response)) => break Ok(response),
Err(error) => break Err(error),
}
};
(outcome, reads)
}
#[tokio::test]
async fn host_reader_documents_are_read_once_at_the_core_selected_point_and_encoded() {
let (base, seen, server) = mock_server(vec![MockResponse::json(json!({
"pages":[{"index":0,"markdown":"file"}]
}))])
.await;
let request = wire_request("mistral/model", &base, json!({})).with_document(
super::OcrDocumentInput::HostReader {
mime_type: Some("application/pdf".into()),
},
);
let (response, reads) = drive_native_file_call(
request,
Ok(super::OcrFileContent {
bytes: b"abc".as_slice().into(),
file_name: Some("scan.png".into()),
}),
)
.await;
server.await.unwrap();
assert_eq!(response.unwrap().pages[0]["markdown"], "file");
assert_eq!(reads, 1);
assert!(seen.lock().unwrap()[0].contains("data:application/pdf;base64,YWJj"));
}
#[tokio::test]
async fn host_reader_failures_and_empty_files_fail_before_the_provider_is_called() {
let (base, seen, _server) = mock_server(vec![]).await;
let request = wire_request("mistral/model", &base, json!({}));
let failure = crate::ocr::Error::InvalidRequest("reader exploded".into());
let (response, reads) = drive_native_file_call(
request.with_document(super::OcrDocumentInput::HostReader { mime_type: None }),
Err(failure.clone()),
)
.await;
assert_eq!(response.unwrap_err(), failure);
assert_eq!(reads, 1);
let request = wire_request("mistral/model", &base, json!({}));
let (response, _) = drive_native_file_call(
request.with_document(super::OcrDocumentInput::HostReader { mime_type: None }),
Ok(super::OcrFileContent {
bytes: Default::default(),
file_name: None,
}),
)
.await;
assert!(matches!(
response.unwrap_err(),
crate::ocr::Error::InvalidRequest(_)
));
assert!(seen.lock().unwrap().is_empty());
}
#[tokio::test]
async fn path_documents_are_read_by_core_without_a_host_operation() {
let (base, seen, server) = mock_server(vec![MockResponse::json(json!({
"pages":[{"index":0,"markdown":"path"}]
}))])
.await;
let dir = std::env::temp_dir().join(format!("litellm-ocr-{}", rand::random::<u64>()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("scan.png");
std::fs::write(&path, b"abc").unwrap();
let request = wire_request("mistral/model", &base, json!({})).with_document(
super::OcrDocumentInput::Path {
path: path.clone(),
mime_type: None,
},
);
let (response, reads) = drive_native_file_call(
request,
Err(crate::ocr::Error::InvalidRequest("unused".into())),
)
.await;
server.await.unwrap();
std::fs::remove_dir_all(&dir).unwrap();
assert_eq!(response.unwrap().pages[0]["markdown"], "path");
assert_eq!(reads, 0);
assert!(seen.lock().unwrap()[0].contains("data:image/png;base64,YWJj"));
let (base, seen, _server) = mock_server(vec![]).await;
let request = wire_request("mistral/model", &base, json!({}));
let (response, _) = drive_native_file_call(
request.with_document(super::OcrDocumentInput::Path {
path: path.clone(),
mime_type: None,
}),
Err(crate::ocr::Error::InvalidRequest("unused".into())),
)
.await;
assert!(matches!(
response.unwrap_err(),
crate::ocr::Error::FileRead { path: failed, kind: std::io::ErrorKind::NotFound, .. } if failed == path
));
assert!(seen.lock().unwrap().is_empty());
}
#[tokio::test]
async fn public_finalization_failure_never_dispatches_success_or_replays_provider() {
use crate::call_lifecycle::host::{HostFailure, HostPhase};
@ -543,9 +676,10 @@ async fn public_finalization_failure_never_dispatches_success_or_replays_provide
| OcrHostOperation::Lifecycle(HostPhase::DeploymentFailure) => {
panic!("finalization failure used provider/success dispatch")
}
OcrHostOperation::ProjectRequest => {
OcrHostResult::Request(Ok((Box::new(request.take().unwrap()), false)))
}
OcrHostOperation::ProjectRequest => OcrHostResult::Request(Ok((
Box::new(request.take().unwrap().into()),
false,
))),
operation => host.invoke(operation).await,
});
}
@ -582,7 +716,7 @@ async fn cancellation_at_provider_hook_prevents_execution_and_further_resumption
OcrCallStep::Host(OcrHostOperation::PreCall(_)) => break,
OcrCallStep::Host(OcrHostOperation::ProjectRequest) => {
result = Some(OcrHostResult::Request(Ok((
Box::new(request.take().unwrap()),
Box::new(request.take().unwrap().into()),
false,
))))
}
@ -799,7 +933,7 @@ async fn cancellation_waits_for_provider_capture_drop_even_when_acknowledgement_
_ = entered.notified() => break,
step = call.resume(result.take()) => {
result = Some(match step.unwrap() {
OcrCallStep::Host(OcrHostOperation::ProjectRequest) => OcrHostResult::Request(Ok((Box::new(request.take().unwrap()), false))),
OcrCallStep::Host(OcrHostOperation::ProjectRequest) => OcrHostResult::Request(Ok((Box::new(request.take().unwrap().into()), false))),
OcrCallStep::Host(operation) => NoopOcrHost.invoke(operation).await,
OcrCallStep::Complete(_) => panic!("pending provider completed"),
});

1
litellm-rust/crates/python-bridge/Cargo.toml
View file

@ -16,6 +16,7 @@ extension-module = ["pyo3/extension-module"]
panic-test = []
[dependencies]
bytes.workspace = true
futures-util.workspace = true
litellm-core.workspace = true
litellm-auth.workspace = true

370
litellm-rust/crates/python-bridge/src/routes/ocr/document.rs
View file

@ -1,97 +1,56 @@
use std::io::Read;
use std::path::PathBuf;
use pyo3::exceptions::{PyFileNotFoundError, PyTypeError, PyValueError};
use bytes::Bytes;
use pyo3::exceptions::{PyTypeError, PyValueError};
use pyo3::gc::{PyTraverseError, PyVisit};
use pyo3::prelude::*;
use pyo3::pybacked::PyBackedBytes;
#[cfg(test)]
use pyo3::types::PyDict;
use pyo3::types::{PyBytes, PyString};
use litellm_core::constants::OCR_INLINE_MAX_BYTES;
use litellm_core::ocr::{OcrDocument, encode_file_document, mime_type_for_name, upload_mime_type};
use litellm_python_interop::to_py_preserving_errors;
use litellm_core::ocr::{OcrDocumentInput, OcrFileContent};
enum FileBytes {
Python(PyBackedBytes),
Native(Vec<u8>),
#[derive(Debug)]
pub(super) struct PythonFileReader {
reader: Py<PyAny>,
name: Option<String>,
}
impl AsRef<[u8]> for FileBytes {
fn as_ref(&self) -> &[u8] {
match self {
Self::Python(bytes) => bytes,
Self::Native(bytes) => bytes,
}
impl PythonFileReader {
pub(super) fn read(&self, py: Python<'_>) -> PyResult<OcrFileContent> {
let value = self.reader.bind(py).call0()?;
let bytes = if value.is_instance_of::<PyString>() {
Bytes::from(value.extract::<String>()?)
} else if value.is_instance_of::<PyBytes>() {
extract_bytes(&value)?
} else {
return Err(PyTypeError::new_err(format!(
"OCR file read must return bytes or str, got {}",
value.get_type(),
)));
};
Ok(OcrFileContent {
bytes,
file_name: self.name.clone(),
})
}
pub(super) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
visit.call(&self.reader)
}
}
fn read_file_input(
py: Python<'_>,
file: &Bound<'_, PyAny>,
) -> PyResult<(FileBytes, Option<String>)> {
if file.is_instance_of::<PyString>() {
return Err(PyValueError::new_err(
"OCR file input does not accept bare str values. Pass bytes, a pathlib.Path, or a file-like object.",
));
fn extract_bytes(value: &Bound<'_, PyAny>) -> PyResult<Bytes> {
if value.is_exact_instance_of::<PyBytes>() {
return Ok(Bytes::from_owner(value.extract::<PyBackedBytes>()?));
}
if file.is_instance(&py.import("os")?.getattr("PathLike")?)? {
let path: PathBuf = file.extract()?;
let name = path
.file_name()
.map(|value| value.to_string_lossy().into_owned());
let bytes = py
.detach(|| {
let mut bytes = Vec::new();
std::fs::File::open(&path)?
.take(OCR_INLINE_MAX_BYTES as u64 + 1)
.read_to_end(&mut bytes)?;
Ok::<_, std::io::Error>(bytes)
})
.map_err(|error| {
if error.kind() == std::io::ErrorKind::NotFound {
PyFileNotFoundError::new_err(format!("File not found: {}", path.display()))
} else {
error.into()
}
})?;
return Ok((FileBytes::Native(bytes), name));
}
if file.is_instance_of::<PyBytes>() {
return Ok((FileBytes::Python(file.extract()?), None));
}
let reader = file
.getattr_opt("read")?
.filter(|value| value.is_callable());
let Some(reader) = reader else {
return Err(PyValueError::new_err(format!(
"Unsupported file input type: {}. Expected pathlib.Path, bytes, or a file-like object.",
file.get_type(),
)));
};
let name = file
.getattr_opt("name")?
.filter(|value| !value.is_none())
.map(|value| value.extract::<String>())
.transpose()?;
let value = reader.call0()?;
let bytes = if value.is_instance_of::<PyString>() {
FileBytes::Native(value.extract::<String>()?.into_bytes())
} else if value.is_instance_of::<PyBytes>() {
FileBytes::Python(value.extract()?)
} else {
return Err(PyTypeError::new_err(format!(
"OCR file read must return bytes or str, got {}",
value.get_type(),
)));
};
Ok((bytes, name))
Ok(Bytes::copy_from_slice(
value.extract::<PyBackedBytes>()?.as_ref(),
))
}
pub(super) struct FileDocumentInput {
bytes: FileBytes,
name: Option<String>,
mime_type: Option<String>,
pub input: OcrDocumentInput,
pub reader: Option<PythonFileReader>,
}
impl FromPyObject<'_, '_> for FileDocumentInput {
@ -104,79 +63,79 @@ impl FromPyObject<'_, '_> for FileDocumentInput {
Err(error) if error.is_instance_of::<pyo3::exceptions::PyKeyError>(py) => None,
Err(error) => return Err(error),
};
let missing = || {
PyValueError::new_err(
"document with type='file' must include a 'file' field containing a pathlib.Path, file-like object, or bytes",
)
};
let file = document.get_item("file").map_err(|error| {
if error.is_instance_of::<pyo3::exceptions::PyKeyError>(py) {
PyValueError::new_err("document with type='file' must include a 'file' field containing a pathlib.Path, file-like object, or bytes")
missing()
} else {
error
}
})?;
if file.is_none() {
return Err(missing());
}
if file.is_instance_of::<PyString>() {
return Err(PyValueError::new_err(
"document with type='file' must include a 'file' field containing a pathlib.Path, file-like object, or bytes",
"OCR file input does not accept bare str values. Pass bytes, a pathlib.Path, or a file-like object.",
));
}
let (bytes, name) = read_file_input(py, &file)?;
if file.is_instance(&py.import("os")?.getattr("PathLike")?)? {
return Ok(Self {
input: OcrDocumentInput::Path {
path: file.extract::<PathBuf>()?,
mime_type,
},
reader: None,
});
}
if file.is_instance_of::<PyBytes>() {
return Ok(Self {
input: OcrDocumentInput::Bytes {
bytes: extract_bytes(&file)?,
file_name: None,
mime_type,
},
reader: None,
});
}
let reader = file
.getattr_opt("read")?
.filter(|value| value.is_callable());
let Some(reader) = reader else {
return Err(PyValueError::new_err(format!(
"Unsupported file input type: {}. Expected pathlib.Path, bytes, or a file-like object.",
file.get_type(),
)));
};
let name = file
.getattr_opt("name")?
.filter(|value| !value.is_none())
.map(|value| value.extract::<String>())
.transpose()?;
Ok(Self {
bytes,
name,
mime_type,
input: OcrDocumentInput::HostReader { mime_type },
reader: Some(PythonFileReader {
reader: reader.unbind(),
name,
}),
})
}
}
pub(super) fn file_document(py: Python<'_>, document: FileDocumentInput) -> PyResult<OcrDocument> {
py.detach(|| {
encode_file_document(
document.bytes.as_ref(),
document.name.as_deref(),
document.mime_type.as_deref(),
)
})
.map_err(|error| PyValueError::new_err(error.to_string()))
}
#[pyfunction]
fn _ocr_file_document(py: Python<'_>, document: Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
to_py_preserving_errors(py, &file_document(py, document.extract()?)?)
}
#[pyfunction]
fn _ocr_mime_type(file_name: &str) -> String {
mime_type_for_name(file_name).into()
}
#[pyfunction]
#[pyo3(signature = (file_content, file_name=None, content_type=None))]
fn _ocr_upload_document(
py: Python<'_>,
file_content: &Bound<'_, PyBytes>,
file_name: Option<&str>,
content_type: Option<&str>,
) -> PyResult<Py<PyAny>> {
let bytes: PyBackedBytes = file_content.extract()?;
let document = py
.detach(|| {
encode_file_document(
&bytes,
None,
Some(upload_mime_type(file_name, content_type)),
)
})
.map_err(|error| PyValueError::new_err(error.to_string()))?;
to_py_preserving_errors(py, &document)
}
pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
module.add("_OCR_MAX_FILE_BYTES", OCR_INLINE_MAX_BYTES)?;
module.add_function(wrap_pyfunction!(_ocr_upload_document, module)?)?;
module.add_function(wrap_pyfunction!(_ocr_file_document, module)?)?;
module.add_function(wrap_pyfunction!(_ocr_mime_type, module)?)
}
#[cfg(test)]
mod tests {
use super::*;
use pyo3::types::PyDict;
fn eval<'py>(py: Python<'py>, source: &std::ffi::CStr) -> Bound<'py, PyDict> {
let locals = PyDict::new(py);
py.run(source, Some(&locals), Some(&locals)).unwrap();
locals
}
#[test]
fn extraction_validates_required_file_and_optional_mime_type() {
@ -196,69 +155,148 @@ mod tests {
let error = document.extract::<FileDocumentInput>().err().unwrap();
assert!(error.is_instance_of::<PyTypeError>(py));
}
let document = py.eval(c"{'file': b'abc'}", None, None).unwrap();
let error = py
.eval(c"{'file': 'scan.pdf'}", None, None)
.unwrap()
.extract::<FileDocumentInput>()
.err()
.unwrap();
assert!(error.is_instance_of::<PyValueError>(py));
assert!(error.to_string().contains("bare str"));
let document = py
.eval(c"{'file': b'abc', 'mime_type': 'image/png'}", None, None)
.unwrap();
let input: FileDocumentInput = document.extract().unwrap();
assert_eq!(input.bytes.as_ref(), b"abc");
assert_eq!(input.name, None);
assert_eq!(input.mime_type, None);
assert!(input.reader.is_none());
assert_eq!(
input.input,
OcrDocumentInput::Bytes {
bytes: b"abc".as_slice().into(),
file_name: None,
mime_type: Some("image/png".into()),
}
);
});
}
#[test]
fn extraction_validates_mime_type_before_consuming_file() {
fn paths_and_readers_are_projected_without_io() {
Python::initialize();
Python::attach(|py| {
let locals = PyDict::new(py);
py.run(
c"class Reader:
let locals = eval(
py,
c"from pathlib import Path
class Reader:
name = 'scan.png'
def __init__(self):
self.reads = 0
def read(self):
self.reads += 1
return b'abc'
reader = Reader()
document = {'file': reader, 'mime_type': 7}",
Some(&locals),
Some(&locals),
)
.unwrap();
document = {'file': reader, 'mime_type': 7}
reader_document = {'file': reader}
path_document = {'file': Path('/nonexistent/ocr-projection-test.pdf'), 'mime_type': 'image/png'}",
);
let document = locals.get_item("document").unwrap().unwrap();
let error = document.extract::<FileDocumentInput>().err().unwrap();
assert!(error.is_instance_of::<PyTypeError>(py));
let reads: usize = locals
.get_item("reader")
.unwrap()
.unwrap()
.getattr("reads")
.unwrap()
.extract()
.unwrap();
assert_eq!(reads, 0);
let document = locals.get_item("reader_document").unwrap().unwrap();
let input: FileDocumentInput = document.extract().unwrap();
assert_eq!(
input.input,
OcrDocumentInput::HostReader { mime_type: None }
);
let reads = || {
locals
.get_item("reader")
.unwrap()
.unwrap()
.getattr("reads")
.unwrap()
.extract::<usize>()
.unwrap()
};
assert_eq!(reads(), 0);
let content = input.reader.unwrap().read(py).unwrap();
assert_eq!(reads(), 1);
assert_eq!(
content,
OcrFileContent {
bytes: b"abc".as_slice().into(),
file_name: Some("scan.png".into()),
}
);
let document = locals.get_item("path_document").unwrap().unwrap();
let input: FileDocumentInput = document.extract().unwrap();
assert!(input.reader.is_none());
assert_eq!(
input.input,
OcrDocumentInput::Path {
path: PathBuf::from("/nonexistent/ocr-projection-test.pdf"),
mime_type: Some("image/png".into()),
}
);
});
}
#[test]
fn extraction_preserves_reader_key_error_identity() {
fn reader_results_are_normalized_and_exceptions_keep_their_identity() {
Python::initialize();
Python::attach(|py| {
let locals = PyDict::new(py);
py.run(
let locals = eval(
py,
c"failure = KeyError('reader failed')
class Reader:
class Raising:
def read(self):
raise failure
document = {'file': Reader()}",
Some(&locals),
Some(&locals),
)
.unwrap();
let document = locals.get_item("document").unwrap().unwrap();
let error = document.extract::<FileDocumentInput>().err().unwrap();
class Text:
def read(self):
return 'héllo'
class Wrong:
def read(self):
return 7
raising = {'file': Raising()}
text = {'file': Text()}
wrong = {'file': Wrong()}",
);
let reader = |name: &str| {
locals
.get_item(name)
.unwrap()
.unwrap()
.extract::<FileDocumentInput>()
.unwrap()
.reader
.unwrap()
};
let error = reader("raising").read(py).unwrap_err();
assert!(
error
.value(py)
.is(locals.get_item("failure").unwrap().unwrap())
);
assert_eq!(
reader("text").read(py).unwrap().bytes.as_ref(),
"héllo".as_bytes()
);
let error = reader("wrong").read(py).unwrap_err();
assert!(error.is_instance_of::<PyTypeError>(py));
assert!(error.to_string().contains("bytes or str"));
});
}
#[test]
fn exact_python_bytes_transfer_without_copying_and_outlive_the_input() {
Python::initialize();
let (bytes, pointer) = Python::attach(|py| {
let value = PyBytes::new(py, b"document bytes");
let pointer = value.as_bytes().as_ptr() as usize;
(extract_bytes(value.as_any()).unwrap(), pointer)
});
assert_eq!(bytes.as_ptr() as usize, pointer);
assert_eq!(bytes.as_ref(), b"document bytes");
}
}

7
litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs
View file

@ -1,4 +1,5 @@
use litellm_core::ocr::Error;
use pyo3::exceptions::{PyFileNotFoundError, PyOSError};
use pyo3::prelude::*;
use crate::errors::{RustUpstreamError, core_error_to_pyerr};
@ -7,6 +8,12 @@ pub(super) fn to_pyerr(error: Error) -> PyErr {
let status = error.http_status_code();
let mapped = match error {
Error::Http { status, body } => RustUpstreamError::new_err((status, body)),
Error::FileRead {
path,
kind: std::io::ErrorKind::NotFound,
..
} => PyFileNotFoundError::new_err(format!("File not found: {}", path.display())),
Error::FileRead { message, .. } => PyOSError::new_err(message),
other => core_error_to_pyerr(other.into()),
};
attach_status(mapped, status)

22
litellm-rust/crates/python-bridge/src/routes/ocr/lifecycle.rs
View file

@ -66,13 +66,27 @@ impl PythonOcrHost {
retained_fields.set_item(name, value)?;
}
}
retained_fields.set_item("document", &self.projected()?.fields.document)?;
let projected = self.projected_mut()?;
let document = match &projected.fields.document {
Some(document) => document.clone_ref(py),
None => to_py(py, &request.document)?,
};
retained_fields.set_item("document", &document)?;
projected.fields.document = Some(document);
projected.retained_fields = Some(retained_fields.unbind());
projected.pre_call = Some((&request).into());
Ok(request)
}
fn read_document(&self, py: Python<'_>) -> PyResult<litellm_core::ocr::OcrFileContent> {
self.projected()?
.fields
.reader
.as_ref()
.ok_or_else(missing_state)?
.read(py)
}
fn acquire_azure_ad_token(&self, py: Python<'_>) -> PyResult<ResolvedCredential> {
let provider = self
.projected()?
@ -193,7 +207,7 @@ impl PythonRoute for PythonOcrHost {
let OcrHostData::Unprojected { request } = &self.data else {
return Err(missing_state());
};
let projected = project_request(py, request.bind(py), self.state.kwargs.bind(py))?;
let projected = project_request(request.bind(py), self.state.kwargs.bind(py))?;
let has_token_provider = projected.fields.azure_ad_token_provider.is_some();
let request = projected.request;
self.data = OcrHostData::Projected(Box::new(ProjectedOcrHost {
@ -205,6 +219,7 @@ impl PythonRoute for PythonOcrHost {
}));
OcrHostResult::Request(Ok((Box::new(request), has_token_provider)))
}
OcrHostOperation::ReadDocument => OcrHostResult::Document(Ok(self.read_document(py)?)),
OcrHostOperation::AcquireAzureAdToken => {
OcrHostResult::AzureAdToken(Ok(self.acquire_azure_ad_token(py)?))
}
@ -258,6 +273,9 @@ impl PythonRoute for PythonOcrHost {
OcrHostData::Projected(projected) => {
visit.call(&projected.fields.boundary_request)?;
visit.call(&projected.fields.document)?;
if let Some(reader) = &projected.fields.reader {
reader.traverse(visit)?;
}
visit.call(&projected.fields.api_key)?;
if let Some(provider) = &projected.fields.azure_ad_token_provider {
provider.traverse(visit)?;

1
litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs
View file

@ -9,6 +9,5 @@ use pyo3::prelude::*;
pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
value::register(module)?;
document::register(module)?;
lifecycle::register(module)
}

141
litellm-rust/crates/python-bridge/src/routes/ocr/project.rs
View file

@ -1,14 +1,15 @@
use std::sync::Arc;
use litellm_core::ocr::wire::{OcrWireRequest, consumed_optional_params, decode_request};
use litellm_core::ocr::{LiteLLMOcrRequest, NativeOutcome, OcrCall};
use litellm_python_interop::{
from_py_preserving_errors as from_py, to_py_preserving_errors as to_py,
use litellm_core::ocr::wire::{
OcrWireRequest, consumed_optional_params, decode_document, decode_request_input,
};
use litellm_core::ocr::{LiteLLMOcrRequest, NativeOutcome, OcrCall, OcrDocumentInput};
use litellm_python_interop::from_py_preserving_errors as from_py;
use pyo3::prelude::*;
use pyo3::types::PyDict;
use serde_json::{Map, Value};
use super::document::{FileDocumentInput, PythonFileReader};
use super::errors::to_pyerr as ocr_error_to_pyerr;
use super::lifecycle::BridgeOcrHooks;
use crate::auth::{AZURE_AD_TOKEN_PROVIDER, PythonTokenProvider};
@ -17,7 +18,8 @@ use crate::marshal::{project_optional_fields, python_timeout_seconds, request_in
pub(super) struct ProjectedOcrFields {
pub boundary_request: Py<PyAny>,
pub document: Py<PyAny>,
pub document: Option<Py<PyAny>>,
pub reader: Option<PythonFileReader>,
pub api_key: Py<PyAny>,
pub azure_ad_token_provider: Option<PythonTokenProvider>,
pub provider: &'static str,
@ -25,7 +27,7 @@ pub(super) struct ProjectedOcrFields {
}
pub(super) struct ProjectedOcrCall {
pub request: LiteLLMOcrRequest,
pub request: LiteLLMOcrRequest<OcrDocumentInput>,
pub fields: ProjectedOcrFields,
}
@ -80,12 +82,12 @@ impl<'py> OcrArguments<'_, 'py> {
}
enum ProjectedDocument {
File { wire: Value, retained: Py<PyAny> },
File(FileDocumentInput),
Other { wire: Value, retained: Py<PyAny> },
}
impl ProjectedDocument {
fn project(py: Python<'_>, document: &Bound<'_, PyAny>) -> PyResult<Self> {
fn project(document: &Bound<'_, PyAny>) -> PyResult<Self> {
let kind: String = document.get_item("type")?.extract()?;
if kind != "file" {
return Ok(Self::Other {
@ -93,25 +95,28 @@ impl ProjectedDocument {
retained: document.clone().unbind(),
});
}
let input = document.extract()?;
let encoded = super::document::file_document(py, input)?;
let wire = serde_json::to_value(encoded)
.map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?;
Ok(Self::File {
retained: to_py(py, &wire)?,
wire,
})
Ok(Self::File(document.extract()?))
}
fn into_parts(self) -> (Value, Py<PyAny>) {
fn into_parts(
self,
) -> PyResult<(
OcrDocumentInput,
Option<Py<PyAny>>,
Option<PythonFileReader>,
)> {
match self {
Self::File { wire, retained } | Self::Other { wire, retained } => (wire, retained),
Self::File(FileDocumentInput { input, reader }) => Ok((input, None, reader)),
Self::Other { wire, retained } => Ok((
decode_document(wire).map_err(ocr_error_to_pyerr)?.into(),
Some(retained),
None,
)),
}
}
}
pub(super) fn project_request(
py: Python<'_>,
request: &Bound<'_, PyAny>,
kwargs: &Bound<'_, PyDict>,
) -> PyResult<ProjectedOcrCall> {
@ -119,8 +124,7 @@ pub(super) fn project_request(
let arguments = OcrArguments { request, kwargs };
let model = arguments.model()?;
let custom_llm_provider = arguments.custom_llm_provider()?;
let (wire_document, retained_document) =
ProjectedDocument::project(py, &arguments.document()?)?.into_parts();
let document = ProjectedDocument::project(&arguments.document()?)?;
let api_key = arguments.api_key()?;
let specs = consumed_optional_params(&model, custom_llm_provider.as_deref())
.map_err(ocr_error_to_pyerr)?;
@ -136,9 +140,10 @@ pub(super) fn project_request(
let azure_ad_token_provider = kwargs
.get_item("azure_ad_token_provider")?
.and_then(|provider| PythonTokenProvider::select(provider, AZURE_AD_TOKEN_PROVIDER));
let (document, retained_document, reader) = document.into_parts()?;
let wire = OcrWireRequest {
model,
document: wire_document,
document,
api_key: api_key.extract()?,
api_base: arguments.api_base()?,
custom_llm_provider,
@ -147,13 +152,14 @@ pub(super) fn project_request(
input_sources,
timeout_seconds: arguments.timeout_seconds()?,
};
let request = decode_request(wire).map_err(ocr_error_to_pyerr)?;
let request = decode_request_input(wire).map_err(ocr_error_to_pyerr)?;
let provider = request.provider_name();
Ok(ProjectedOcrCall {
request: request.with_host_hooks(Arc::new(BridgeOcrHooks), None),
fields: ProjectedOcrFields {
boundary_request,
document: retained_document,
reader,
api_key: api_key.unbind(),
azure_ad_token_provider,
provider,
@ -197,10 +203,21 @@ mod tests {
}
fn project_document(
py: Python<'_>,
document: &Bound<'_, PyAny>,
) -> PyResult<(Value, Py<PyAny>)> {
ProjectedDocument::project(py, document).map(ProjectedDocument::into_parts)
) -> PyResult<(
OcrDocumentInput,
Option<Py<PyAny>>,
Option<PythonFileReader>,
)> {
ProjectedDocument::project(document)?.into_parts()
}
fn url_document(url: &str) -> OcrDocumentInput {
litellm_core::ocr::OcrDocument::DocumentUrl {
document_url: url.into(),
extra_fields: Map::new(),
}
.into()
}
fn stub_timeout_conversion(py: Python<'_>) {
@ -374,7 +391,7 @@ kwargs = {}
}
#[test]
fn document_reader_mutations_are_visible_to_later_field_reads() {
fn document_readers_are_not_consumed_during_projection() {
Python::initialize();
Python::attach(|py| {
stub_timeout_conversion(py);
@ -406,7 +423,12 @@ kwargs = {}
.unwrap();
let arguments = arguments(&request, &kwargs);
let document = arguments.document().unwrap();
project_document(py, &document).unwrap();
let (input, retained, reader) = project_document(&document).unwrap();
assert_eq!(input, OcrDocumentInput::HostReader { mime_type: None });
assert!(retained.is_none());
assert_eq!(arguments.api_base().unwrap().as_deref(), Some("original"));
assert_eq!(arguments.timeout_seconds().unwrap(), Some(1.0));
reader.unwrap().read(py).unwrap();
assert_eq!(arguments.api_base().unwrap().as_deref(), Some("mutated"));
assert_eq!(arguments.timeout_seconds().unwrap(), Some(9.0));
});
@ -444,7 +466,7 @@ kwargs = {'api_key': key}
}
#[test]
fn file_documents_are_encoded_and_other_documents_keep_the_python_object() {
fn file_documents_become_typed_inputs_and_other_documents_keep_the_python_object() {
Python::initialize();
Python::attach(|py| {
let file = py
@ -454,13 +476,17 @@ kwargs = {'api_key': key}
None,
)
.unwrap();
let (input, retained, reader) = project_document(&file).unwrap();
assert_eq!(
project_document(py, &file).unwrap().0,
serde_json::json!({
"type": "document_url",
"document_url": "data:application/pdf;base64,JVBERi0xLjQ=",
})
input,
OcrDocumentInput::Bytes {
bytes: b"%PDF-1.4".as_slice().into(),
file_name: None,
mime_type: Some("application/pdf".into()),
}
);
assert!(retained.is_none());
assert!(reader.is_none());
let original = py
.eval(
@ -469,44 +495,21 @@ kwargs = {'api_key': key}
None,
)
.unwrap();
let (wire, retained) = project_document(py, &original).unwrap();
assert_eq!(
wire,
serde_json::json!({
"type": "document_url",
"document_url": "https://example.com/a.pdf",
})
);
assert!(retained.bind(py).is(&original));
let (input, retained, _) = project_document(&original).unwrap();
assert_eq!(input, url_document("https://example.com/a.pdf"));
assert!(retained.unwrap().bind(py).is(&original));
});
}
#[test]
fn unknown_document_types_reach_existing_downstream_validation() {
fn unknown_document_types_reach_existing_core_validation() {
Python::initialize();
Python::attach(|py| {
let document = py
.eval(c"{'type': 'mystery', 'mystery': 'x'}", None, None)
.unwrap();
let wire_document = project_document(py, &document).unwrap().0;
assert_eq!(
wire_document,
serde_json::json!({"type": "mystery", "mystery": "x"})
);
let error = match decode_request(OcrWireRequest {
model: "mistral/mistral-ocr-latest".into(),
document: wire_document,
api_key: None,
api_base: None,
custom_llm_provider: None,
extra_headers: None,
optional_params: Map::new(),
input_sources: Default::default(),
timeout_seconds: None,
}) {
Ok(_) => panic!("unknown discriminators belong to core validation"),
Err(error) => error,
};
let error = project_document(&document).unwrap_err();
assert!(error.is_instance_of::<PyValueError>(py));
assert!(error.to_string().contains("document"));
});
}
@ -517,14 +520,14 @@ kwargs = {'api_key': key}
Python::attach(|py| {
let missing = py.eval(c"{}", None, None).unwrap();
assert!(
project_document(py, &missing)
project_document(&missing)
.unwrap_err()
.is_instance_of::<PyKeyError>(py)
);
let non_string = py.eval(c"{'type': 1}", None, None).unwrap();
assert!(
project_document(py, &non_string)
project_document(&non_string)
.unwrap_err()
.is_instance_of::<PyTypeError>(py)
);
@ -540,7 +543,7 @@ document = Document()
",
);
let error =
project_document(py, &locals.get_item("document").unwrap().unwrap()).unwrap_err();
project_document(&locals.get_item("document").unwrap().unwrap()).unwrap_err();
assert!(
error
.value(py)
@ -569,9 +572,9 @@ document = Document()
",
);
let document = locals.get_item("document").unwrap().unwrap();
let (wire, retained) = project_document(py, &document).unwrap();
assert_eq!(wire["type"], "document_url");
assert!(!retained.bind(py).is(&document));
let (input, retained, _) = project_document(&document).unwrap();
assert!(matches!(input, OcrDocumentInput::Bytes { .. }));
assert!(retained.is_none());
let reads: Vec<String> = document.getattr("reads").unwrap().extract().unwrap();
assert_eq!(reads, ["type", "mime_type", "file"]);
});

2
litellm/constants.py
View file

@ -40,6 +40,7 @@ ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset(
"router_general_settings",
"ignore_invalid_deployments",
"fallback_access_check",
"fallback_budget_check",
"auto_router_capability_limit",
}
)
@ -53,6 +54,7 @@ S3_BOUNDED_OBJECT_KEY_HEAD_BYTES: Final = 64
S3_PREFIX_DIGEST_CHARS: Final = 16
# s3 allows 2048 bytes of combined metadata headers, which Content-Disposition counts against
MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES: Final = 1024
S3_LOG_PROMPTS_ONLY_ENV_VAR: Final = "S3_LOG_PROMPTS_ONLY"
MAX_FILE_LIST_LIMIT: Final = 10000
DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10))
DEFAULT_NUM_WORKERS_LITELLM_PROXY: Final = int(os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1))

8
litellm/experimental_mcp_client/client.py
View file

@ -346,13 +346,17 @@ class MCPClient:
self.update_auth_value(auth_value)
async def discovery_auth_fingerprint(self) -> str:
return self._hash_discovery_auth(await self.prepare_request_auth())
async def prepare_request_auth(self) -> httpx.Request:
"""Preview the authenticated request without sending it, closing the auth flow afterwards."""
request: Final = httpx.Request("POST", self.server_url or "http://localhost/", headers=self._get_auth_headers())
if self._resolved_auth is None:
return self._hash_discovery_auth(request)
return request
flow: Final = self._resolved_auth.async_auth_flow(request)
try:
authenticated: Final = await flow.__anext__()
return self._hash_discovery_auth(authenticated)
return authenticated
finally:
await flow.aclose()

6
litellm/integrations/callback_configs.json
View file

@ -446,6 +446,12 @@
"ui_name": "S3 Path Prefix",
"description": "Path prefix within the bucket for organizing logs",
"required": false
},
"s3_log_prompts_only": {
"type": "boolean",
"ui_name": "Log Prompts Only",
"description": "Log request messages to S3 but drop the model response from each logged object",
"required": false
}
},
"description": "S3 Bucket (AWS) Logging Integration"

1
litellm/integrations/custom_logger.py
View file

@ -118,6 +118,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
alias_map: Final = {
"langfuse_otel": "langfuse",
"s3_v2": "s3",
}
lookup_name: Final = alias_map.get(normalized_name, normalized_name)

11
litellm/integrations/opentelemetry.py
View file

@ -2965,16 +2965,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
)
propagator: Final = TraceContextTextMapPropagator()
carrier: Final = {"traceparent": _traceparent}
carrier: Final = {key: headers[key] for key in ("traceparent", "tracestate") if headers.get(key) is not None}
_parent_context: Final = propagator.extract(carrier=carrier)
return _parent_context
def _get_span_context(self, kwargs, default_span: Span | None = None):
from opentelemetry import context, trace
from opentelemetry.trace.propagation.tracecontext import (
TraceContextTextMapPropagator,
)
litellm_params: Final = kwargs.get("litellm_params", {}) or {}
proxy_server_request: Final = litellm_params.get("proxy_server_request", {}) or {}
@ -2998,11 +2995,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
# Priority 2: HTTP traceparent header
if traceparent is not None:
verbose_logger.debug("OpenTelemetry: Using traceparent header for context propagation")
carrier: Final = {"traceparent": traceparent}
return (
TraceContextTextMapPropagator().extract(carrier=carrier),
None,
)
return self.get_traceparent_from_header(headers=headers), None
# Priority 3: Active span from global context (auto-detection)
try:

32
litellm/integrations/otel/plumbing/context.py
View file

@ -26,6 +26,7 @@ if TYPE_CHECKING:
from litellm.integrations.otel.model.destination import OtelDestination
_PROPAGATOR: Final = TraceContextTextMapPropagator()
_W3C_TRACE_HEADERS: Final = frozenset(("traceparent", "tracestate"))
# The request's root span — the FastAPI-owned SERVER span — captured ONCE when the
# proxy first resolves it, so request-level spans (the LLM call, guardrails) can
@ -310,6 +311,37 @@ def extract_traceparent(headers: Mapping[str, str]) -> Context | None:
return _PROPAGATOR.extract(carrier)
def _outgoing_trace_context(parent_span: object) -> Context | None:
if isinstance(parent_span, Span) and is_recordable_span(parent_span):
return context_from_span(parent_span)
root: Final = request_root_span()
if root is not None:
return context_from_span(root)
current: Final = get_current()
if is_recordable_span(get_current_span(current)):
return current
return None
def inject_trace_context(headers: Mapping[str, str], parent_span: object = None) -> dict[str, str]:
"""``headers`` plus W3C ``traceparent``/``tracestate`` for this request's span.
Parent preference: ``parent_span`` (the request span auth stashed on the key), then
the anchored request root span, then the ambient active span. Only trace context is
injected, never Baggage. Unchanged when no valid span exists anywhere.
"""
context: Final = _outgoing_trace_context(parent_span)
if context is None:
return dict(headers) # mutable-ok: OpenTelemetry propagator requires a mutable carrier
carrier: Final = { # mutable-ok: OpenTelemetry propagator requires a mutable carrier
key: value for key, value in headers.items() if key.lower() not in _W3C_TRACE_HEADERS
}
_PROPAGATOR.inject(carrier, context=context)
return carrier
# The OTLP destinations this request's key or team pointed its traces at, resolved
# once during auth. A ``ContextVar`` for the same reason the root span above is one:
# it rides the request task's context into the ``asyncio.create_task`` children that

69
litellm/integrations/s3.py
View file

@ -2,19 +2,42 @@
# On success + failure, log events to Supabase
import hashlib
import os
from collections.abc import Mapping
from datetime import datetime
from typing import Final, cast
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import print_verbose, verbose_logger
from litellm.constants import (
MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES,
MAX_S3_OBJECT_KEY_BYTES,
S3_BOUNDED_OBJECT_KEY_HEAD_BYTES,
S3_LOG_PROMPTS_ONLY_ENV_VAR,
S3_PREFIX_DIGEST_CHARS,
)
from litellm.types.utils import StandardLoggingPayload
_S3_LOG_PROMPTS_ONLY: Final = TypeAdapter(bool)
def resolve_s3_log_prompts_only(configured: object, environ: Mapping[str, str] | None = None) -> bool:
env: Final = os.environ if environ is None else environ
raw: Final = env.get(S3_LOG_PROMPTS_ONLY_ENV_VAR) if configured is None else configured
if raw is None or raw == "":
return False
try:
return _S3_LOG_PROMPTS_ONLY.validate_python(raw.strip() if isinstance(raw, str) else raw)
except ValidationError:
verbose_logger.warning("s3 logging: s3_log_prompts_only=%r is not a boolean, logging prompts only", raw)
return True
def prompts_only_payload(payload: StandardLoggingPayload) -> StandardLoggingPayload:
return {**payload, "response": None}
class S3Logger:
# Class variables or attributes
@ -33,6 +56,7 @@ class S3Logger:
s3_config=None,
s3_server_side_encryption: str | None = None,
s3_sse_kms_key_id: str | None = None,
s3_log_prompts_only: bool | None = None,
**kwargs,
):
import boto3
@ -41,29 +65,30 @@ class S3Logger:
verbose_logger.debug("in init s3 logger - s3_callback_params %s", litellm.s3_callback_params)
s3_use_team_prefix = False
params: Final = {
key: litellm.get_secret(value) if isinstance(value, str) and value.startswith("os.environ/") else value
for key, value in (litellm.s3_callback_params or {}).items()
}
if litellm.s3_callback_params is not None:
# read in .env variables - example os.environ/AWS_BUCKET_NAME
for key, value in litellm.s3_callback_params.items():
if isinstance(value, str) and value.startswith("os.environ/"):
litellm.s3_callback_params[key] = litellm.get_secret(value)
# now set s3 params from litellm.s3_logger_params
s3_bucket_name = litellm.s3_callback_params.get("s3_bucket_name")
s3_region_name = litellm.s3_callback_params.get("s3_region_name")
s3_api_version = litellm.s3_callback_params.get("s3_api_version")
s3_use_ssl = litellm.s3_callback_params.get("s3_use_ssl", True)
s3_verify = litellm.s3_callback_params.get("s3_verify")
s3_endpoint_url = litellm.s3_callback_params.get("s3_endpoint_url")
s3_aws_access_key_id = litellm.s3_callback_params.get("s3_aws_access_key_id")
s3_aws_secret_access_key = litellm.s3_callback_params.get("s3_aws_secret_access_key")
s3_aws_session_token = litellm.s3_callback_params.get("s3_aws_session_token")
s3_config = litellm.s3_callback_params.get("s3_config")
s3_path = litellm.s3_callback_params.get("s3_path")
s3_server_side_encryption = litellm.s3_callback_params.get("s3_server_side_encryption")
s3_sse_kms_key_id = litellm.s3_callback_params.get("s3_sse_kms_key_id")
# done reading litellm.s3_callback_params
s3_use_team_prefix = bool(litellm.s3_callback_params.get("s3_use_team_prefix", False))
s3_bucket_name = params.get("s3_bucket_name")
s3_region_name = params.get("s3_region_name")
s3_api_version = params.get("s3_api_version")
s3_use_ssl = params.get("s3_use_ssl", True)
s3_verify = params.get("s3_verify")
s3_endpoint_url = params.get("s3_endpoint_url")
s3_aws_access_key_id = params.get("s3_aws_access_key_id")
s3_aws_secret_access_key = params.get("s3_aws_secret_access_key")
s3_aws_session_token = params.get("s3_aws_session_token")
s3_config = params.get("s3_config")
s3_path = params.get("s3_path")
s3_server_side_encryption = params.get("s3_server_side_encryption")
s3_sse_kms_key_id = params.get("s3_sse_kms_key_id")
s3_use_team_prefix = bool(params.get("s3_use_team_prefix", False))
self.s3_use_team_prefix = s3_use_team_prefix
self.s3_log_prompts_only: object = (
params.get("s3_log_prompts_only") if s3_log_prompts_only is None else s3_log_prompts_only
)
self.bucket_name = s3_bucket_name
self.s3_path = s3_path
self.s3_server_side_encryption, self.s3_sse_kms_key_id = resolve_sse_params(
@ -144,7 +169,9 @@ class S3Logger:
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
payload_str: Final = safe_dumps(payload)
payload_str: Final = safe_dumps(
prompts_only_payload(payload) if resolve_s3_log_prompts_only(self.s3_log_prompts_only) else payload
)
print_verbose(f"\ns3 Logger - Logging payload = {payload_str}")

16
litellm/integrations/s3_v2.py
View file

@ -21,6 +21,8 @@ from litellm.constants import DEFAULT_S3_BATCH_SIZE, DEFAULT_S3_FLUSH_INTERVAL_S
from litellm.integrations.s3 import (
get_s3_object_download_filename,
get_s3_object_key,
prompts_only_payload,
resolve_s3_log_prompts_only,
resolve_sse_params,
)
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
@ -68,6 +70,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
s3_use_virtual_hosted_style: bool = False,
s3_server_side_encryption: str | None = None,
s3_sse_kms_key_id: str | None = None,
s3_log_prompts_only: bool | None = None,
s3_callback_params_override: dict | None = None,
**kwargs,
):
@ -108,6 +111,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
s3_use_virtual_hosted_style=s3_use_virtual_hosted_style,
s3_server_side_encryption=s3_server_side_encryption,
s3_sse_kms_key_id=s3_sse_kms_key_id,
s3_log_prompts_only=s3_log_prompts_only,
)
verbose_logger.debug("s3 logger using endpoint url %s", s3_endpoint_url)
@ -163,6 +167,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
s3_use_virtual_hosted_style: bool = False,
s3_server_side_encryption: str | None = None,
s3_sse_kms_key_id: str | None = None,
s3_log_prompts_only: bool | None = None,
params_source: dict | None = None,
):
"""
@ -212,6 +217,10 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
bool(params.get("s3_use_virtual_hosted_style", False)) or s3_use_virtual_hosted_style
)
self.s3_log_prompts_only: object = (
params.get("s3_log_prompts_only") if s3_log_prompts_only is None else s3_log_prompts_only
)
self.s3_server_side_encryption, self.s3_sse_kms_key_id = resolve_sse_params(
params.get("s3_server_side_encryption") or s3_server_side_encryption,
params.get("s3_sse_kms_key_id") or s3_sse_kms_key_id,
@ -489,8 +498,13 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
s3_object_download_filename: Final = get_s3_object_download_filename(start_time, standard_logging_payload["id"])
payload: Final = (
prompts_only_payload(standard_logging_payload)
if resolve_s3_log_prompts_only(self.s3_log_prompts_only)
else standard_logging_payload
)
return s3BatchLoggingElement(
payload=dict(standard_logging_payload),
payload=dict(payload),
s3_object_key=s3_object_key,
s3_object_download_filename=s3_object_download_filename,
)

27
litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py
View file

@ -78,6 +78,7 @@ _PROPAGATED_METADATA_KEYS: Final = (
"user_api_key_end_user_id",
"user_api_end_user_max_budget",
"user_api_key_model_max_budget",
"user_api_key_team_model_max_budget",
"user_api_key_user_model_max_budget",
"user_api_key_end_user_model_max_budget",
"litellm_call_id",
@ -395,9 +396,9 @@ async def _check_summary_model_budget(
``user_api_key_auth`` runs for the client-requested model. Returns True outside the proxy or when no
per-model budget is configured.
All three scopes are checked because the summary's spend is charged to all
three: this file propagates the key, user and end-user budgets into the
subrequest's metadata, so enforcing only two of them would let compaction
Every scope is checked because the summary's spend is charged to every
scope: this file propagates the key, team, user and end-user budgets into the
subrequest's metadata, so skipping one of them would let compaction
increment a counter it can never be refused by.
"""
if user_api_key_auth is None:
@ -444,6 +445,26 @@ async def _check_summary_model_budget(
)
return False
team_model_max_budget: Final = user_api_key_auth.team_model_max_budget
team_id: Final = user_api_key_auth.team_id
if isinstance(team_model_max_budget, dict) and team_model_max_budget and team_id is not None:
try:
await model_max_budget_limiter.is_team_within_model_budget(
team_id=team_id,
team_model_max_budget=team_model_max_budget,
key_model_max_budget=model_max_budget if isinstance(model_max_budget, dict) else None,
model=summary_model,
)
except litellm.BudgetExceededError:
return False
except Exception as e: # noqa: BLE001 # a budget gate denies on any failure, as the other scopes do
verbose_logger.warning(
"compact_20260112: unexpected error during team model-budget check for summary_model=%s; denying: %s",
summary_model,
e,
)
return False
end_user_model_max_budget: Final[dict[str, object] | None] = getattr(
user_api_key_auth, "end_user_model_max_budget", None
)

6
litellm/llms/dashscope/chat/transformation.py
View file

@ -12,6 +12,12 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig
class DashScopeChatConfig(OpenAIGPTConfig):
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: base class contract returns a list
return [ # mutable-ok: base class contract returns a list
*super().get_supported_openai_params(model=model),
"reasoning_effort",
]
def remove_cache_control_flag_from_messages_and_tools(
self,
model: str,

4
litellm/llms/openai/chat/gpt_5_transformation.py
View file

@ -4,9 +4,9 @@ from typing import Final
import litellm
from litellm.utils import (
_is_explicitly_disabled_factory,
_supports_factory,
declared_value_factory,
is_explicitly_disabled_factory,
)
from .gpt_transformation import OpenAIGPTConfig
@ -192,7 +192,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
Use this for opt-out checks where unknown models should be allowed through.
"""
return _is_explicitly_disabled_factory(
return is_explicitly_disabled_factory(
model=cls._model_map_lookup_name(model),
custom_llm_provider=None,
key=f"supports_{level}_reasoning_effort",

37
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -79,6 +79,7 @@ from litellm.utils import (
CustomStreamWrapper,
ModelResponse,
is_base64_encoded,
is_explicitly_disabled_factory,
supports_reasoning,
)
@ -866,6 +867,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
else:
raise _unsupported_reasoning_effort(reasoning_effort)
@staticmethod
def _supports_minimal_thinking_level(model: str) -> bool:
lowered: Final = model.lower()
is_gemini3flash: Final = "gemini-3" in lowered and "flash" in lowered
return is_gemini3flash and not is_explicitly_disabled_factory(
model=model, custom_llm_provider=None, key="supports_minimal_reasoning_effort"
)
@staticmethod
def _map_reasoning_effort_to_thinking_level(
reasoning_effort: str,
@ -880,13 +889,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
Returns:
GeminiThinkingConfig with thinkingLevel and includeThoughts
"""
# Check if this is gemini-3-flash which supports MINIMAL thinking level
# Covers gemini-3-flash, gemini-3-flash-preview, gemini-3.1-flash, gemini-3.1-flash-lite-preview,
# gemini-3.5-flash, and any future 3.x-flash variants.
is_gemini3flash: Final = model and ("flash" in model.lower() and "gemini-3" in model.lower())
supports_minimal: Final = bool(model) and VertexGeminiConfig._supports_minimal_thinking_level(model)
is_gemini31pro: Final = model and ("gemini-3.1-pro-preview" in model.lower())
if reasoning_effort == "minimal":
if is_gemini3flash:
if supports_minimal:
return {"thinkingLevel": "minimal", "includeThoughts": True}
else:
return {"thinkingLevel": "low", "includeThoughts": True}
@ -899,18 +906,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
return {"thinkingLevel": "high", "includeThoughts": True}
elif reasoning_effort == "high":
return {"thinkingLevel": "high", "includeThoughts": True}
elif reasoning_effort == "disable":
# Gemini 3 cannot fully disable thinking, so we use "minimal" for gemini-3-flash-preview, "low" for others
if is_gemini3flash:
return {"thinkingLevel": "minimal", "includeThoughts": False}
else:
return {"thinkingLevel": "low", "includeThoughts": False}
elif reasoning_effort == "none":
# For gemini-3-flash-preview, use "minimal" instead of "low"
if is_gemini3flash:
return {"thinkingLevel": "minimal", "includeThoughts": False}
else:
return {"thinkingLevel": "low", "includeThoughts": False}
elif reasoning_effort in ("disable", "none"):
return {
"thinkingLevel": "minimal" if supports_minimal else "low",
"includeThoughts": False,
}
else:
raise _unsupported_reasoning_effort(reasoning_effort)
@ -977,8 +977,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
params["includeThoughts"] = True
# Follow provider defaults unless explicitly opted into legacy behavior.
if litellm.enable_gemini_default_thinking_level_low is True:
is_gemini3flash: Final = "gemini-3" in model.lower() and "flash" in model.lower()
params["thinkingLevel"] = "minimal" if is_gemini3flash else "low"
params["thinkingLevel"] = (
"minimal" if VertexGeminiConfig._supports_minimal_thinking_level(model) else "low"
)
else:
# Thinking disabled
params["includeThoughts"] = False

24
litellm/model_prices_and_context_window_backup.json
View file

@ -377,8 +377,7 @@
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_tool_choice": true,
"cache_read_input_token_cost": 1.5e-08
"supports_tool_choice": true
},
"amazon.nova-2-lite-v1:0": {
"cache_read_input_token_cost": 7.5e-08,
@ -561,8 +560,7 @@
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"cache_read_input_token_cost": 8.75e-09
"supports_tool_choice": true
},
"amazon.nova-pro-v1:0": {
"cache_read_input_token_cost": 2e-07,
@ -578,8 +576,7 @@
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_tool_choice": true,
"cache_read_input_token_cost": 2e-07
"supports_tool_choice": true
},
"amazon.nova-sonic-v1:0": {
"deprecation_date": "2026-09-14",
@ -26108,6 +26105,7 @@
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_minimal_reasoning_effort": false,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
@ -26165,6 +26163,7 @@
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_minimal_reasoning_effort": false,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
@ -28114,6 +28113,7 @@
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_minimal_reasoning_effort": false,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
@ -28173,6 +28173,7 @@
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_minimal_reasoning_effort": false,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
@ -28595,6 +28596,7 @@
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_minimal_reasoning_effort": false,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
@ -28652,6 +28654,7 @@
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_minimal_reasoning_effort": false,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
@ -45794,8 +45797,7 @@
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_tool_choice": true,
"cache_read_input_token_cost": 1.5e-08
"supports_tool_choice": true
},
"us.amazon.nova-micro-v1:0": {
"cache_read_input_token_cost": 8.75e-09,
@ -45809,8 +45811,7 @@
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"cache_read_input_token_cost": 8.75e-09
"supports_tool_choice": true
},
"us.amazon.nova-premier-v1:0": {
"deprecation_date": "2026-09-14",
@ -45842,8 +45843,7 @@
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_tool_choice": true,
"cache_read_input_token_cost": 2e-07
"supports_tool_choice": true
},
"us.anthropic.claude-3-5-haiku-20241022-v1:0": {
"cache_creation_input_token_cost": 1e-06,

2
litellm/models/verification_token.py
View file

@ -18,6 +18,7 @@ class LiteLLM_VerificationToken(LiteLLMPydanticObjectBase):
key_name: str | None = None
key_alias: str | None = None
spend: float = 0.0
total_spend: float = 0.0
max_budget: float | None = None
expires: str | datetime | None = None
models: list = []
@ -69,6 +70,7 @@ class LiteLLM_DeletedVerificationToken(LiteLLM_VerificationToken):
"""Audit record for deleted keys; mirrors the token plus deletion metadata."""
id: str | None = None
organization_id: str | None = None
deleted_at: datetime | None = None
deleted_by: str | None = None
deleted_by_api_key: str | None = None

112
litellm/ocr/input.py
View file

@ -1,112 +0,0 @@
from collections.abc import Mapping
from os import PathLike
from typing import Final, Literal, Protocol, cast # noqa: TID251 # native callables are validated when loaded
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.rust_bridge.bindings import NativeBinding
from litellm.rust_bridge.configuration import rust_ocr_enabled
class FileReader(Protocol):
def read(self) -> bytes | str: ...
class FileDocument(TypedDict):
type: ReadOnly[Literal["file"]]
file: ReadOnly[bytes | PathLike[str] | FileReader]
mime_type: ReadOnly[NotRequired[str]]
class NativeFileDocument(Protocol):
def __call__(self, document: Mapping[str, object]) -> dict[str, str]: ...
class NativeUploadDocument(Protocol):
def __call__(self, file_content: bytes, file_name: str | None, content_type: str | None) -> dict[str, str]: ...
class NativeMimeType(Protocol):
def __call__(self, file_name: str) -> str: ...
_FILE_DOCUMENT: Final = NativeBinding(
"_ocr_file_document",
validate=lambda value: (
cast( # cast-ok: native export owns the callable signature
NativeFileDocument, value
)
if callable(value)
else None
),
)
_UPLOAD_DOCUMENT: Final = NativeBinding(
"_ocr_upload_document",
validate=lambda value: (
cast( # cast-ok: native export owns the callable signature
NativeUploadDocument, value
)
if callable(value)
else None
),
)
_MAX_FILE_BYTES: Final = NativeBinding(
"_OCR_MAX_FILE_BYTES", validate=lambda value: value if isinstance(value, int) and value > 0 else None
)
_MIME_TYPE: Final = NativeBinding(
"_ocr_mime_type",
validate=lambda value: (
cast( # cast-ok: native export owns the callable signature
NativeMimeType, value
)
if callable(value)
else None
),
)
_PYTHON_MAX_FILE_BYTES: Final = 50 * 1024 * 1024
def get_mime_type(file_path: str) -> str:
native: Final = _MIME_TYPE.load() if rust_ocr_enabled() else None
if native is None:
from litellm.ocr import legacy
return legacy.get_mime_type(file_path)
return native(file_path)
def get_max_file_bytes() -> int:
limit: Final = _MAX_FILE_BYTES.load() if rust_ocr_enabled() else None
if limit is None:
return _PYTHON_MAX_FILE_BYTES
return limit
def convert_file_document_to_url_document(document: FileDocument) -> dict[str, str]:
native: Final = _FILE_DOCUMENT.load() if rust_ocr_enabled() else None
if native is None:
from litellm.ocr import legacy
return legacy.convert_file_document_to_url_document(document)
return native(document)
def convert_upload_to_url_document(
file_content: bytes, filename: str | None, content_type: str | None
) -> dict[str, str]:
native: Final = _UPLOAD_DOCUMENT.load() if rust_ocr_enabled() else None
if native is None:
from litellm.ocr import legacy
if len(file_content) > _PYTHON_MAX_FILE_BYTES:
raise ValueError("OCR file exceeds the size limit")
content_mime: Final = content_type.split(";")[0].strip() if content_type else None
mime_type: Final = (
legacy.get_mime_type(filename)
if filename and (not content_mime or content_mime == "application/octet-stream")
else content_mime or "application/octet-stream"
)
return legacy.convert_file_document_to_url_document(
{"type": "file", "file": file_content, "mime_type": mime_type}
)
return native(file_content, filename, content_type)

7
litellm/ocr/legacy.py
View file

@ -11,7 +11,7 @@ from collections.abc import Coroutine, Mapping
from dataclasses import dataclass
from io import IOBase
from types import MappingProxyType
from typing import Final, cast # noqa: TID251 # adapters preserve the legacy untyped contracts
from typing import Final, Protocol, cast # noqa: TID251 # adapters preserve the legacy untyped contracts
import httpx
@ -26,7 +26,6 @@ from litellm.llms.base_llm.ocr.transformation import (
parse_ocr_request_format,
)
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.ocr.input import FileReader
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import CustomPricingLiteLLMParams
from litellm.utils import ProviderConfigManager, client
@ -34,6 +33,10 @@ from litellm.utils import ProviderConfigManager, client
base_llm_http_handler: Final = BaseLLMHTTPHandler()
class FileReader(Protocol):
def read(self) -> bytes | str: ...
@dataclass(frozen=True, slots=True)
class _PreparedOCRRequest:
model: str

2
litellm/ocr/main.py
View file

@ -5,7 +5,7 @@ import httpx
from litellm.llms.base_llm.ocr.transformation import OCRResponse
from litellm.ocr import legacy
from litellm.ocr.input import convert_file_document_to_url_document, get_mime_type
from litellm.ocr.legacy import convert_file_document_to_url_document, get_mime_type
from litellm.rust_bridge.bindings import native_exception_types
from litellm.rust_bridge.configuration import rust_ocr_enabled
from litellm.rust_bridge.ocr import LiteLLMOcrRequest

51
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -102,6 +102,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials import (
UpstreamCredentialProvider,
)
from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import (
prepare_mcp_client,
raise_public,
raise_token_exchange_challenge,
raise_user_oauth_challenge,
@ -2804,6 +2805,8 @@ class MCPServerManager:
headers=headers,
server_label=server.name or server.server_name or server.alias or server.server_id,
relays_upstream_auth=server.is_client_forwarded_token,
auth_type=server.auth_type,
upstream_token_header=server.upstream_token_header,
)
tool_func.__name__ = prefixed_tool_name
tool_func.__doc__ = description
@ -4259,15 +4262,20 @@ class MCPServerManager:
user_api_key_auth=user_api_key_auth,
extra_headers=extra_headers,
)
return MCPClient(
server_url=server_url,
transport_type=transport,
auth_type=resolved_server.auth_type,
timeout=(resolved_server.timeout if resolved_server.timeout is not None else MCP_CLIENT_TIMEOUT),
extra_headers=extra_headers,
resolved_auth=resolved_auth,
sampling_callback=sampling_cb,
elicitation_callback=elicitation_cb,
return await prepare_mcp_client(
resolved_server,
MCPClient(
server_url=server_url,
transport_type=transport,
auth_type=resolved_server.auth_type,
timeout=(
resolved_server.timeout if resolved_server.timeout is not None else MCP_CLIENT_TIMEOUT
),
extra_headers=extra_headers,
resolved_auth=resolved_auth,
sampling_callback=sampling_cb,
elicitation_callback=elicitation_cb,
),
)
# Create SigV4 auth if configured
@ -4297,17 +4305,20 @@ class MCPServerManager:
else AuthResolution.no_auth
)
record_auth_resolution(server.server_id, legacy_source)
return MCPClient(
server_url=server_url,
transport_type=transport,
auth_type=resolved_server.auth_type,
auth_value=auth_value,
auth_header_name=auth_header_name,
timeout=(resolved_server.timeout if resolved_server.timeout is not None else MCP_CLIENT_TIMEOUT),
extra_headers=extra_headers,
aws_auth=aws_auth,
sampling_callback=sampling_cb,
elicitation_callback=elicitation_cb,
return await prepare_mcp_client(
resolved_server,
MCPClient(
server_url=server_url,
transport_type=transport,
auth_type=resolved_server.auth_type,
auth_value=auth_value,
auth_header_name=auth_header_name,
timeout=(resolved_server.timeout if resolved_server.timeout is not None else MCP_CLIENT_TIMEOUT),
extra_headers=extra_headers,
aws_auth=aws_auth,
sampling_callback=sampling_cb,
elicitation_callback=elicitation_cb,
),
)
async def _get_tools_from_server(

16
litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py
View file

@ -54,7 +54,7 @@ from litellm.llms.custom_httpx.http_handler import (
from litellm.proxy._experimental.mcp_server.tool_registry import (
global_mcp_tool_registry,
)
from litellm.types.mcp import credential_redirect_hook, custom_credential_slot
from litellm.types.mcp import MCPAuthType, credential_redirect_hook, custom_credential_slot
class _OpenAPIJSONSchema(TypedDict, total=False):
@ -471,6 +471,8 @@ def create_tool_function(
headers: dict[str, str] | None = None,
server_label: str | None = None,
relays_upstream_auth: bool = False,
auth_type: MCPAuthType = None,
upstream_token_header: str | None = None,
):
"""Create a tool function for an OpenAPI operation.
@ -503,6 +505,18 @@ def create_tool_function(
by using **kwargs instead of named parameters.
"""
effective_headers: Final = _merge_openapi_tool_request_headers(headers)
if auth_type is not None:
from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import (
raise_public,
validate_static_credential,
)
from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error, Ok
match validate_static_credential(auth_type, effective_headers, upstream_token_header):
case Error(error):
raise_public(error)
case Ok():
pass
# Build URL from base_url and path
url = base_url + path

96
litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py
View file

@ -13,15 +13,17 @@ from __future__ import annotations
import base64
import os
from collections.abc import Mapping
from typing import TYPE_CHECKING, Final, Literal, NoReturn
from fastapi import HTTPException
from pydantic import SecretStr
from typing_extensions import assert_never
from litellm.experimental_mcp_client.client import strip_auth_scheme, to_basic_credentials
from litellm.experimental_mcp_client.client import MCPClient, strip_auth_scheme, to_basic_credentials
from litellm.proxy._experimental.mcp_server.exceptions import MCPServerURLCredentialsError
from litellm.proxy._experimental.mcp_server.oauth_utils import resolve_upstream_resource
from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error, Ok, Result
from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
DEFAULT_CREDENTIAL_HEADER,
ApiKeyConfig,
@ -39,7 +41,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
Subject,
TokenExchangeConfig,
)
from litellm.types.mcp import DEFAULT_SUBJECT_TOKEN_TYPE, MCPAuth
from litellm.types.mcp import DEFAULT_SUBJECT_TOKEN_TYPE, MCPAuth, MCPAuthType, MCPTransport
if TYPE_CHECKING:
from litellm.proxy._types import UserAPIKeyAuth
@ -79,7 +81,7 @@ def to_server_spec(server: MCPServer) -> ServerSpec | None:
BYOK is the per-user source of the ``api_key`` mode; its scheme rides on ``auth_type`` just
like a shared key, but the value is per-user and not migrated yet, so a BYOK server defers
to v1 regardless of ``auth_type`` (this guard is the seam the BYOK arm replaces later).
to v1 for its static schemes. Declared OBO always stays with the exchange arm.
Dispatches on the declared ``auth_type``. The match is exhaustive over ``MCPAuthType`` with
an ``assert_never`` tail, so a newly added auth mode fails the type gate here until it is
@ -90,8 +92,8 @@ def to_server_spec(server: MCPServer) -> ServerSpec | None:
modes ``true_passthrough`` / ``oauth_delegate`` (``PassthroughConfig``); delegated/passthrough
oauth2 and SigV4 return None and stay on v1.
"""
if server.is_byok:
return None # per-user BYOK source not migrated yet -> defer to v1 (any auth_type)
if server.is_byok and server.auth_type != MCPAuth.oauth2_token_exchange:
return None # per-user BYOK source not migrated yet -> defer to v1
resource: Final = server.url or server.server_id
auth_type: Final = server.auth_type
match auth_type:
@ -165,21 +167,9 @@ def _client_credentials_spec(server: MCPServer, resource: str) -> ServerSpec:
)
def _token_exchange_spec(server: MCPServer, resource: str) -> ServerSpec | None:
"""Build a token_exchange (OBO) spec, or defer (None) when it is not OBO-configured.
An OBO server with ``client_id``/``client_secret`` is owned by the v2 arm even if the
``token_exchange_endpoint``/``token_url`` is absent: a missing endpoint then fails closed (412) at
the exchanger rather than silently deferring to v1 and connecting unauthenticated, since the
gateway must not guess the IdP or fall back to a weaker source. Without client credentials there is
nothing to own, so the server stays on v1 (parity-safe). ``profile`` selects the wire dialect
(``rfc8693`` default, ``entra_obo`` for Microsoft Entra On-Behalf-Of); an unrecognized value
normalizes to ``rfc8693`` so a bad config value cannot crash spec-building. ``audience`` is
forwarded only when the operator set it; a missing one is omitted, not derived.
"""
def _token_exchange_spec(server: MCPServer, resource: str) -> ServerSpec:
"""Keep declared OBO owned by the resolver, including incomplete client configuration."""
endpoint: Final = server.token_exchange_endpoint or server.effective_token_url
if not server.client_id or not server.client_secret:
return None
profile: Final[Literal["rfc8693", "entra_obo"]] = (
"entra_obo" if server.token_exchange_profile == "entra_obo" else "rfc8693"
)
@ -193,7 +183,7 @@ def _token_exchange_spec(server: MCPServer, resource: str) -> ServerSpec | None:
token_exchange_endpoint=endpoint,
audience=server.audience,
client_id=server.client_id,
client_secret=SecretStr(server.client_secret),
client_secret=SecretStr(server.client_secret) if server.client_secret else None,
token_endpoint_auth_method=server.token_endpoint_auth_method,
scopes=tuple(server.scopes or ()),
),
@ -397,3 +387,69 @@ def raise_token_exchange_challenge(
detail="Unauthorized",
headers={"WWW-Authenticate": www_authenticate},
)
_STATIC_MODES: Final = frozenset(
(MCPAuth.api_key, MCPAuth.bearer_token, MCPAuth.basic, MCPAuth.token, MCPAuth.authorization)
)
def _usable_credential_value(auth_type: MCPAuthType, name: str, value: str) -> bool:
if not value:
return False
if auth_type == MCPAuth.api_key and name != "authorization":
return True
if value.lower() in ("bearer", "basic", "token", "apikey"):
return False
if auth_type == MCPAuth.api_key:
api_scheme: Final = value.split(None, 1)[0]
if api_scheme.lower() in ("bearer", "token", "apikey"):
api_credential: Final = strip_auth_scheme(value, api_scheme).strip()
return api_credential.lower() != api_scheme.lower()
if auth_type in (MCPAuth.bearer_token, MCPAuth.token):
scheme: Final = "Bearer" if auth_type == MCPAuth.bearer_token else "token"
credential: Final = strip_auth_scheme(value, scheme).strip()
return bool(credential) and credential.lower() != scheme.lower()
if auth_type == MCPAuth.basic:
parts: Final = value.split(None, 1)
if len(parts) != 2 or parts[0].lower() != "basic":
return False
try:
decoded: Final = base64.b64decode(parts[1], validate=True).strip()
return b":" in decoded
except ValueError:
return False
return True
def validate_static_credential(
auth_type: MCPAuthType,
headers: Mapping[str, str],
upstream_token_header: str | None = None,
) -> Result[None, CredError]:
if auth_type not in _STATIC_MODES:
return Ok(None)
default_slot: Final = "X-API-Key" if auth_type == MCPAuth.api_key else "Authorization"
slots: Final = frozenset(
name.lower()
for name in (
upstream_token_header or default_slot,
default_slot,
"Authorization",
)
)
values: Final = tuple((name.lower(), value.strip()) for name, value in headers.items() if name.lower() in slots)
if any(_usable_credential_value(auth_type, name, value) for name, value in values):
return Ok(None)
return Error(CredError.of_misconfigured(f"{auth_type} requires a usable upstream credential"))
async def prepare_mcp_client(server: MCPServer, client: MCPClient) -> MCPClient:
if server.auth_type not in _STATIC_MODES or client.transport_type == MCPTransport.stdio:
return client
request: Final = await client.prepare_request_auth()
match validate_static_credential(server.auth_type, request.headers, server.upstream_token_header):
case Error(error):
raise_public(error)
case Ok():
return client

17
litellm/proxy/_types.py
View file

@ -2004,6 +2004,13 @@ RouterSettingsDict = Annotated[
class NewTeamRequest(TeamBase):
router_settings: RouterSettingsDict | None = None
model_aliases: dict | None = None
model_max_budget: GenericBudgetConfigType | None = Field(
default=None,
description=(
"Max budget per model for every key on the team, overridable per key "
"(e.g. {'gpt-4o': {'max_budget': 10, 'budget_duration': '1d'}})"
),
)
tags: list | None = None
guardrails: list[str] | None = None
policies: list[str] | None = None
@ -2105,6 +2112,13 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase):
access_group_ids: list[str] | None = None
budget_limits: list[BudgetLimitEntry] | None = None # multiple concurrent budget windows
default_team_member_models: list[str] | None = None # default allowed_models seeded onto new team members
model_max_budget: GenericBudgetConfigType | None = Field(
default=None,
description=(
"Max budget per model for every key on the team, overridable per key "
"(e.g. {'gpt-4o': {'max_budget': 10, 'budget_duration': '1d'}})"
),
)
class PatchTeamRequest(UpdateTeamRequest):
@ -3032,6 +3046,7 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken):
team_tpd_limit: int | None = None
team_max_budget: float | None = None
team_soft_budget: float | None = None
team_model_max_budget: dict[str, object] | None = None
team_models: list = []
team_blocked: bool = False
soft_budget: float | None = None
@ -3710,6 +3725,7 @@ class AllCallbacks(LiteLLMPydanticObjectBase):
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
"AWS_REGION_NAME",
"S3_LOG_PROMPTS_ONLY",
],
)
@ -4462,6 +4478,7 @@ class TeamInfoResponseObjectTeamTable(LiteLLM_TeamTable):
# Parent org's model ceiling, reported only to callers who can manage the team.
# None = no org or not a manager; [] or ["all-proxy-models"] = no ceiling.
organization_models: list[str] | None = None
model_max_budget_usage: Mapping[str, Mapping[str, object]] | None = None
class TeamInfoResponseObject(TypedDict):

15
litellm/proxy/anthropic_endpoints/endpoints.py
View file

@ -8,7 +8,6 @@ from fastapi import APIRouter, Depends, HTTPException, Request, Response
from fastapi.responses import JSONResponse
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.anthropic_interface.exceptions import AnthropicErrorResponse, AnthropicExceptionMapping
from litellm.integrations.custom_guardrail import ModifyResponseException
from litellm.llms.anthropic.experimental_pass_through.context_management import (
@ -22,13 +21,16 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_request_processing import (
ProxyBaseLLMRequestProcessing,
create_response,
log_llm_api_exception,
proxy_exception_from_http_exception,
resolve_litellm_call_id,
)
from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
from litellm.proxy.common_utils.openai_error_payload import (
error_status_code,
openai_error_param,
openai_error_type,
with_litellm_call_id,
)
from litellm.types.utils import TokenCountResponse
@ -218,10 +220,12 @@ async def anthropic_response(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=base_llm_response_processor.data
)
verbose_proxy_logger.exception("litellm.proxy.proxy_server.anthropic_response(): Exception occured - %s", e)
log_llm_api_exception(e, base_llm_response_processor.litellm_call_id)
if isinstance(e, ProxyException):
return _anthropic_error_json_response(e, request)
return _anthropic_error_json_response(
with_litellm_call_id(e, base_llm_response_processor.litellm_call_id), request
)
# Extract model_id from request metadata (same as success path)
litellm_metadata: Final = data.get("litellm_metadata", {}) or {}
@ -231,7 +235,7 @@ async def anthropic_response(
# Get headers
headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=user_api_key_dict,
call_id=data.get("litellm_call_id", ""),
call_id=base_llm_response_processor.litellm_call_id,
model_id=model_id,
version=version,
response_cost=0,
@ -288,6 +292,7 @@ async def count_tokens(
"""
from litellm.proxy.proxy_server import token_counter as internal_token_counter
litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id"))
try:
request_data: Final = await _read_request_body(request=request)
data: Final[dict] = {**request_data}
@ -339,7 +344,7 @@ async def count_tokens(
detail=detail,
)
except Exception as e:
verbose_proxy_logger.exception("litellm.proxy.anthropic_endpoints.count_tokens(): Exception occurred - %s", e)
log_llm_api_exception(e, litellm_call_id)
raise HTTPException(status_code=500, detail={"error": f"Internal server error: {e}"})

166
litellm/proxy/auth/fallback_budget.py Normal file
View file

@ -0,0 +1,166 @@
"""
Enforce the caller's budget against router fallback targets.
Budget is checked once, during auth, against the *requested* model group. A zero-cost group takes
`_is_model_cost_zero`'s bypass and waives every budget check; the router then picks a fallback
target after auth, inside `run_async_fallback`, and nothing re-checks budget on the group that
actually bills. So a free model with a paid fallback spends without a gate.
This predicate is injected into the router to re-check budget for each fallback target before it is
attempted, mirroring `fallback_model_access.py`. It deliberately leaves the primary attempt alone:
a zero-cost model is never blocked by budget, and only the paid fallback is refused. On by default;
set `general_settings.enforce_fallback_budget: false` to restore the unguarded behaviour.
Scope: the key's and the user's `max_budget`. Not covered yet, and each needs a read-only evaluation
path before it can be: team, team-member, end-user, org, global and per-model budgets, whose
auth-path functions enforce rather than report (they raise), so reusing them would fire threshold
alerts and take spend reservations for a target that is then skipped; and the key's rolling
`budget_limits` windows, whose accumulated spend lives only in per-window counters
(`spend:key:{token}:window:{budget_duration}`), so enforcing them means more counter reads on the
fallback path rather than reusing state auth already loaded.
Two known limitations of that narrow scope, both shared with `fallback_model_access.py`:
* This reads the spend counter, it does not reserve against it. Requests already in flight all
observe the same pre-billing figure, so a cap can be crossed by roughly the number of concurrent
fallbacks times their cost. Auth-time enforcement avoids this by pre-filling the counter through
`reserve_budget_for_request`, which the zero-cost bypass skips. Turning the soft cap into a hard
one means reserving per fallback attempt and reconciling on completion.
* A request that reaches the router without `metadata["user_api_key_auth"]` is not restricted.
Only `add_litellm_data_to_request` populates that key, so endpoints that assemble metadata by
hand (for example `/queue/chat/completions`) fall through as unauthenticated.
"""
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from typing import Final
from pydantic import BaseModel, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.auth_checks import (
_is_model_cost_zero, # pyright: ignore[reportPrivateUsage] # the zero-cost predicate the auth-time budget checks use; no public equivalent
)
from litellm.router import Router
class _RequestMetadata(BaseModel):
user_api_key_auth: UserAPIKeyAuth | None = None
class _FallbackBudgetSettings(BaseModel):
enforce_fallback_budget: bool = True
def _token_in_metadata(metadata: object) -> UserAPIKeyAuth | None:
try:
return _RequestMetadata.model_validate(metadata).user_api_key_auth
except ValidationError:
return None
def _user_api_key_auth_from_request(request_kwargs: Mapping[str, object]) -> UserAPIKeyAuth | None:
return next(
(
token
for field in ("metadata", "litellm_metadata")
if (token := _token_in_metadata(request_kwargs.get(field))) is not None
),
None,
)
def _enforced_by_general_settings() -> bool:
from litellm.proxy.proxy_server import general_settings
return _FallbackBudgetSettings.model_validate(general_settings).enforce_fallback_budget
def _applies_user_budget_to_team_keys() -> bool:
from litellm.proxy.proxy_server import general_settings
return general_settings.get("apply_user_budget_to_team_keys") is True
async def _counter_spend(counter_key: str, fallback_spend: float, max_budget: float) -> float:
"""
Read a spend counter the same way the auth-time budget checks do.
`max_budget` is not advisory: it makes `get_current_spend` re-check the counter against the
authoritative recorded spend before admitting. A counter restored from an older Redis snapshot
reads as a hit rather than a clean miss, so without this the reseed path never runs and a
stale-low counter would keep admitting paid fallbacks past the cap.
"""
from litellm.proxy.proxy_server import get_current_spend
return await get_current_spend(
counter_key=counter_key,
fallback_spend=fallback_spend,
max_budget=max_budget,
)
async def is_token_within_budget_for_model(*, model: str, valid_token: UserAPIKeyAuth, llm_router: Router) -> bool:
"""
True when the key and the user behind it can still pay for `model`.
A zero-cost fallback target is always allowed: refusing it would deny a request on spend some
other model accrued, which is the same reasoning behind the auth-time bypass.
"""
if _is_model_cost_zero(model=model, llm_router=llm_router):
return True
key_budget: Final = valid_token.max_budget
if key_budget is not None and valid_token.token is not None:
key_spend: Final = await _counter_spend(
counter_key=f"spend:key:{valid_token.token}",
fallback_spend=valid_token.spend or 0.0,
max_budget=key_budget,
)
if key_spend >= key_budget:
return False
# Mirrors `_PROXY_MaxBudgetLimiter`: a team key does not carry the key owner's personal budget
# unless the proxy opts in, so the personal cap must not gate the fallback either.
user_budget: Final = valid_token.user_max_budget
if (
user_budget is not None
and valid_token.user_id is not None
and (valid_token.team_id is None or _applies_user_budget_to_team_keys())
):
user_spend: Final = await _counter_spend(
counter_key=f"spend:user:{valid_token.user_id}",
fallback_spend=valid_token.user_spend or 0.0,
max_budget=user_budget,
)
if user_spend >= user_budget:
return False
return True
@dataclass(frozen=True, slots=True)
class RouterFallbackBudgetCheck:
"""
`FallbackBudgetCheck` for the proxy's router: while `is_enforced()` is true, a paid fallback
target is attempted only when the caller is still within budget. Requests that carry no key
(for example internal health checks) are not restricted.
"""
is_enforced: Callable[[], bool]
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: Router) -> bool:
if not self.is_enforced():
return True
valid_token: Final = _user_api_key_auth_from_request(request_kwargs)
if valid_token is None:
return True
try:
return await is_token_within_budget_for_model(model=model, valid_token=valid_token, llm_router=llm_router)
except Exception as e: # noqa: BLE001 # fail closed: a spend lookup failure must not bill the caller
verbose_proxy_logger.warning("Skipping fallback to model=%s: budget lookup failed: %s", model, e)
return False
router_fallback_budget_check: Final = RouterFallbackBudgetCheck(is_enforced=_enforced_by_general_settings)

2
litellm/proxy/auth/team_grants.py
View file

@ -59,6 +59,7 @@ class TeamGrants(TypedDict, total=False):
team_tpd_limit: ReadOnly[int | None]
team_max_budget: ReadOnly[float | None]
team_soft_budget: ReadOnly[float | None]
team_model_max_budget: ReadOnly[dict[str, object] | None]
team_spend: ReadOnly[float | None]
team_models: ReadOnly[Sequence[str]]
team_blocked: ReadOnly[bool]
@ -101,6 +102,7 @@ def team_grants(
team_tpd_limit=team_object.tpd_limit,
team_max_budget=team_object.max_budget,
team_soft_budget=team_object.soft_budget,
team_model_max_budget=team_object.model_max_budget,
team_spend=team_object.spend,
team_models=tuple(team_object.models),
team_blocked=team_object.blocked,

47
litellm/proxy/auth/user_api_key_auth.py
View file

@ -304,6 +304,16 @@ class _UserModelBudgetLimiter(Protocol):
) -> bool: ...
class _TeamModelBudgetLimiter(Protocol):
async def is_team_within_model_budget(
self,
team_id: str,
team_model_max_budget: Mapping[str, object],
key_model_max_budget: Mapping[str, object] | None,
model: str,
) -> bool: ...
class _TokenTeamModels(Protocol):
@property
def team_models(self) -> list[str]: ...
@ -374,6 +384,25 @@ async def _check_user_model_budget(
)
async def _check_team_model_budget(
valid_token: UserAPIKeyAuth,
model_max_budget_limiter: _TeamModelBudgetLimiter,
models: list[str],
) -> None:
"""Enforce the team's `model_max_budget` for every requested model the key does not override."""
team_model_max_budget: Final = valid_token.team_model_max_budget
if valid_token.team_id is None or not team_model_max_budget:
return
key_model_max_budget: Final[Mapping[str, object] | None] = valid_token.model_max_budget
for model_name in models:
await model_max_budget_limiter.is_team_within_model_budget(
team_id=valid_token.team_id,
team_model_max_budget=team_model_max_budget,
key_model_max_budget=key_model_max_budget,
model=model_name,
)
async def _check_key_model_budget_with_fallback(
valid_token: UserAPIKeyAuth,
model_max_budget_limiter: _KeyModelBudgetLimiter,
@ -2376,6 +2405,7 @@ async def _user_api_key_auth_builder(
team_id=valid_token.team_id,
max_budget=valid_token.team_max_budget,
soft_budget=valid_token.team_soft_budget,
model_max_budget=valid_token.team_model_max_budget,
spend=valid_token.team_spend,
tpm_limit=valid_token.team_tpm_limit,
rpm_limit=valid_token.team_rpm_limit,
@ -2530,6 +2560,7 @@ def _team_obj_from_token(valid_token: UserAPIKeyAuth) -> LiteLLM_TeamTableCached
team_id=valid_token.team_id,
max_budget=valid_token.team_max_budget,
soft_budget=valid_token.team_soft_budget,
model_max_budget=valid_token.team_model_max_budget,
spend=valid_token.team_spend,
tpm_limit=valid_token.team_tpm_limit,
rpm_limit=valid_token.team_rpm_limit,
@ -2606,6 +2637,7 @@ async def _run_centralized_common_checks(
litellm_proxy_admin_name,
llm_router,
master_key,
model_max_budget_limiter,
prisma_client,
proxy_logging_obj,
user_api_key_cache,
@ -2874,6 +2906,21 @@ async def _run_centralized_common_checks(
finally:
release_spend_counter_batch()
if not skip_budget_checks:
await _check_team_model_budget(
valid_token=user_api_key_auth_obj,
model_max_budget_limiter=model_max_budget_limiter,
models=_get_model_names_for_budget_checks(
model=_get_model_from_request_context(
request_data=request_data,
route=route,
request=request,
llm_router=llm_router,
team_id=user_api_key_auth_obj.team_id,
)
),
)
await _reserve_budget_after_common_checks(
user_api_key_auth_obj=user_api_key_auth_obj,
request=request,

30
litellm/proxy/batches_endpoints/endpoints.py
View file

@ -7,6 +7,7 @@
import asyncio
import os
from collections.abc import Mapping
from types import MappingProxyType
from typing import Any, Final, cast
from fastapi import APIRouter, Depends, HTTPException, Path, Request, Response
@ -17,7 +18,11 @@ from litellm.batches.main import CancelBatchRequest, RetrieveBatchRequest
from litellm.proxy._types import *
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.batches_endpoints.common_utils import validate_batch_list_limit
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.proxy.common_request_processing import (
ProxyBaseLLMRequestProcessing,
log_llm_api_exception,
request_litellm_call_id,
)
from litellm.proxy.common_utils.callback_utils import sanitize_openai_provider_metadata
from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
from litellm.proxy.common_utils.openai_endpoint_utils import (
@ -383,8 +388,9 @@ async def create_batch(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
verbose_proxy_logger.exception("litellm.proxy.proxy_server.create_batch(): Exception occured - %s", e)
raise handle_exception_on_proxy(e)
litellm_call_id: Final = request_litellm_call_id(data)
log_llm_api_exception(e, litellm_call_id)
raise handle_exception_on_proxy(e, litellm_call_id)
@router.get(
@ -674,8 +680,9 @@ async def retrieve_batch(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
verbose_proxy_logger.exception("litellm.proxy.proxy_server.retrieve_batch(): Exception occured - %s", e)
raise handle_exception_on_proxy(e)
litellm_call_id: Final = request_litellm_call_id(data)
log_llm_api_exception(e, litellm_call_id)
raise handle_exception_on_proxy(e, litellm_call_id)
@router.get(
@ -725,6 +732,7 @@ async def list_batches(
)
verbose_proxy_logger.debug("GET /v1/batches after=%s limit=%s", after, limit)
data: Mapping[str, object] = MappingProxyType({})
try:
if llm_router is None:
raise HTTPException(
@ -854,10 +862,11 @@ async def list_batches(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data={"after": after, "limit": limit},
request_data={**data, "after": after, "limit": limit},
)
verbose_proxy_logger.error("litellm.proxy.proxy_server.retrieve_batch(): Exception occured - %s", e)
raise handle_exception_on_proxy(e)
litellm_call_id: Final = request_litellm_call_id(data)
log_llm_api_exception(e, litellm_call_id)
raise handle_exception_on_proxy(e, litellm_call_id)
@router.post(
@ -1079,8 +1088,9 @@ async def cancel_batch(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
verbose_proxy_logger.exception("litellm.proxy.proxy_server.create_batch(): Exception occured - %s", e)
raise handle_exception_on_proxy(e)
litellm_call_id: Final = request_litellm_call_id(data)
log_llm_api_exception(e, litellm_call_id)
raise handle_exception_on_proxy(e, litellm_call_id)
######################################################################

103
litellm/proxy/common_request_processing.py
View file

@ -7,7 +7,18 @@ from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequen
from datetime import datetime
from functools import lru_cache
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, Protocol, TypeAlias, TypeVar, overload
from typing import (
TYPE_CHECKING,
Any,
Final,
Literal,
NamedTuple,
Protocol,
TypeAlias,
TypeVar,
overload,
runtime_checkable,
)
import anyio
import httpx
@ -34,7 +45,11 @@ from litellm.constants import (
UNSAFE_PROXY_RESPONSE_HEADERS,
)
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket, is_expected_client_error
from litellm.litellm_core_utils.core_helpers import (
get_or_create_metadata_bucket,
independent_snapshot,
is_expected_client_error,
)
from litellm.litellm_core_utils.dd_tracing import NullTracer, tracer
from litellm.litellm_core_utils.get_supported_openai_params import (
get_supported_openai_params,
@ -1452,7 +1467,19 @@ def _has_attribute_error_in_chain(exc: Exception) -> bool:
_CLIENT_DISCONNECT_DETAIL: Final = "Client disconnected the request"
def _log_llm_api_exception(e: Exception, litellm_call_id: str | None) -> None:
@runtime_checkable
class _CarriesLitellmCallId(Protocol):
litellm_call_id: str | None
def request_litellm_call_id(data: Mapping[str, object]) -> str | None:
logging_obj: Final = data.get("litellm_logging_obj")
logged_id: Final = logging_obj.litellm_call_id if isinstance(logging_obj, _CarriesLitellmCallId) else None
call_id: Final = logged_id or data.get("litellm_call_id")
return call_id if isinstance(call_id, str) else None
def log_llm_api_exception(e: Exception, litellm_call_id: str | None) -> None:
if getattr(e, "status_code", None) == 499 and getattr(e, "detail", None) == _CLIENT_DISCONNECT_DETAIL:
verbose_proxy_logger.info(
"litellm.proxy.proxy_server._handle_llm_api_exception(): client disconnected, "
@ -1532,6 +1559,10 @@ class ProxyBaseLLMRequestProcessing:
def __init__(self, data: dict):
self.data = data
@property
def litellm_call_id(self) -> str | None:
return request_litellm_call_id(self.data)
@staticmethod
def _merge_passthrough_streaming_headers(
response_headers: httpx.Headers | dict | None,
@ -2062,6 +2093,13 @@ class ProxyBaseLLMRequestProcessing:
) -> tuple[dict, LiteLLMLoggingObj]:
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
configured_fallbacks: Final = (
self._configured_fallbacks(llm_router=llm_router, user_api_key_dict=user_api_key_dict)
if llm_router is not None and not self.data.get("disable_fallbacks")
else None
)
pristine: Final = independent_snapshot(self.data) if configured_fallbacks else None
try:
return await self.common_processing_pre_call_logic(
request=request,
@ -2080,14 +2118,19 @@ class ProxyBaseLLMRequestProcessing:
llm_router=llm_router,
)
except ProxyRateLimitError as original_exc:
original_model: Final = self.data.get("model")
if not original_model or not llm_router or self.data.get("disable_fallbacks"):
rate_limited_data: Final = self.data
original_model: Final = rate_limited_data.get("model")
if (
pristine is None
or not configured_fallbacks
or rate_limited_data.get("disable_fallbacks")
or not isinstance(original_model, str)
):
raise
fallback_models: Final = self._resolve_fallback_models(
model=original_model,
llm_router=llm_router,
user_api_key_dict=user_api_key_dict,
fallbacks=configured_fallbacks,
)
if not fallback_models:
raise
@ -2102,6 +2145,7 @@ class ProxyBaseLLMRequestProcessing:
for fallback_model in fallback_models:
if fallback_model == original_model:
continue
self.data = independent_snapshot(pristine)
self.data["model"] = fallback_model
try:
return await self.common_processing_pre_call_logic(
@ -2123,39 +2167,30 @@ class ProxyBaseLLMRequestProcessing:
except ProxyRateLimitError:
continue
except BaseException:
self.data["model"] = original_model
self.data = rate_limited_data
raise
self.data["model"] = original_model
self.data = rate_limited_data
raise original_exc
def _resolve_fallback_models(
self,
model: str,
llm_router: Router,
user_api_key_dict: UserAPIKeyAuth,
) -> list | None:
from litellm.router_utils.fallback_event_handlers import get_fallback_model_group
fallbacks = None
@staticmethod
def _configured_fallbacks(llm_router: Router, user_api_key_dict: UserAPIKeyAuth) -> list | None:
key_router_settings: Final = user_api_key_dict.router_settings
if isinstance(key_router_settings, dict) and "fallbacks" in key_router_settings:
fallbacks = key_router_settings["fallbacks"]
key_fallbacks: Final = key_router_settings.get("fallbacks") if isinstance(key_router_settings, dict) else None
fallbacks: Final = key_fallbacks if key_fallbacks is not None else llm_router.fallbacks
return fallbacks if isinstance(fallbacks, list) and fallbacks else None
if fallbacks is None:
fallbacks = llm_router.fallbacks
if not fallbacks:
return None
@staticmethod
def _resolve_fallback_models(model: str, fallbacks: list) -> list | None:
from litellm.router_utils.fallback_event_handlers import get_fallback_model_group
fallback_model_group, generic_fallback_idx = get_fallback_model_group(
fallbacks=fallbacks,
model_group=model,
)
if fallback_model_group is None and generic_fallback_idx is not None:
fallback_model_group = fallbacks[generic_fallback_idx]["*"]
return fallback_model_group
if fallback_model_group is not None:
return fallback_model_group
return fallbacks[generic_fallback_idx]["*"] if generic_fallback_idx is not None else None
@staticmethod
def _get_model_id_from_response(hidden_params: Mapping[str, object], data: Mapping[str, object]) -> str:
@ -3429,11 +3464,7 @@ class ProxyBaseLLMRequestProcessing:
version: str | None = None,
):
"""Raises ProxyException (OpenAI API compatible) if an exception is raised"""
logging_obj: Final[LiteLLMLoggingObj | None] = self.data.get("litellm_logging_obj", None)
_log_llm_api_exception(
e,
(logging_obj.litellm_call_id if logging_obj is not None else None) or self.data.get("litellm_call_id"),
)
log_llm_api_exception(e, self.litellm_call_id)
# Allow callbacks to transform the error response
transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
@ -3463,9 +3494,7 @@ class ProxyBaseLLMRequestProcessing:
custom_headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=user_api_key_dict,
call_id=(
_litellm_logging_obj.litellm_call_id if _litellm_logging_obj else self.data.get("litellm_call_id")
),
call_id=self.litellm_call_id,
model_id=model_id,
version=version,
response_cost=0,

23
litellm/proxy/common_utils/openai_error_payload.py
View file

@ -9,6 +9,9 @@ from typing import Final
from fastapi import status
from litellm.constants import STRINGIFIED_NONE
from litellm.proxy._types import ProxyException
LITELLM_CALL_ID_HEADER: Final = "x-litellm-call-id"
_OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType(
{
@ -52,3 +55,23 @@ def openai_error_param(exc: object) -> str | None:
serializes as JSON ``null``."""
carried: Final = attribute_of(exc, "param")
return carried if isinstance(carried, str) and carried != STRINGIFIED_NONE else None
def litellm_call_id_headers(litellm_call_id: str | None) -> dict[str, str] | None: # mutable-ok: ProxyException.headers
if litellm_call_id is None:
return None
return {LITELLM_CALL_ID_HEADER: litellm_call_id} # mutable-ok: ProxyException mutates its headers dict
def with_litellm_call_id(exc: ProxyException, litellm_call_id: str | None) -> ProxyException:
"""The same error object, answering with ``x-litellm-call-id`` when it was raised without one."""
if litellm_call_id is not None:
exc.headers.setdefault(LITELLM_CALL_ID_HEADER, litellm_call_id)
return exc
def headers_with_litellm_call_id(headers: Mapping[str, str] | None, litellm_call_id: str) -> Mapping[str, str]:
"""``headers`` plus ``x-litellm-call-id``, keeping the value they already carry under that name."""
if headers is None:
return MappingProxyType({LITELLM_CALL_ID_HEADER: litellm_call_id})
return MappingProxyType({LITELLM_CALL_ID_HEADER: litellm_call_id, **headers})

2
litellm/proxy/common_utils/proxy_rate_limit_error.py
View file

@ -11,7 +11,7 @@ exception types:
an upstream LLM provider returns 429.
* :class:`fastapi.HTTPException` (status 429) — raised directly by proxy hooks
such as ``parallel_request_limiter``, ``dynamic_rate_limiter``,
``batch_rate_limiter``, ``max_budget_limiter``, ``max_iterations_limiter``,
``batch_rate_limiter``, ``max_iterations_limiter``,
etc.
* :class:`litellm.llms.base_llm.chat.transformation.BaseLLMException` (status
429) — raised by some provider transports.

1
litellm/proxy/db/create_views.py
View file

@ -78,6 +78,7 @@ async def create_missing_views(db: SupportsRawQueries) -> None:
v.*,
t.spend AS team_spend,
t.max_budget AS team_max_budget,
t.model_max_budget AS team_model_max_budget,
t.tpm_limit AS team_tpm_limit,
t.rpm_limit AS team_rpm_limit,
t.tpd_limit AS team_tpd_limit,

10
litellm/proxy/db/db_spend_update_writer.py
View file

@ -18,6 +18,8 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, overload
from urllib.parse import quote, unquote
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.caching import RedisCache
@ -109,6 +111,10 @@ def _batch_cost_row_to_write(payload: SpendLogsPayload, disable_spend_logs: bool
return MappingProxyType({field: value for field, value in payload.items() if field in _BATCH_COST_CLAIM_FIELDS})
class _SpendIncrement(TypedDict):
increment: ReadOnly[float]
class _SpendBatch(Protocol):
litellm_usertable: BatchTable
litellm_verificationtoken: BatchTable
@ -1615,10 +1621,12 @@ class DBSpendUpdateWriter:
async with transaction.batch_() as batcher:
# Sort by token for consistent lock ordering across pods to prevent deadlocks.
for token, response_cost in sorted(key_list_transactions.items()):
spend_increment: _SpendIncrement = {"increment": response_cost}
batcher.litellm_verificationtoken.update_many( # 'update_many' prevents error from being raised if no row exists
where={"token": token},
data={
"spend": {"increment": response_cost},
"spend": spend_increment,
"total_spend": spend_increment,
"last_active": datetime.now(timezone.utc),
},
)

4
litellm/proxy/hooks/__init__.py
View file

@ -4,7 +4,6 @@ from typing import Final, Literal
from . import *
from .cache_control_check import _PROXY_CacheControlCheck
from .litellm_skills import SkillsInjectionHook
from .max_budget_limiter import _PROXY_MaxBudgetLimiter
from .max_budget_per_session_limiter import _PROXY_MaxBudgetPerSessionHandler
from .max_iterations_limiter import _PROXY_MaxIterationsHandler
from .parallel_request_limiter import _PROXY_MaxParallelRequestsHandler
@ -18,7 +17,6 @@ from .sensitive_data_routing import _PROXY_SensitiveDataRoutingHandler
# transitively through `enterprise.enterprise_hooks` can resolve `PROXY_HOOKS`
# and `get_proxy_hook` from this partially-initialized module without circling.
PROXY_HOOKS: Final = {
"max_budget_limiter": _PROXY_MaxBudgetLimiter,
"parallel_request_limiter": _PROXY_MaxParallelRequestsHandler_v3,
"cache_control_check": _PROXY_CacheControlCheck,
"responses_id_security": ResponsesIDSecurity,
@ -35,7 +33,7 @@ if os.getenv("LEGACY_MULTI_INSTANCE_RATE_LIMITING", "false").lower() == "true":
def get_proxy_hook(
hook_name: Literal["max_budget_limiter", "managed_files", "parallel_request_limiter", "cache_control_check"] | str,
hook_name: Literal["managed_files", "parallel_request_limiter", "cache_control_check"] | str,
):
"""
Factory method to get a proxy hook instance by name

84
litellm/proxy/hooks/max_budget_limiter.py
View file

@ -1,84 +0,0 @@
from typing import Final
from fastapi import HTTPException
from litellm import verbose_logger
from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
from litellm.exceptions import RateLimitType
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
class _PROXY_MaxBudgetLimiter(CustomLogger):
# Class variables or attributes
def __init__(self):
pass
async def async_pre_call_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
cache: DualCache,
data: dict,
call_type: str,
):
try:
verbose_proxy_logger.debug("Inside Max Budget Limiter Pre-Call Hook")
max_budget: Final = user_api_key_dict.user_max_budget
user_id: Final = user_api_key_dict.user_id
if max_budget is None or user_id is None:
return
from litellm.proxy.proxy_server import general_settings
if (
user_api_key_dict.team_id is not None
and general_settings.get("apply_user_budget_to_team_keys") is not True
):
return
# The reservation path admits at the strict-`<` boundary and
# atomically pre-fills the same counter we'd read here. Re-checking
# with `>=` would reject a request the reservation already admitted
# when the reservation fills the counter to exactly max_budget.
# Imported lazily to avoid a circular import via proxy.utils.
from litellm.proxy.spend_tracking.budget_reservation import (
get_reserved_counter_keys,
)
user_counter_key: Final = f"spend:user:{user_id}"
if user_counter_key in get_reserved_counter_keys(user_api_key_dict.budget_reservation):
return
from litellm.proxy.proxy_server import get_current_spend
curr_spend: Final = await get_current_spend(
counter_key=user_counter_key,
fallback_spend=user_api_key_dict.user_spend or 0.0,
)
verbose_proxy_logger.debug(
"MaxBudgetLimiter: user_id=%s, spend=%.6f, max=%.6f",
user_id,
curr_spend,
max_budget,
)
# CHECK IF REQUEST ALLOWED
if curr_spend >= max_budget:
resolved_model, llm_provider = resolve_llm_provider_for_rate_limit(data.get("model") if data else None)
raise ProxyRateLimitError(
detail="Max budget limit reached.",
rate_limit_type=RateLimitType.BUDGET,
model=resolved_model,
llm_provider=llm_provider,
)
except HTTPException as e:
raise e
except Exception as e:
verbose_logger.exception(
"litellm.proxy.hooks.max_budget_limiter.py::async_pre_call_hook(): Exception occured - %s", e
)

58
litellm/proxy/hooks/model_max_budget_limiter.py
View file

@ -19,12 +19,14 @@ from litellm.types.utils import BudgetConfig, StandardLoggingPayload
VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX: Final = "virtual_key_spend"
END_USER_SPEND_CACHE_KEY_PREFIX: Final = "end_user_model_spend"
USER_SPEND_CACHE_KEY_PREFIX: Final = "user_model_spend"
TEAM_SPEND_CACHE_KEY_PREFIX: Final = "team_model_spend"
_SPEND_CACHE_KEY_PREFIXES: Final = MappingProxyType(
{
Litellm_EntityType.KEY: VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX,
Litellm_EntityType.USER: USER_SPEND_CACHE_KEY_PREFIX,
Litellm_EntityType.END_USER: END_USER_SPEND_CACHE_KEY_PREFIX,
Litellm_EntityType.TEAM: TEAM_SPEND_CACHE_KEY_PREFIX,
}
)
@ -37,6 +39,7 @@ _BUDGET_START_TIME_KEY_PREFIXES: Final = MappingProxyType(
Litellm_EntityType.KEY: "virtual_key_budget_start_time",
Litellm_EntityType.USER: "user_model_budget_start_time",
Litellm_EntityType.END_USER: "end_user_budget_start_time",
Litellm_EntityType.TEAM: "team_model_budget_start_time",
}
)
@ -139,6 +142,18 @@ def resolve_model_budget(model: str, model_max_budget: Mapping[str, object]) ->
return None
def team_model_budget_applies(model: str, key_model_max_budget: Mapping[str, object] | None) -> bool:
"""A key entry that spend-gates `model` overrides the team cap: it is then gated on and billed to the key alone."""
if not key_model_max_budget:
return True
resolved: Final = resolve_model_budget(model=model, model_max_budget=key_model_max_budget)
return resolved is None or not _spend_gated(resolved.budget_config)
def _spend_gated(budget_config: BudgetConfig) -> bool:
return budget_config.max_budget is not None and budget_config.max_budget >= 0
def _budget_model_candidates(model: str) -> tuple[str, ...]:
"""Names a budget may be configured under for a request on `model`, most specific first.
@ -346,6 +361,30 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
exceeded_message=f"LiteLLM End User: {end_user_id}, exceeded budget for model={model}",
)
async def is_team_within_model_budget(
self,
team_id: str,
team_model_max_budget: Mapping[str, object],
key_model_max_budget: Mapping[str, object] | None,
model: str,
) -> bool:
"""
Check if the team is within the model budget, unless the key's own
`model_max_budget` overrides it for `model`
Raises:
BudgetExceededError: If the team has exceeded the model budget
"""
if not team_model_budget_applies(model=model, key_model_max_budget=key_model_max_budget):
return True
return await self._is_entity_within_model_budget(
entity_type=Litellm_EntityType.TEAM,
entity_id=team_id,
model_max_budget=team_model_max_budget,
model=model,
exceeded_message=f"LiteLLM Team: {team_id}, exceeded budget for model={model}",
)
async def _is_entity_within_model_budget(
self,
entity_type: Litellm_EntityType,
@ -456,11 +495,26 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
return
response_cost: Final[float] = standard_logging_payload.get("response_cost", 0)
key_model_max_budget: Final = _metadata.get("user_api_key_model_max_budget")
entity_budgets: Final = (
(
Litellm_EntityType.KEY,
payload_metadata.get("user_api_key_hash"),
_metadata.get("user_api_key_model_max_budget"),
key_model_max_budget,
),
(
Litellm_EntityType.TEAM,
payload_metadata.get("user_api_key_team_id"),
(
_metadata.get("user_api_key_team_model_max_budget")
if team_model_budget_applies(
model=model,
key_model_max_budget=(
key_model_max_budget if isinstance(key_model_max_budget, Mapping) else None
),
)
else None
),
),
(
Litellm_EntityType.USER,
@ -478,7 +532,7 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
if not resolved_budgets:
verbose_proxy_logger.debug(
"Not running _PROXY_VirtualKeyModelMaxBudgetLimiter.async_log_success_event: "
"no key, user or end-user model_max_budget covers model=%s",
"no key, team, user or end-user model_max_budget covers model=%s",
model,
)
return

78
litellm/proxy/hooks/parallel_request_limiter_v3.py
View file

@ -531,6 +531,7 @@ class RequestRateLimiterStash:
owner_litellm_call_id: str | None = None
rate_limit_response: RateLimitResponse | None = None
parallel_slot: ParallelSlotAcquisition | None = None
parallel_slot_release_lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False, compare=False)
reserved_tokens: int = 0
reserved_model: str | None = None
reserved_scopes: frozenset[tuple[str, str]] = field(default_factory=frozenset)
@ -1620,6 +1621,20 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
statuses.append(self._gauge_status(gauge, in_flight + 1, "OK"))
return RateLimitResponse(overall_code="OK", statuses=statuses)
async def _release_stashed_parallel_slot(
self,
stash: RequestRateLimiterStash | None,
parent_otel_span: Span | None,
) -> None:
if stash is None:
return
async with stash.parallel_slot_release_lock:
acquisition: Final = stash.parallel_slot
if acquisition is None:
return
await self._release_parallel_request_slots(acquisition, parent_otel_span)
stash.parallel_slot = None # rebind-ok: marks this request's slot as released
async def _release_parallel_request_slots(
self,
acquisition: ParallelSlotAcquisition,
@ -3379,13 +3394,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
parent_otel_span=user_api_key_dict.parent_otel_span,
)
stash.reservation_released = True
acquisition: Final = stash.parallel_slot
if acquisition is not None:
await self._release_parallel_request_slots(
acquisition=acquisition,
parent_otel_span=user_api_key_dict.parent_otel_span,
)
stash.parallel_slot = None
await self._release_stashed_parallel_slot(stash, user_api_key_dict.parent_otel_span)
self._handle_rate_limit_error(
response=io_response,
descriptors=descriptors,
@ -3700,13 +3709,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
if tpm_response["overall_code"] == "OVER_LIMIT":
acquisition: Final = stash.parallel_slot
if acquisition is not None:
await self._release_parallel_request_slots(
acquisition=acquisition,
parent_otel_span=user_api_key_dict.parent_otel_span,
)
stash.parallel_slot = None
await self._release_stashed_parallel_slot(stash, user_api_key_dict.parent_otel_span)
self._handle_rate_limit_error(
response=tpm_response,
descriptors=descriptors,
@ -4524,13 +4527,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
verbose_proxy_logger.debug("INSIDE parallel request limiter ASYNC SUCCESS LOGGING")
stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs))
acquisition: Final = stash.parallel_slot if stash is not None else None
if stash is not None and acquisition is not None:
await self._release_parallel_request_slots(
acquisition=acquisition,
parent_otel_span=litellm_parent_otel_span,
)
stash.parallel_slot = None
await self._release_stashed_parallel_slot(stash, litellm_parent_otel_span)
pipeline_operations: Final = self._build_success_event_pipeline_operations(
kwargs=kwargs,
@ -4650,13 +4647,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
pipeline_operations: Final[list[RedisPipelineIncrementOperation]] = []
stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs))
acquisition: Final = stash.parallel_slot if stash is not None else None
if stash is not None and acquisition is not None:
await self._release_parallel_request_slots(
acquisition=acquisition,
parent_otel_span=litellm_parent_otel_span,
)
stash.parallel_slot = None
await self._release_stashed_parallel_slot(stash, litellm_parent_otel_span)
# Skip the reservation refund if async_post_call_failure_hook
# already released it (proxy-level rejection that also bubbles up
@ -4764,23 +4755,23 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
object's current max_parallel_requests configuration, which can
change mid-request) decides whether there is anything to release.
"""
stash: Final = get_request_stash()
if stash is None or stash.parallel_slot is None:
return
await self._release_parallel_request_slots(
acquisition=stash.parallel_slot,
parent_otel_span=None,
)
stash.parallel_slot = None
await self._release_stashed_parallel_slot(get_request_stash(), None)
async def async_post_call_success_hook(self, data: dict, user_api_key_dict: UserAPIKeyAuth, response):
"""
Post-call hook to update rate limit headers in the response.
Release completed-request slots and update rate limit headers in the response.
"""
try:
stash: Final = get_request_stash()
litellm_proxy_rate_limit_response: Final = stash.rate_limit_response if stash is not None else None
slot_stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(data))
await self._release_stashed_parallel_slot(slot_stash, user_api_key_dict.parent_otel_span)
except Exception as e:
verbose_proxy_logger.exception("Error releasing parallel request slot in post-call hook: %s", e)
try:
header_stash: Final = get_request_stash()
litellm_proxy_rate_limit_response: Final = (
header_stash.rate_limit_response if header_stash is not None else None
)
if litellm_proxy_rate_limit_response is not None and response_has_hidden_params(response):
additional_headers: Final = ensure_response_additional_headers(response)
@ -4848,12 +4839,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
stash: Final = get_request_stash()
if stash is None:
return
if stash.parallel_slot is not None:
await self._release_parallel_request_slots(
acquisition=stash.parallel_slot,
parent_otel_span=user_api_key_dict.parent_otel_span,
)
stash.parallel_slot = None
await self._release_stashed_parallel_slot(stash, user_api_key_dict.parent_otel_span)
if stash.batch_enqueued_reservation is not None:
await self.batch_enqueued_token_store.refund(

27
litellm/proxy/image_endpoints/endpoints.py
View file

@ -1,6 +1,5 @@
import asyncio
import io
import traceback
from collections.abc import Sequence
from typing import Final, get_type_hints
@ -9,19 +8,23 @@ from fastapi import APIRouter, Depends, File, HTTPException, Request, Response,
from fastapi.responses import ORJSONResponse
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
)
from litellm.proxy._types import *
from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.proxy.common_request_processing import (
ProxyBaseLLMRequestProcessing,
log_llm_api_exception,
resolve_litellm_call_id,
)
from litellm.proxy.common_utils.http_parsing_utils import (
coerce_numeric_form_fields,
numeric_form_fields,
)
from litellm.proxy.common_utils.openai_error_payload import (
error_status_code,
litellm_call_id_headers,
openai_error_param,
openai_error_type,
)
@ -91,11 +94,12 @@ async def image_generation(
version,
)
data = {}
litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id"))
data = {"litellm_call_id": litellm_call_id}
try:
# Use orjson to parse JSON data, orjson speeds up requests significantly
body: Final = await request.body()
data = orjson.loads(body)
data = orjson.loads(body) | data
# Include original request and headers in the data
data = await add_litellm_data_to_request(
@ -153,9 +157,7 @@ async def image_generation(
response = await llm_call
### ALERTING ###
asyncio.create_task(
proxy_logging_obj.update_request_status(litellm_call_id=data.get("litellm_call_id", ""), status="success")
)
asyncio.create_task(proxy_logging_obj.update_request_status(litellm_call_id=litellm_call_id, status="success"))
### CALL HOOKS ### - modify outgoing data (guardrails, otel, etc.)
response = await proxy_logging_obj.post_call_success_hook(
@ -168,7 +170,7 @@ async def image_generation(
cache_key: Final = hidden_params.get("cache_key", None) or ""
api_base: Final = hidden_params.get("api_base", None) or ""
response_cost: Final = hidden_params.get("response_cost", None) or ""
litellm_call_id: Final = hidden_params.get("litellm_call_id", None) or ""
response_call_id: Final = hidden_params.get("litellm_call_id", None) or ""
fastapi_response.headers.update(
ProxyBaseLLMRequestProcessing.get_custom_headers(
@ -179,7 +181,7 @@ async def image_generation(
version=version,
response_cost=response_cost,
model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
call_id=litellm_call_id,
call_id=response_call_id,
request_data=data,
hidden_params=hidden_params,
)
@ -200,13 +202,13 @@ async def image_generation(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
verbose_proxy_logger.error("litellm.proxy.proxy_server.image_generation(): Exception occured - %s", e)
verbose_proxy_logger.debug(traceback.format_exc())
log_llm_api_exception(e, litellm_call_id)
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "message", str(e)),
type=openai_error_type(e, error_status_code(e, status.HTTP_400_BAD_REQUEST)),
param=openai_error_param(e),
headers=litellm_call_id_headers(litellm_call_id),
code=error_status_code(e, status.HTTP_400_BAD_REQUEST),
)
else:
@ -215,6 +217,7 @@ async def image_generation(
message=getattr(e, "message", error_msg),
type=openai_error_type(e, error_status_code(e, 500)),
param=openai_error_param(e),
headers=litellm_call_id_headers(litellm_call_id),
openai_code=getattr(e, "code", None),
code=error_status_code(e, 500),
)

1
litellm/proxy/litellm_pre_call_utils.py
View file

@ -2334,6 +2334,7 @@ async def add_litellm_data_to_request(
# Team spend, budget - used by prometheus.py
data[_metadata_variable_name]["user_api_key_team_max_budget"] = user_api_key_dict.team_max_budget
data[_metadata_variable_name]["user_api_key_team_spend"] = user_api_key_dict.team_spend
data[_metadata_variable_name]["user_api_key_team_model_max_budget"] = user_api_key_dict.team_model_max_budget
data[_metadata_variable_name]["user_api_key_request_route"] = user_api_key_dict.request_route
# API Key spend, budget - used by prometheus.py

51
litellm/proxy/management_endpoints/common_utils.py
View file

@ -55,6 +55,7 @@ def validate_budget_duration(budget_duration: str | None, status_code: int = 400
from litellm._logging import verbose_proxy_logger
from litellm.caching import DualCache
from litellm.proxy._types import (
CommonProxyErrors,
KeyRequestBase,
LiteLLM_ManagementEndpoint_MetadataFields,
LiteLLM_ManagementEndpoint_MetadataFields_Premium,
@ -73,12 +74,62 @@ from litellm.proxy._types import ( # noqa: F401 re-exported
from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
from litellm.proxy.utils import _premium_user_check
from litellm.repositories.team_repository import TeamRepository
from litellm.types.utils import BudgetConfig
if TYPE_CHECKING:
from litellm.proxy._types import NewProjectRequest, UpdateProjectRequest
from litellm.proxy.utils import PrismaClient, ProxyLogging
def validate_team_model_max_budget(
model_max_budget: Mapping[str, BudgetConfig] | None,
premium_user: bool,
) -> None:
"""Reject a team `model_max_budget` the limiter could not enforce (no duration, bad cap, tpm/rpm limits)."""
if not model_max_budget:
return
if premium_user is not True:
raise HTTPException(
status_code=403,
detail={
"error": f"Setting model_max_budget on a team is an enterprise feature. {CommonProxyErrors.not_premium_user.value}"
},
)
for model_name, budget_config in model_max_budget.items():
if not model_name.strip():
raise HTTPException(
status_code=400,
detail={"error": "model_max_budget keys must be non-empty model names"},
)
max_budget = budget_config.max_budget
if max_budget is None or not math.isfinite(max_budget) or max_budget < 0:
raise HTTPException(
status_code=400,
detail={
"error": (
f"model_max_budget[{model_name!r}].max_budget must be a non-negative finite number. "
f"Received: {max_budget}"
)
},
)
if budget_config.budget_duration is None:
raise HTTPException(
status_code=400,
detail={"error": f"model_max_budget[{model_name!r}] requires a budget_duration, e.g. '1d' or '30d'"},
)
validate_budget_duration(budget_config.budget_duration)
if budget_config.tpm_limit is not None or budget_config.rpm_limit is not None:
raise HTTPException(
status_code=400,
detail={
"error": (
f"model_max_budget[{model_name!r}] tpm_limit/rpm_limit are not enforced on a team; "
"set per-model rate limits on the key instead"
)
},
)
def require_caller_user_id_for_non_admin(
user_api_key_dict: UserAPIKeyAuth,
) -> str:

102
litellm/proxy/management_endpoints/key_management_endpoints.py
View file

@ -4166,7 +4166,10 @@ async def info_key_fn(
Returns:
- key: str - The key that was looked up, echoed back as it was passed in
- info: dict - The key's row, minus the hashed token
- info: dict - The key's row, minus the hashed token. Deleted keys are served from the
LiteLLM_DeletedVerificationToken archive and carry deleted_at / deleted_by
- status: "active" | "expired" | "revoked" | "deleted" - Derived from blocked, expires and
whether the row came from the archive
- key_alias: str | None - User-friendly key alias
- spend: float - Amount spent by the key. When budget_duration is set this covers only the
current budget window, not the key's lifetime
@ -4220,10 +4223,15 @@ async def info_key_fn(
hashed_key: str | None = key
if key is not None:
hashed_key = _hash_token_if_needed(token=key)
key_info = await _prisma_table(VerificationTokenRepository(prisma_client)).find_unique(
live_key_info: Final = await _prisma_table(VerificationTokenRepository(prisma_client)).find_unique(
where={"token": hashed_key},
include={"litellm_budget_table": True},
)
key_info: Final = (
live_key_info
if live_key_info is not None
else await _find_deleted_key_info(prisma_client=prisma_client, hashed_key=hashed_key)
)
if key_info is None:
raise ProxyException(
message="Key not found in database",
@ -4231,7 +4239,6 @@ async def info_key_fn(
param="key",
code=status.HTTP_404_NOT_FOUND,
)
if (
await _can_user_query_key_info(
user_api_key_dict=user_api_key_dict,
@ -4245,38 +4252,46 @@ async def info_key_fn(
detail=f"You are not allowed to access this key's info. Your role={user_api_key_dict.user_role}",
)
## REMOVE HASHED TOKEN INFO BEFORE RETURNING ##
try:
key_info = key_info.model_dump()
except Exception:
# if using pydantic v1
key_info = key_info.dict() # pyright: ignore[reportDeprecated] # deliberate pydantic v1 fallback
key_token_hash: Final[str | None] = key_info.pop("token")
key_info_dict: Final = key_info.model_dump()
key_token_hash: Final[str | None] = key_info_dict.pop("token")
key_info_dict["status"] = (
"deleted" if live_key_info is None else _derive_key_status(key_info_dict, now=datetime.now(timezone.utc))
)
model_max_budget = key_info.get("model_max_budget") or {}
budget_table: Final = key_info.get("litellm_budget_table") or {}
model_max_budget = key_info_dict.get("model_max_budget") or {}
budget_table: Final = key_info_dict.get("litellm_budget_table") or {}
if not model_max_budget and isinstance(budget_table, dict):
model_max_budget = budget_table.get("model_max_budget") or {}
if model_max_budget and key_token_hash:
key_info["model_max_budget_usage"] = await _build_model_max_budget_usage(
key_info_dict["model_max_budget_usage"] = await _build_model_max_budget_usage(
api_key_hash=key_token_hash,
model_max_budget=model_max_budget,
user_api_key_cache=model_max_budget_limiter.dual_cache,
)
budget_limits_usage: Final = await _build_budget_limits_usage(
budget_limits=key_info.get("budget_limits"),
budget_limits=key_info_dict.get("budget_limits"),
api_key_hash=key_token_hash,
)
if budget_limits_usage is not None:
key_info["budget_limits_usage"] = budget_limits_usage
key_info_dict["budget_limits_usage"] = budget_limits_usage
# Attach object_permission if object_permission_id is set
key_info = await attach_object_permission_to_dict(key_info, prisma_client)
return {"key": key, "info": key_info}
return {"key": key, "info": await attach_object_permission_to_dict(key_info_dict, prisma_client)}
except Exception as e:
raise handle_exception_on_proxy(e)
async def _find_deleted_key_info(
prisma_client: PrismaClient, hashed_key: str | None
) -> LiteLLM_DeletedVerificationToken | None:
archived_row: Final = await _deleted_verification_token_table(prisma_client).find_first(
where={"token": hashed_key},
order={"deleted_at": "desc"},
)
if archived_row is None:
return None
return LiteLLM_DeletedVerificationToken.model_validate(archived_row.model_dump())
def _check_model_access_group(models: list[str] | None, llm_router: Router | None, premium_user: bool) -> Literal[True]:
"""
if is_model_access_group is True + is_wildcard_route is True, check if user is a premium user
@ -6216,6 +6231,24 @@ async def get_member_team_ids(
VALID_EXPIRES_FILTER_VALUES: Final = frozenset({"active", "expired"})
KeyStatus = Literal["active", "expired", "revoked", "deleted"]
VALID_STATUS_FILTER_VALUES: Final[frozenset[KeyStatus]] = frozenset({"active", "expired", "revoked", "deleted"})
class _KeyStatusSource(BaseModel):
blocked: bool | None = None
expires: datetime | None = None
def _derive_key_status(row: Mapping[str, object], now: datetime) -> KeyStatus:
source: Final = _KeyStatusSource.model_validate(row)
if source.blocked is True:
return "revoked"
if source.expires is None:
return "active"
expires_utc: Final = source.expires if source.expires.tzinfo else source.expires.replace(tzinfo=timezone.utc)
return "expired" if expires_utc < now else "active"
@router.get(
"/key/list",
@ -6252,7 +6285,10 @@ async def list_keys(
),
sort_order: str = Query(default="desc", description="Sort order ('asc' or 'desc')"),
expand: list[str] | None = Query(None, description="Expand related objects (e.g. 'user')"),
status: str | None = Query(None, description="Filter by status (e.g. 'deleted')"),
status: str | None = Query(
None,
description="Filter by status: 'active' (not blocked, not expired), 'expired' (not blocked, past expiry), 'revoked' (blocked) or 'deleted' (archived keys). Omit to return live keys regardless of status.",
),
project_id: str | None = Query(None, description="Filter keys by project ID"),
access_group_id: str | None = Query(None, description="Filter keys by access group ID"),
agent_id: str | None = Query(None, description="Filter keys by agent ID"),
@ -6270,7 +6306,9 @@ async def list_keys(
Parameters:
expand: Optional[List[str]] - Expand related objects (e.g. 'user' to include user information)
status: Optional[str] - Filter by status. Currently supports "deleted" to query deleted keys.
status: Optional[str] - Filter by status: "active", "expired", "revoked" (blocked) or "deleted".
"deleted" reads the LiteLLM_DeletedVerificationToken archive; the other values partition the
live key table, so every live key matches exactly one of them.
Returns:
{
@ -6292,11 +6330,10 @@ async def list_keys(
verbose_proxy_logger.error("Database not connected")
raise Exception("Database not connected")
# Validate status parameter
if status is not None and status != "deleted":
if status is not None and status not in VALID_STATUS_FILTER_VALUES:
raise HTTPException(
status_code=400,
detail={"error": "Invalid status value. Currently only 'deleted' is supported."},
detail={"error": "Invalid status value. Supported: 'active', 'expired', 'revoked', 'deleted'."},
)
if isinstance(expires, str) and expires not in VALID_EXPIRES_FILTER_VALUES:
@ -6608,6 +6645,18 @@ def _build_expires_where_clause(expires_filter: str, now: datetime) -> dict[str,
return {"OR": [{"expires": None}, {"expires": {"gte": now}}]}
def _not_blocked_where_clause() -> dict[str, object]:
return {"OR": [{"blocked": None}, {"blocked": False}]}
def _build_status_where_clause(status_filter: str | None, now: datetime) -> dict[str, object] | None:
if status_filter == "revoked":
return {"blocked": True}
if status_filter in ("expired", "active"):
return {"AND": [_not_blocked_where_clause(), _build_expires_where_clause(status_filter, now)]}
return None
def _build_key_search_where(search: str) -> KeySearchWhere:
search_where: Final[KeySearchWhere] = {
"OR": (
@ -6635,6 +6684,7 @@ def _build_key_filter_conditions(
use_key_alias_substring_matching: bool = False,
expires_filter: str | None = None,
search: str | None = None,
status_filter: str | None = None,
) -> Mapping[str, object]:
"""Build filter conditions for key listing.
@ -6724,6 +6774,8 @@ def _build_key_filter_conditions(
# Apply team_id, project_id and access_group_id as global AND filters so they
# narrow results across all visibility conditions (own keys, team keys, etc.)
now: Final = datetime.now(timezone.utc)
status_where: Final = _build_status_where_clause(status_filter, now)
global_filters: Final[tuple[Mapping[str, object], ...]] = (
*(
(
@ -6741,10 +6793,11 @@ def _build_key_filter_conditions(
*(({"access_group_ids": {"hasSome": [access_group_id]}},) if access_group_id else ()),
*(({"agent_id": agent_id},) if agent_id and isinstance(agent_id, str) else ()),
*(
(_build_expires_where_clause(expires_filter, datetime.now(timezone.utc)),)
(_build_expires_where_clause(expires_filter, now),)
if expires_filter is not None and expires_filter in VALID_EXPIRES_FILTER_VALUES
else ()
),
*((status_where,) if status_where is not None else ()),
)
combined_where: Final[Mapping[str, object]] = {"AND": [where, *global_filters]} if global_filters else where
verbose_proxy_logger.debug("Filter conditions: %s", combined_where)
@ -6817,6 +6870,7 @@ async def _list_key_helper(
use_key_alias_substring_matching=use_key_alias_substring_matching,
expires_filter=expires_filter,
search=search,
status_filter=status,
)
# Calculate skip for pagination

86
litellm/proxy/management_endpoints/team_endpoints.py
View file

@ -22,7 +22,7 @@ from typing import TYPE_CHECKING, Annotated, Final, NamedTuple, NoReturn, Protoc
import fastapi
from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
from pydantic import BaseModel, JsonValue
from pydantic import BaseModel, JsonValue, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
@ -38,6 +38,7 @@ from litellm.proxy._types import (
DeleteTeamRequest,
LiteLLM_AuditLogs,
LiteLLM_DeletedTeamTable,
Litellm_EntityType,
LiteLLM_ManagementEndpoint_MetadataFields,
LiteLLM_ManagementEndpoint_MetadataFields_Premium,
LiteLLM_ModelTable,
@ -95,6 +96,10 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars
from litellm.proxy.common_utils.json_merge_patch import apply_json_merge_patch
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.hooks.model_max_budget_limiter import (
build_model_max_budget_usage,
resolve_model_budget,
)
from litellm.proxy.management_endpoints.common_daily_activity import (
get_daily_activity_aggregated,
)
@ -108,6 +113,7 @@ from litellm.proxy.management_endpoints.common_utils import (
_upsert_budget_and_membership,
_user_has_admin_view,
validate_budget_duration,
validate_team_model_max_budget,
)
from litellm.proxy.management_endpoints.organization_endpoints import (
add_member_to_organization,
@ -177,6 +183,7 @@ from litellm.types.proxy.management_endpoints.team_endpoints import (
TeamUserSpendRow,
UpdateTeamMemberPermissionsRequest,
)
from litellm.types.utils import BudgetConfig
if TYPE_CHECKING:
from prisma import Prisma
@ -1170,6 +1177,62 @@ def _check_team_budget_update_authority(
)
def _existing_model_cap(raw_budget_config: object) -> BudgetConfig | None:
try:
return BudgetConfig.model_validate(raw_budget_config)
except ValidationError:
return None
def _check_team_model_budget_update_authority(
data: UpdateTeamRequest,
user_api_key_dict: UserAPIKeyAuth,
existing_model_max_budget: Mapping[str, object] | None,
) -> None:
"""Like `_check_team_budget_update_authority`: only a proxy admin may raise, re-window or drop a per-model cap."""
if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
return
if "model_max_budget" not in data.model_fields_set or not existing_model_max_budget:
return
requested: Final[Mapping[str, BudgetConfig]] = data.model_max_budget or {}
for model_name, raw_existing in existing_model_max_budget.items():
existing = _existing_model_cap(raw_existing)
if existing is None or existing.max_budget is None or model_name in requested:
continue
raise HTTPException(
status_code=403,
detail={
"error": (
f"Only a proxy admin can remove a team's model_max_budget for {model_name!r}. "
f"Current max_budget={existing.max_budget}."
)
},
)
for model_name, proposed in requested.items():
governing = resolve_model_budget(model=model_name, model_max_budget=existing_model_max_budget)
if governing is None:
continue
cap = governing.budget_config
if cap.max_budget is None:
continue
if (
proposed.max_budget is None
or proposed.max_budget > cap.max_budget
or proposed.budget_duration != cap.budget_duration
):
raise HTTPException(
status_code=403,
detail={
"error": (
f"Only a proxy admin can raise a team's model_max_budget for {model_name!r} or change its "
f"budget_duration. Current max_budget={cap.max_budget} per {cap.budget_duration} "
f"(entry {governing.budget_model!r}), requested={proposed.max_budget} per "
f"{proposed.budget_duration}."
)
},
)
def _should_auto_add_team_creator(
user_api_key_dict: UserAPIKeyAuth,
general_settings: Mapping[str, object],
@ -1230,6 +1293,7 @@ async def new_team(
- prompts: Optional[List[str]] - List of prompts that the team is allowed to use.
- organization_id: Optional[str] - The organization id of the team. Default is None. Create via `/organization/new`.
- model_aliases: Optional[dict] - Model aliases for the team. [Docs](https://docs.litellm.ai/docs/proxy/team_based_routing#create-team-with-model-alias)
- model_max_budget: Optional[dict] - Per-model max budget every key on the team inherits unless the key sets its own for that model. Example: {"gpt-4o": {"max_budget": 10, "budget_duration": "1d"}}
- guardrails: Optional[List[str]] - Guardrails for the team. [Docs](https://docs.litellm.ai/docs/proxy/guardrails)
- policies: Optional[List[str]] - Policies for the team. [Docs](https://docs.litellm.ai/docs/proxy/guardrails/guardrail_policies)
- disable_global_guardrails: Optional[bool] - Whether to disable global guardrails for the key.
@ -1291,6 +1355,7 @@ async def new_team(
general_settings,
litellm_proxy_admin_name,
llm_router,
premium_user,
prisma_client,
user_api_key_cache,
)
@ -1321,6 +1386,7 @@ async def new_team(
validate_budget_duration(data.budget_duration)
validate_budget_duration(data.team_member_budget_duration)
validate_team_model_max_budget(model_max_budget=data.model_max_budget, premium_user=premium_user)
if data.soft_budget is not None:
if data.max_budget is not None:
@ -1980,6 +2046,7 @@ async def update_team(
- tags: Optional[List[str]] - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing).
- organization_id: Optional[str] - The organization id of the team. Default is None. Create via `/organization/new`.
- model_aliases: Optional[dict] - Model aliases for the team. [Docs](https://docs.litellm.ai/docs/proxy/team_based_routing#create-team-with-model-alias)
- model_max_budget: Optional[dict] - Per-model max budget every key on the team inherits unless the key sets its own for that model. Example: {"gpt-4o": {"max_budget": 10, "budget_duration": "1d"}}
- guardrails: Optional[List[str]] - Guardrails for the team. [Docs](https://docs.litellm.ai/docs/proxy/guardrails)
- policies: Optional[List[str]] - Policies for the team. [Docs](https://docs.litellm.ai/docs/proxy/guardrails/guardrail_policies)
- disable_global_guardrails: Optional[bool] - Whether to disable global guardrails for the key.
@ -2031,6 +2098,7 @@ async def update_team(
from litellm.proxy.proxy_server import (
litellm_proxy_admin_name,
llm_router,
premium_user,
prisma_client,
proxy_logging_obj,
user_api_key_cache,
@ -2069,6 +2137,7 @@ async def update_team(
validate_budget_duration(data.budget_duration)
validate_budget_duration(data.team_member_budget_duration)
validate_team_model_max_budget(model_max_budget=data.model_max_budget, premium_user=premium_user)
existing_team_row = await _raw_team_db(TeamRepository(prisma_client)).find_unique(
where={"team_id": data.team_id}
@ -2204,8 +2273,15 @@ async def update_team(
user_api_key_dict=user_api_key_dict,
existing_team_max_budget=existing_team_row.max_budget,
)
_check_team_model_budget_update_authority(
data=data,
user_api_key_dict=user_api_key_dict,
existing_model_max_budget=existing_team_row.model_max_budget,
)
updated_kv = data.json(exclude_unset=True)
if "model_max_budget" in updated_kv and updated_kv["model_max_budget"] is None:
updated_kv["model_max_budget"] = {}
# Drop server-owned metadata keys from caller input so they can only
# be written by the same code path that creates the underlying rows.
@ -4473,7 +4549,7 @@ async def team_info(
```
"""
from litellm.proxy._types import TeamInfoResponseObjectTeamTable
from litellm.proxy.proxy_server import prisma_client
from litellm.proxy.proxy_server import model_max_budget_limiter, prisma_client
try:
if prisma_client is None:
@ -4573,6 +4649,12 @@ async def team_info(
update={ # mutable-ok: pydantic update payload
"members_with_roles": hydrated_members,
"organization_models": organization_models,
"model_max_budget_usage": await build_model_max_budget_usage(
entity_type=Litellm_EntityType.TEAM,
entity_id=team_id,
model_max_budget=resolved_team_info.model_max_budget,
cache=model_max_budget_limiter.dual_cache,
),
}
)

17
litellm/proxy/ocr_endpoints/endpoints.py
View file

@ -15,12 +15,13 @@ from litellm.llms.base_llm.ocr.transformation import (
OCRResponse,
parse_ocr_request_format,
)
from litellm.ocr.input import convert_upload_to_url_document, get_max_file_bytes
from litellm.ocr.legacy import convert_file_document_to_url_document, get_mime_type
from litellm.proxy._types import *
from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
router: Final = APIRouter()
_MAX_FILE_BYTES: Final = 50 * 1024 * 1024
def _build_document_from_upload(
@ -28,7 +29,15 @@ def _build_document_from_upload(
filename: str | None,
content_type: str | None,
) -> dict[str, str]:
return convert_upload_to_url_document(file_content, filename, content_type)
supplied_mime: Final = content_type.split(";")[0].strip() if content_type else None
mime_type: Final = (
get_mime_type(filename)
if filename and (not supplied_mime or supplied_mime == "application/octet-stream")
else supplied_mime
)
return convert_file_document_to_url_document(
{"type": "file", "file": file_content, "mime_type": mime_type or "application/octet-stream"}
)
def _with_request_format(data: Mapping[str, Any], request: Request) -> Mapping[str, Any]:
@ -103,9 +112,11 @@ async def _parse_multipart_form(request: Request) -> dict[str, Any]:
# Seek to start in case the file was already partially read by middleware
await uploaded_file.seek(0)
file_content: Final = await uploaded_file.read(get_max_file_bytes() + 1)
file_content: Final = await uploaded_file.read(_MAX_FILE_BYTES + 1)
if not file_content:
raise ValueError("Uploaded file is empty")
if len(file_content) > _MAX_FILE_BYTES:
raise ValueError("OCR file exceeds the size limit")
document: Final = _build_document_from_upload(
file_content=file_content,

67
litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
View file

@ -72,7 +72,9 @@ from litellm.proxy.auth.auth_utils import request_dispatched_to_pass_through_end
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_request_processing import (
ProxyBaseLLMRequestProcessing,
log_llm_api_exception,
open_sse_before_first_byte,
resolve_litellm_call_id,
)
from litellm.proxy.common_utils.http_parsing_utils import (
_read_request_body,
@ -80,6 +82,7 @@ from litellm.proxy.common_utils.http_parsing_utils import (
)
from litellm.proxy.common_utils.openai_error_payload import (
error_status_code,
litellm_call_id_headers,
openai_error_param,
openai_error_type,
)
@ -196,14 +199,15 @@ async def chat_completion_pass_through_endpoint(
version,
)
data = {}
litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id"))
data = {"litellm_call_id": litellm_call_id}
try:
body: Final = await request.body()
body_str: Final = body.decode()
try:
data = ast.literal_eval(body_str)
data = ast.literal_eval(body_str) | data
except Exception:
data = json.loads(body_str)
data = json.loads(body_str) | data
data["adapter_id"] = adapter_id
@ -290,9 +294,7 @@ async def chat_completion_pass_through_endpoint(
response_cost: Final = hidden_params.get("response_cost", None) or ""
### ALERTING ###
asyncio.create_task(
proxy_logging_obj.update_request_status(litellm_call_id=data.get("litellm_call_id", ""), status="success")
)
asyncio.create_task(proxy_logging_obj.update_request_status(litellm_call_id=litellm_call_id, status="success"))
verbose_proxy_logger.debug("final response: %s", response)
@ -313,12 +315,13 @@ async def chat_completion_pass_through_endpoint(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
verbose_proxy_logger.exception("litellm.proxy.proxy_server.completion(): Exception occured - %s", e)
log_llm_api_exception(e, litellm_call_id)
error_msg: Final = f"{e}"
raise ProxyException(
message=getattr(e, "message", error_msg),
type=openai_error_type(e, error_status_code(e, 500)),
param=openai_error_param(e),
headers=litellm_call_id_headers(litellm_call_id),
code=error_status_code(e, 500),
)
@ -609,6 +612,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
# merely shares the name.
if not request_dispatched_to_pass_through_endpoint(request):
_metadata["user_api_key_model_max_budget"] = user_api_key_dict.model_max_budget
_metadata["user_api_key_team_model_max_budget"] = user_api_key_dict.team_model_max_budget
_metadata["user_api_key_user_model_max_budget"] = user_api_key_dict.user_model_max_budget
_metadata["user_api_key_end_user_model_max_budget"] = user_api_key_dict.end_user_model_max_budget
_metadata.update(
@ -985,6 +989,7 @@ async def pass_through_request(
headers=headers,
forward_headers=forward_headers,
)
upstream_headers: Final = _with_trace_context(headers, parent_span=user_api_key_dict.parent_otel_span)
requested_query_params: dict | None = query_params or dict(request.query_params)
@ -1018,7 +1023,7 @@ async def pass_through_request(
verbose_proxy_logger.debug(
"Pass through endpoint sending request to \nURL %s\nheaders: %s\nbody: %s\n",
url,
headers,
upstream_headers,
_parsed_body,
)
@ -1256,7 +1261,7 @@ async def pass_through_request(
additional_args={
"complete_input_dict": _parsed_body,
"api_base": str(logging_url),
"headers": headers,
"headers": upstream_headers,
},
)
stream = HttpPassThroughEndpointHelpers._update_stream_param_based_on_request_body(
@ -1273,7 +1278,7 @@ async def pass_through_request(
request=request,
async_client=async_client,
url=url,
headers=headers,
headers=upstream_headers,
requested_query_params=requested_query_params,
stream=True,
)
@ -1285,7 +1290,7 @@ async def pass_through_request(
request.method,
url,
params=requested_query_params,
headers=headers,
headers=upstream_headers,
content=state_raw_body,
)
if state_raw_body is not None
@ -1293,7 +1298,7 @@ async def pass_through_request(
request.method,
url,
params=requested_query_params,
headers=headers,
headers=upstream_headers,
json=_parsed_body,
)
)
@ -1370,7 +1375,7 @@ async def pass_through_request(
raw_body_request: Final = async_client.build_request(
request.method,
url,
headers=headers,
headers=upstream_headers,
params=requested_query_params,
content=state_raw_body,
)
@ -1380,7 +1385,7 @@ async def pass_through_request(
request=request,
async_client=async_client,
url=url,
headers=headers,
headers=upstream_headers,
requested_query_params=requested_query_params,
_parsed_body=_parsed_body,
forward_multipart=is_multipart,
@ -2157,6 +2162,17 @@ def _upstream_close_to_relay(task_results: Iterable[object]) -> Close | None:
return upstream_close
_WEBSOCKET_FORWARDED_HEADERS: Final = frozenset(("authorization", "x-api-key", "x-goog-user-project"))
def _with_trace_context(headers: Mapping[str, str], parent_span: object) -> dict[str, str]:
try:
from litellm.integrations.otel.plumbing.context import inject_trace_context
except ImportError:
return dict(headers) # mutable-ok: matches inject_trace_context's carrier return type
return inject_trace_context(headers, parent_span=parent_span)
async def websocket_passthrough_request(
websocket: WebSocket,
target: str,
@ -2199,20 +2215,15 @@ async def websocket_passthrough_request(
await websocket.accept()
verbose_proxy_logger.debug("WebSocket passthrough (%s): WebSocket connection accepted", endpoint)
# Prepare headers for the upstream connection
upstream_headers: Final = custom_headers.copy()
if forward_headers:
# Forward relevant headers from the incoming request
incoming_headers: Final = dict(websocket.headers)
for header_name, header_value in incoming_headers.items():
# Only forward certain headers to avoid conflicts
if header_name.lower() in [
"authorization",
"x-api-key",
"x-goog-user-project",
]:
upstream_headers[header_name] = header_value
forwarded_headers: Final = { # mutable-ok: one-shot upstream header dict, read as a Mapping
**custom_headers,
**{
header_name: header_value
for header_name, header_value in websocket.headers.items()
if forward_headers and header_name.lower() in _WEBSOCKET_FORWARDED_HEADERS
},
}
upstream_headers: Final = _with_trace_context(forwarded_headers, parent_span=user_api_key_dict.parent_otel_span)
# Initialize logging object similar to HTTP passthrough
team_callbacks: Final = _resolve_team_callback_wiring(

74
litellm/proxy/proxy_server.py
View file

@ -323,6 +323,7 @@ from litellm.proxy.auth.auth_utils import (
log_once_if_budget_reservation_disabled,
warn_once_if_custom_auth_skips_common_checks,
)
from litellm.proxy.auth.fallback_budget import router_fallback_budget_check
from litellm.proxy.auth.fallback_model_access import router_fallback_access_check
from litellm.proxy.auth.handle_jwt import JWTHandler
from litellm.proxy.auth.litellm_license import AUTO_ROUTER_LICENSE_REMEDY, LicenseCheck
@ -349,7 +350,10 @@ from litellm.proxy.common_request_processing import (
_is_azure_model_router_request,
_should_return_raw_model_name,
create_response,
log_llm_api_exception,
open_sse_before_first_byte,
request_litellm_call_id,
resolve_litellm_call_id,
ttft_keepalive_interval,
)
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
@ -388,6 +392,11 @@ from litellm.proxy.common_utils.model_listing_utils import (
from litellm.proxy.common_utils.openai_endpoint_utils import (
remove_sensitive_info_from_deployment,
)
from litellm.proxy.common_utils.openai_error_payload import (
headers_with_litellm_call_id,
litellm_call_id_headers,
with_litellm_call_id,
)
from litellm.proxy.common_utils.periodic_reload_schedule import (
MODEL_COST_MAP_RELOAD_PARAM_NAME,
clear_reload_interval,
@ -6161,6 +6170,7 @@ class ProxyConfig:
),
ignore_invalid_deployments=True, # don't raise an error if a deployment is invalid
fallback_access_check=router_fallback_access_check,
fallback_budget_check=router_fallback_budget_check,
auto_router_capability_limit=_license_check.auto_router_capability_limit,
)
@ -6622,6 +6632,7 @@ class ProxyConfig:
search_tools=search_tools,
ignore_invalid_deployments=True,
fallback_access_check=router_fallback_access_check,
fallback_budget_check=router_fallback_budget_check,
auto_router_capability_limit=_license_check.auto_router_capability_limit,
)
verbose_proxy_logger.debug("updated llm_router: %s", llm_router)
@ -11299,12 +11310,14 @@ async def completion(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
verbose_proxy_logger.exception("litellm.proxy.proxy_server.completion(): Exception occured - %s", e)
litellm_call_id: Final = request_litellm_call_id(data)
log_llm_api_exception(e, litellm_call_id)
error_msg: Final = f"{e}"
raise ProxyException(
message=getattr(e, "message", error_msg),
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
headers=litellm_call_id_headers(litellm_call_id),
openai_code=getattr(e, "code", None),
code=getattr(e, "status_code", 500),
)
@ -11461,11 +11474,12 @@ async def moderations(
```
"""
global proxy_logging_obj
data: dict = {}
litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id"))
data: dict = {"litellm_call_id": litellm_call_id}
try:
# Use orjson to parse JSON data, orjson speeds up requests significantly
body: Final = await request.body()
data = orjson.loads(body)
data = orjson.loads(body) | data
# Include original request and headers in the data
data = await add_litellm_data_to_request(
@ -11502,9 +11516,7 @@ async def moderations(
response: Final = await llm_call
### ALERTING ###
asyncio.create_task(
proxy_logging_obj.update_request_status(litellm_call_id=data.get("litellm_call_id", ""), status="success")
)
asyncio.create_task(proxy_logging_obj.update_request_status(litellm_call_id=litellm_call_id, status="success"))
### RESPONSE HEADERS ###
hidden_params: Final = getattr(response, "_hidden_params", {}) or {}
@ -11530,14 +11542,15 @@ async def moderations(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
verbose_proxy_logger.exception("litellm.proxy.proxy_server.moderations(): Exception occured - %s", e)
log_llm_api_exception(e, litellm_call_id)
if isinstance(e, ProxyException):
raise
raise with_litellm_call_id(e, litellm_call_id)
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "message", str(e)),
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
headers=litellm_call_id_headers(litellm_call_id),
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
else:
@ -11546,6 +11559,7 @@ async def moderations(
message=getattr(e, "message", error_msg),
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
headers=litellm_call_id_headers(litellm_call_id),
code=getattr(e, "status_code", 500),
)
@ -11583,11 +11597,12 @@ async def audio_speech(
https://platform.openai.com/docs/api-reference/audio/createSpeech
"""
global proxy_logging_obj
data: dict = {}
litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id"))
data: dict = {"litellm_call_id": litellm_call_id}
try:
# Use orjson to parse JSON data, orjson speeds up requests significantly
body: Final = await request.body()
data = orjson.loads(body)
data = orjson.loads(body) | data
# Include original request and headers in the data
data = await add_litellm_data_to_request(
@ -11620,9 +11635,7 @@ async def audio_speech(
response: Final = await llm_call
### ALERTING ###
asyncio.create_task(
proxy_logging_obj.update_request_status(litellm_call_id=data.get("litellm_call_id", ""), status="success")
)
asyncio.create_task(proxy_logging_obj.update_request_status(litellm_call_id=litellm_call_id, status="success"))
### RESPONSE HEADERS ###
hidden_params: Final = getattr(response, "_hidden_params", {}) or {}
@ -11630,7 +11643,7 @@ async def audio_speech(
cache_key: Final = hidden_params.get("cache_key", None) or ""
api_base: Final = hidden_params.get("api_base", None) or ""
response_cost: Final = hidden_params.get("response_cost", None) or ""
litellm_call_id: Final = hidden_params.get("litellm_call_id", None) or ""
response_call_id: Final = hidden_params.get("litellm_call_id", None) or ""
custom_headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=user_api_key_dict,
@ -11641,7 +11654,7 @@ async def audio_speech(
response_cost=response_cost,
model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
fastest_response_batch_completion=None,
call_id=litellm_call_id,
call_id=response_call_id,
request_data=data,
hidden_params=hidden_params,
)
@ -11677,14 +11690,20 @@ async def audio_speech(
original_exception=e,
request_data=data,
)
verbose_proxy_logger.error("litellm.proxy.proxy_server.audio_speech(): Exception occured - %s", e)
verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, (ProxyException, HTTPException)):
raise e
log_llm_api_exception(e, litellm_call_id)
if isinstance(e, ProxyException):
raise with_litellm_call_id(e, litellm_call_id)
if isinstance(e, HTTPException):
raise HTTPException(
status_code=e.status_code,
detail=e.detail,
headers=headers_with_litellm_call_id(e.headers, litellm_call_id),
)
raise ProxyException(
message=getattr(e, "message", f"{e}"),
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
headers=litellm_call_id_headers(litellm_call_id),
openai_code=getattr(e, "code", None),
code=getattr(e, "status_code", 500),
)
@ -11712,11 +11731,12 @@ async def audio_transcriptions(
https://platform.openai.com/docs/api-reference/audio/createTranscription?lang=curl
"""
global proxy_logging_obj
data: dict = {}
litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id"))
data: dict = {"litellm_call_id": litellm_call_id}
try:
# Use orjson to parse JSON data, orjson speeds up requests significantly
form_data: Final = await get_form_data(request)
data = {key: value for key, value in form_data.items() if key != "file"}
data = {key: value for key, value in form_data.items() if key != "file"} | data
# Include original request and headers in the data
data = await add_litellm_data_to_request(
@ -11783,9 +11803,7 @@ async def audio_transcriptions(
file_object.close() # close the file read in by io library
### ALERTING ###
asyncio.create_task(
proxy_logging_obj.update_request_status(litellm_call_id=data.get("litellm_call_id", ""), status="success")
)
asyncio.create_task(proxy_logging_obj.update_request_status(litellm_call_id=litellm_call_id, status="success"))
### RESPONSE HEADERS ###
hidden_params: Final = getattr(response, "_hidden_params", {}) or {}
@ -11793,7 +11811,7 @@ async def audio_transcriptions(
cache_key: Final = hidden_params.get("cache_key", None) or ""
api_base: Final = hidden_params.get("api_base", None) or ""
response_cost: Final = hidden_params.get("response_cost", None) or ""
litellm_call_id: Final = hidden_params.get("litellm_call_id", None) or ""
response_call_id: Final = hidden_params.get("litellm_call_id", None) or ""
additional_headers: Final[dict] = hidden_params.get("additional_headers", {}) or {}
fastapi_response.headers.update(
@ -11805,7 +11823,7 @@ async def audio_transcriptions(
version=version,
response_cost=response_cost,
model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
call_id=litellm_call_id,
call_id=response_call_id,
request_data=data,
hidden_params=hidden_params,
**additional_headers,
@ -11827,12 +11845,13 @@ async def audio_transcriptions(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
verbose_proxy_logger.exception("litellm.proxy.proxy_server.audio_transcription(): Exception occured - %s", e)
log_llm_api_exception(e, litellm_call_id)
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "message", str(e.detail)),
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
headers=litellm_call_id_headers(litellm_call_id),
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
else:
@ -11841,6 +11860,7 @@ async def audio_transcriptions(
message=getattr(e, "message", error_msg),
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
headers=litellm_call_id_headers(litellm_call_id),
openai_code=getattr(e, "code", None),
code=getattr(e, "status_code", 500),
)

23
litellm/proxy/rerank_endpoints/endpoints.py
View file

@ -7,12 +7,16 @@ import orjson
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
from fastapi.responses import ORJSONResponse
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import *
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.proxy.common_request_processing import (
ProxyBaseLLMRequestProcessing,
log_llm_api_exception,
resolve_litellm_call_id,
)
from litellm.proxy.common_utils.openai_error_payload import (
error_status_code,
litellm_call_id_headers,
openai_error_param,
openai_error_type,
)
@ -54,10 +58,11 @@ async def rerank(
version,
)
data = {}
litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id"))
data = {"litellm_call_id": litellm_call_id}
try:
body: Final = await request.body()
data = orjson.loads(body)
data = orjson.loads(body) | data
# Include original request and headers in the data
data = await add_litellm_data_to_request(
@ -82,9 +87,7 @@ async def rerank(
response: Final = await llm_call
### ALERTING ###
asyncio.create_task(
proxy_logging_obj.update_request_status(litellm_call_id=data.get("litellm_call_id", ""), status="success")
)
asyncio.create_task(proxy_logging_obj.update_request_status(litellm_call_id=litellm_call_id, status="success"))
### RESPONSE HEADERS ###
hidden_params: Final = getattr(response, "_hidden_params", {}) or {}
@ -95,7 +98,7 @@ async def rerank(
fastapi_response.headers.update(
ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=user_api_key_dict,
call_id=hidden_params.get("litellm_call_id", None) or data.get("litellm_call_id", None),
call_id=hidden_params.get("litellm_call_id", None) or litellm_call_id,
model_id=model_id,
cache_key=cache_key,
api_base=api_base,
@ -113,12 +116,13 @@ async def rerank(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
verbose_proxy_logger.error("litellm.proxy.proxy_server.rerank(): Exception occured - %s", e)
log_llm_api_exception(e, litellm_call_id)
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "message", str(e)),
type=openai_error_type(e, error_status_code(e, status.HTTP_400_BAD_REQUEST)),
param=openai_error_param(e),
headers=litellm_call_id_headers(litellm_call_id),
code=error_status_code(e, status.HTTP_400_BAD_REQUEST),
)
else:
@ -127,5 +131,6 @@ async def rerank(
message=getattr(e, "message", error_msg),
type=openai_error_type(e, error_status_code(e, 500)),
param=openai_error_param(e),
headers=litellm_call_id_headers(litellm_call_id),
code=error_status_code(e, 500),
)

2
litellm/proxy/schema.prisma
View file

@ -426,6 +426,7 @@ model LiteLLM_VerificationToken {
key_alias String?
soft_budget_cooldown Boolean @default(false) // key-level state on if budget alerts need to be cooled down
spend Float @default(0.0)
total_spend Float @default(0.0)
expires DateTime?
models String[]
aliases Json @default("{}")
@ -528,6 +529,7 @@ model LiteLLM_DeletedVerificationToken {
key_alias String?
soft_budget_cooldown Boolean @default(false)
spend Float @default(0.0)
total_spend Float @default(0.0)
expires DateTime?
models String[]
aliases Json @default("{}")

1
litellm/proxy/spend_tracking/carried_budget_state.py
View file

@ -25,6 +25,7 @@ def carry_team_and_user_budget_state(
budget_reset_at=team_object.budget_reset_at,
max_budget=team_object.max_budget,
)
valid_token.team_model_max_budget = team_object.model_max_budget # rebind-ok: caller keeps this object
if user_object is not None:
valid_token.user_budget_snapshot = UserBudgetSnapshot( # rebind-ok: same object the caller keeps using
budget_reset_at=user_object.budget_reset_at,

21
litellm/proxy/utils.py
View file

@ -38,7 +38,11 @@ from litellm.proxy._types import (
SpendLogsMetadata,
SpendLogsPayload,
)
from litellm.proxy.common_utils.openai_error_payload import openai_error_param
from litellm.proxy.common_utils.openai_error_payload import (
litellm_call_id_headers,
openai_error_param,
with_litellm_call_id,
)
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.proxy.model_listing import ModelInfoResponse
@ -164,7 +168,6 @@ from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrai
)
from litellm.proxy.hooks import PROXY_HOOKS, get_proxy_hook
from litellm.proxy.hooks.cache_control_check import _PROXY_CacheControlCheck
from litellm.proxy.hooks.max_budget_limiter import _PROXY_MaxBudgetLimiter
from litellm.proxy.hooks.parallel_request_limiter import (
_PROXY_MaxParallelRequestsHandler,
)
@ -982,7 +985,6 @@ class ProxyLogging:
dual_cache=DualCache(default_in_memory_ttl=1) # ping redis cache every 1s
)
self.max_parallel_request_limiter = _PROXY_MaxParallelRequestsHandler(self.internal_usage_cache)
self.max_budget_limiter = _PROXY_MaxBudgetLimiter()
self.cache_control_check = _PROXY_CacheControlCheck()
self.alerting: list[str] | None = None
self.alerting_threshold: float = 300 # default to 5 min. threshold
@ -3052,7 +3054,7 @@ class ProxyLogging:
if litellm_logging_obj is None:
from litellm._uuid import uuid
request_data["litellm_call_id"] = str(uuid.uuid4())
request_data.setdefault("litellm_call_id", str(uuid.uuid4()))
user_api_key_logged_metadata: Final = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(
user_api_key_dict=user_api_key_dict
)
@ -3580,7 +3582,7 @@ class ProxyLogging:
caps: Final = ProxyLogging._callback_capabilities()
post_call_pipelines: Final = _streamable_post_call_pipelines(request_data, user_api_key_dict)
# Fast path: no real overrides. Internal proxy CustomLogger callbacks
# (e.g. _PROXY_MaxBudgetLimiter, ManagedFiles) inherit the default
# (e.g. _PROXY_CacheControlCheck, ManagedFiles) inherit the default
# ``async for chunk: yield chunk`` body, so wrapping the iterator
# through each of them adds N pass-through trampolines per chunk for
# zero behavior change. Skip the chain entirely and stream through.
@ -4340,6 +4342,7 @@ class PrismaClient:
v.*,
t.spend AS team_spend,
t.max_budget AS team_max_budget,
t.model_max_budget AS team_model_max_budget,
t.tpm_limit AS team_tpm_limit,
t.rpm_limit AS team_rpm_limit,
t.tpd_limit AS team_tpd_limit
@ -4779,6 +4782,7 @@ class PrismaClient:
t.spend AS team_spend,
t.max_budget AS team_max_budget,
t.soft_budget AS team_soft_budget,
t.model_max_budget AS team_model_max_budget,
t.tpm_limit AS team_tpm_limit,
t.rpm_limit AS team_rpm_limit,
t.tpd_limit AS team_tpd_limit,
@ -7659,7 +7663,7 @@ def _recreate_writer_on_read_only_transaction(prisma_client: "PrismaClient | Non
asyncio.create_task(prisma_client.recreate_read_only_writer(reason="postgres_read_only_transaction"))
def handle_exception_on_proxy(e: Exception) -> ProxyException:
def handle_exception_on_proxy(e: Exception, litellm_call_id: str | None = None) -> ProxyException:
"""
Returns an Exception as ProxyException, this ensures all exceptions are OpenAI API compatible
"""
@ -7671,20 +7675,23 @@ def handle_exception_on_proxy(e: Exception) -> ProxyException:
_recreate_writer_on_read_only_transaction(prisma_client)
headers: Final = litellm_call_id_headers(litellm_call_id)
if isinstance(e, HTTPException):
return ProxyException(
message=getattr(e, "detail", f"error({e})"),
type=ProxyErrorTypes.internal_server_error,
param=openai_error_param(e),
headers=headers,
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
)
elif isinstance(e, ProxyException):
return e
return with_litellm_call_id(e, litellm_call_id)
_status_code: Final = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR)
return ProxyException(
message=str(e),
type=ProxyErrorTypes.internal_server_error,
param=openai_error_param(e),
headers=headers,
code=_status_code,
)

4
litellm/router.py
View file

@ -243,6 +243,7 @@ from litellm.types.router import (
DeploymentModelListingInfo,
DeploymentTypedDict,
FallbackAccessCheck,
FallbackBudgetCheck,
GuardrailTypedDict,
LiteLLM_Params,
MockRouterTestingParams,
@ -780,6 +781,7 @@ class Router:
background_health_check_model_groups: Sequence[str] | None = None,
enable_weighted_failover: bool = False,
fallback_access_check: FallbackAccessCheck | None = None,
fallback_budget_check: FallbackBudgetCheck | None = None,
auto_router_capability_limit: AutoRouterCapabilityLimit | None = None,
) -> None:
"""
@ -818,6 +820,7 @@ class Router:
ignore_invalid_deployments (bool): Ignores invalid deployments, and continues with other deployments. Default is to raise an error.
enable_weighted_failover (bool): When True and the routing strategy is "simple-shuffle", a retryable failure on one deployment causes the request to re-pick (weighted) across the other deployments in the same model group before any cross-group fallback runs. Bounded by `max_fallbacks`. Async-only: currently honored by `router.acompletion()` and other async entrypoints. The sync `router.completion()` path falls back to the regular fallback flow. Defaults to False.
fallback_access_check (Optional[FallbackAccessCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects is skipped. Defaults to None (every configured fallback is attempted).
fallback_budget_check (Optional[FallbackBudgetCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects as over budget is skipped. Defaults to None (budget is not re-checked on fallback).
Returns:
Router: An instance of the litellm.Router class.
@ -859,6 +862,7 @@ class Router:
self.ignore_invalid_deployments = ignore_invalid_deployments
self.auto_router_capability_limit = auto_router_capability_limit
self.fallback_access_check: Final = fallback_access_check
self.fallback_budget_check: Final = fallback_budget_check
self.debug_level = debug_level
self.enable_pre_call_checks = enable_pre_call_checks
self.enable_tag_filtering = enable_tag_filtering

21
litellm/router_utils/fallback_event_handlers.py
View file

@ -421,6 +421,25 @@ async def _is_fallback_target_authorized(
return False
async def _is_fallback_target_within_budget(
litellm_router: LitellmRouter,
fallback_entry: str | Mapping[str, object],
original_model_group: str,
kwargs: Mapping[str, object],
) -> bool:
budget_check: Final = litellm_router.fallback_budget_check
target: Final = _get_fallback_target_model_group(fallback_entry)
if budget_check is None or target is None or target == original_model_group:
return True
if await budget_check(model=target, request_kwargs=kwargs, llm_router=litellm_router):
return True
verbose_router_logger.info(
"Skipping fallback to model_group = %s: caller is over budget",
mask_sensitive_structure(fallback_entry),
)
return False
def references_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool:
"""
True when a file, batch, or fine-tuning job operation names an id that only exists
@ -528,6 +547,8 @@ async def run_async_fallback(
continue
if not await _is_fallback_target_authorized(litellm_router, mg, original_model_group, kwargs):
continue
if not await _is_fallback_target_within_budget(litellm_router, mg, original_model_group, kwargs):
continue
attempt_key = fallback_attempt_key(mg)
if attempt_key is not None:
if attempt_key in attempted:

13
litellm/rust_bridge/_native.pyi
View file

@ -33,15 +33,6 @@ def aocr(
timeout_seconds: float | None = None,
) -> Future[dict[str, object]]: ...
_OCR_MAX_FILE_BYTES: int
def _ocr_upload_document(
file_content: bytes,
file_name: str | None = None,
content_type: str | None = None,
) -> dict[str, str]: ...
def _ocr_file_document(document: Mapping[str, object]) -> dict[str, str]: ...
def _ocr_mime_type(file_name: str) -> str: ...
def _ocr_lifecycle(
request: LiteLLMOcrRequest,
args: tuple[object, ...],
@ -139,15 +130,11 @@ class TokenCounter:
def gil_stats() -> dict[str, int]: ...
__all__ = [
"_OCR_MAX_FILE_BYTES",
"ResponsesWebSocketConnection",
"RustBridgeDeclined",
"RustUpstreamError",
"TokenCounter",
"_ocr_file_document",
"_ocr_lifecycle",
"_ocr_mime_type",
"_ocr_upload_document",
"achat_completions",
"amessages",
"aocr",

13
litellm/types/router.py
View file

@ -963,6 +963,19 @@ class FallbackAccessCheck(Protocol):
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: "Router") -> bool: ...
class FallbackBudgetCheck(Protocol):
"""
Decides whether the caller behind `request_kwargs` is still within budget for fallback `model`.
Budget is enforced once during auth, against the *requested* model group. A fallback target is
chosen later, inside the router, so a zero-cost group that falls back to a priced one bills
without any budget gate. The router runs this before every cross-model-group fallback attempt
and skips targets it rejects, leaving the free attempt itself untouched.
"""
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: "Router") -> bool: ...
class AutoRouterCapabilityLimit(Protocol):
"""
Resolves how many complexity routers may claim each licensed capability right now; None means unlimited.

6
litellm/utils.py
View file

@ -2689,7 +2689,7 @@ def declared_value_factory(model: str, custom_llm_provider: str | None, key: str
"""Return a string value the model map declares for *key*, or ``None`` when it says nothing.
The string-valued sibling of :func:`_supports_factory` and
:func:`_is_explicitly_disabled_factory`, public where those two are not because it is read
:func:`is_explicitly_disabled_factory`, public like the latter because both are read
from the provider configs rather than from this module, sharing their
``get_llm_provider`` -> ``_get_model_info_helper`` chain and their unprefixed-twin
fallback (#20885), so a provider-prefixed entry that omits the key still answers
@ -2725,7 +2725,7 @@ def declared_value_factory(model: str, custom_llm_provider: str | None, key: str
return None
def _is_explicitly_disabled_factory(model: str, custom_llm_provider: str | None, key: str) -> bool:
def is_explicitly_disabled_factory(model: str, custom_llm_provider: str | None, key: str) -> bool:
"""Return True only when the model map explicitly sets *key* to ``False``.
This is the opt-out mirror of :func:`_supports_factory`. Where
@ -2844,7 +2844,7 @@ def is_vision_explicitly_disabled(model: str, custom_llm_provider: str | None =
The opt-out mirror of :func:`supports_vision`: a missing declaration reads as not
disabled, so unknown or newly added models stay eligible for image routing.
"""
return _is_explicitly_disabled_factory(model, custom_llm_provider, "supports_vision")
return is_explicitly_disabled_factory(model, custom_llm_provider, "supports_vision")
def supports_vision(model: str, custom_llm_provider: str | None = None) -> bool:

24
model_prices_and_context_window.json
View file

@ -377,8 +377,7 @@
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_tool_choice": true,
"cache_read_input_token_cost": 1.5e-08
"supports_tool_choice": true
},
"amazon.nova-2-lite-v1:0": {
"cache_read_input_token_cost": 7.5e-08,
@ -561,8 +560,7 @@
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"cache_read_input_token_cost": 8.75e-09
"supports_tool_choice": true
},
"amazon.nova-pro-v1:0": {
"cache_read_input_token_cost": 2e-07,
@ -578,8 +576,7 @@
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_tool_choice": true,
"cache_read_input_token_cost": 2e-07
"supports_tool_choice": true
},
"amazon.nova-sonic-v1:0": {
"deprecation_date": "2026-09-14",
@ -26108,6 +26105,7 @@
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_minimal_reasoning_effort": false,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
@ -26165,6 +26163,7 @@
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_minimal_reasoning_effort": false,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
@ -28114,6 +28113,7 @@
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_minimal_reasoning_effort": false,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
@ -28173,6 +28173,7 @@
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_minimal_reasoning_effort": false,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
@ -28595,6 +28596,7 @@
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_minimal_reasoning_effort": false,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
@ -28652,6 +28654,7 @@
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_minimal_reasoning_effort": false,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
@ -45794,8 +45797,7 @@
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_tool_choice": true,
"cache_read_input_token_cost": 1.5e-08
"supports_tool_choice": true
},
"us.amazon.nova-micro-v1:0": {
"cache_read_input_token_cost": 8.75e-09,
@ -45809,8 +45811,7 @@
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"cache_read_input_token_cost": 8.75e-09
"supports_tool_choice": true
},
"us.amazon.nova-premier-v1:0": {
"deprecation_date": "2026-09-14",
@ -45842,8 +45843,7 @@
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_tool_choice": true,
"cache_read_input_token_cost": 2e-07
"supports_tool_choice": true
},
"us.anthropic.claude-3-5-haiku-20241022-v1:0": {
"cache_creation_input_token_cost": 1e-06,

2
schema.prisma
View file

@ -426,6 +426,7 @@ model LiteLLM_VerificationToken {
key_alias String?
soft_budget_cooldown Boolean @default(false) // key-level state on if budget alerts need to be cooled down
spend Float @default(0.0)
total_spend Float @default(0.0)
expires DateTime?
models String[]
aliases Json @default("{}")
@ -528,6 +529,7 @@ model LiteLLM_DeletedVerificationToken {
key_alias String?
soft_budget_cooldown Boolean @default(false)
spend Float @default(0.0)
total_spend Float @default(0.0)
expires DateTime?
models String[]
aliases Json @default("{}")

54
tests/_live_test_helpers.py
View file

@ -1,6 +1,8 @@
import os
from datetime import date
import pytest
from pydantic import BaseModel, ConfigDict
def _skip_live_prompt_caching_test():
@ -8,3 +10,55 @@ def _skip_live_prompt_caching_test():
pytest.skip("Live prompt-caching E2E tests are opt-in")
if os.environ.get("CASSETTE_REDIS_URL"):
pytest.skip("Live prompt-caching E2E tests cannot run under VCR replay")
class TogetherCostEntry(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
litellm_provider: str | None = None
mode: str | None = None
deprecation_date: str | None = None
input_cost_per_token: float | None = None
output_cost_per_token: float | None = None
supports_function_calling: bool | None = None
supports_response_schema: bool | None = None
def cheapest_together_chat_model(
*, function_calling: bool = False, response_schema: bool = False
) -> str:
import litellm
today = date.today().isoformat()
def qualifies(name: str, entry: TogetherCostEntry) -> bool:
return (
name.startswith("together_ai/")
and entry.litellm_provider == "together_ai"
and entry.mode == "chat"
and (entry.deprecation_date is None or entry.deprecation_date > today)
and (entry.input_cost_per_token or 0.0) > 0
and (entry.output_cost_per_token or 0.0) > 0
and (not function_calling or bool(entry.supports_function_calling))
and (not response_schema or bool(entry.supports_response_schema))
)
registry: dict[str, TogetherCostEntry] = {
name: TogetherCostEntry.model_validate(raw)
for name, raw in litellm.model_cost.items()
if isinstance(raw, dict) and name.startswith("together_ai/")
}
candidates = sorted(
(name for name, entry in registry.items() if qualifies(name, entry)),
key=lambda name: (
registry[name].input_cost_per_token or 0.0,
registry[name].output_cost_per_token or 0.0,
name,
),
)
assert candidates, (
"no live together_ai chat model in the cost map satisfies "
f"function_calling={function_calling} response_schema={response_schema}"
)
return candidates[0]

962
tests/code_coverage_tests/test_provider_cache.py
View file

File diff suppressed because it is too large Load diff

44
tests/e2e/PROVIDER_CACHE.md
View file

@ -1,11 +1,41 @@
# Shared provider-response cache
`E2E_PROVIDER_CACHE=1` enables automatic response reuse in the live E2E mode. Standard OpenAI and Anthropic model registrations use the provider edge. Existing custom API bases, named credentials, mocked models and realtime WebSocket deployments keep their existing routing. Other provider protocols remain live
`E2E_PROVIDER_CACHE=1` enables automatic response reuse in the live E2E mode. Standard OpenAI and Anthropic model registrations use the provider edge, as do Anthropic-on-Bedrock registrations that carry no AWS identity of their own. Existing custom API bases, named credentials, mocked models and realtime WebSocket deployments keep their existing routing. Other provider protocols remain live
The edge caches complete successful POST responses for `/v1/chat/completions` and `/v1/messages`, including streams. Unsupported endpoints pass through. It matches the method, original URL, effective outbound headers (including authentication and HTTP-library defaults), body presence and exact body bytes using a full keyed digest. It sends the same prepared request used for matching. No prompts, random markers, JSON values or credentials are normalized away. Provider `Set-Cookie` headers are dropped before validation and never recorded: the edge already withholds them from the proxy, and OpenAI responses always carry Cloudflare bot-management cookies
The edge caches complete successful POST responses for `/v1/chat/completions`, `/v1/messages`, `/v1/embeddings` and `/v1/responses` on the OpenAI and Anthropic mounts, SSE streams included, and for `/model/{id}/converse` and `/model/{id}/invoke` on a Bedrock mount. Unsupported endpoints pass through. Each endpoint family has its own completeness rule, so a truncated embedding or a Responses run that never reached `response.completed` is not stored
Bedrock's streaming endpoints, `converse-stream` and `invoke-with-response-stream`, cache too. AWS frames those as binary `vnd.amazon.eventstream` rather than SSE, so botocore's own parser reads the frames and validates both CRCs, and each endpoint is then held to its terminal grammar. That matters more than the endpoint count suggests: the Claude Code compat cells drive the real CLI, which always streams, so streaming is most of the suite's Bedrock traffic
Two details of that rule are worth knowing before changing it. A ConverseStream ends with `metadata`, not with `messageStop`, and the `metadata` frame is what carries the token usage litellm prices the call from, so the rule requires it: a stream cut between the two still names a stop reason but would replay as a free call. And a dropped connection is invisible to the parser, which yields the frames it did receive and silently discards a trailing partial one, so the body is also checked against the frame lengths it declares. A stream cut one byte short parses clean and has to be caught that way
## Request identity
A recording belongs to one test. The key is a keyed digest over the test's node id, the method, the URL, the effective outbound headers (including authentication and HTTP-library defaults), body presence and the body bytes, with one normalization: a 12-hex-digit run, the shape `unique_marker()` mints, is replaced by a placeholder in both the URL and a UTF-8 body. Nothing else is normalized away. No prompts, JSON values or credentials are rewritten, and the rule is the one `fixture_canonical.py` already applies for record/replay, so there is a single definition of what a marker is
Requests that differ only by their markers therefore share a canonical identity, which is what makes the cache reusable across builds: every e2e test salts its prompt afresh, so an exact-byte key would miss on every call. Within one test, calls that share a canonical identity are still recorded and replayed separately, by a FIFO slot index appended to the key. That matters because a replayed response carries the recorded provider response id, `LiteLLM_SpendLogs.request_id` is that id, and one shared recording answering two calls would collapse two spend rows into one
Two different tests never share a recording, and a provider call made outside any test (fixtures, session setup) is never cached, because the identity has no test node id to bind to
A client that varies its own request between runs defeats that identity without breaking any rule, and the Claude Code compat cells did. The CLI sends a device id and a session id in `metadata.user_id`, and its system prompt names both its memory directory and its working directory, adding the branch and recent commits when that directory is a git repository. Driven with a fresh HOME and the checkout as its working directory, every cell sent different bytes every build. The fix belongs in the driver rather than here: `claude_code/cli_driver.py` pins the config directory, the working directory and both identifiers, which is why the cache needs no rule for any of it. Normalizing them instead would have hidden a real defect class, since a rule cannot tell a client's own churn from a value a test means to assert on
Provider `Set-Cookie` headers are dropped before validation and never recorded: the edge already withholds them from the proxy, and OpenAI responses always carry Cloudflare bot-management cookies
An eligible miss calls the provider. A complete successful response is stored immediately even if a later test assertion fails. Provider errors, malformed responses, truncated streams and cancelled captures are not stored. Cache reads, writes and lease failures fall through to normal provider behavior; they introduce no provider retry. An already-started response cannot be restarted after a delivery failure
## Bedrock
Bedrock could not be mounted before because SigV4 signs the `Host` header, so a rewritten `api_base` failed signature verification at the provider. The edge now re-signs: it drops the proxy's signature headers, signs the upstream request with the run pod's own AWS identity from its EKS Pod Identity association, and forwards that. The signature headers are excluded from the key, since `x-amz-date` is a timestamp and keying on it would make every Bedrock call a permanent miss
Almost every Bedrock deployment in the suite declares its region as `os.environ/AWS_REGION`, which only the proxy can resolve, and the run pod does not share that environment. A `us.` inference profile fans out across the US regions and is reachable from any of them, so those route to the default mount whatever the proxy resolved. A model that is not cross-region and declares its region that way keeps its direct path rather than being sent to a region it may not exist in.
Only deployments that carry no AWS identity of their own route to the edge. A deployment with `aws_role_name`, `aws_access_key_id`, an `api_base` or an `aws_bedrock_runtime_endpoint` keeps its direct path, because re-signing it would quietly replace the very credential chain that test exists to prove
Which models route is an explicit allowlist in `provider_cache_routing.py`, mirroring the runner role's IAM policy, which names its models one by one. That coupling is deliberate: the edge re-signs with the run pod's identity, so a model the role cannot invoke comes back 403 from Bedrock rather than falling back. An unlisted model keeps its direct path and loses only caching, so adding a Bedrock model to the suite can never turn it red. Adding one to the edge is a policy edit in litellm-ops plus a line here
Vertex and Gemini are not mounted, for different reasons. litellm grafts the default Vertex path onto an `api_base` only when that `api_base` has no path of its own, so a path-prefixed Vertex mount instead becomes `{api_base}:{endpoint}`, dropping project, location and model. Vertex needs a root-mounted edge on its own port, or a change in litellm
Gemini reaches a path-prefixed mount perfectly well and was mounted for one build, then backed out, because litellm's two Gemini endpoints disagree about what `api_base` means. Chat composes `{api_base}/models/{model}:{endpoint}` and defaults `api_base` to `https://generativelanguage.googleapis.com/v1beta`, so the version has to be inside it. File upload composes `{api_base}/upload/v1beta/files` and defaults to the host root, so the version has to be outside it. One `api_base` cannot satisfy both, and a deployment gives no signal at registration time about which it will be used for, so mounting Gemini turned `TestGeminiFiles::test_gemini_file_upload` red in build 227. Anyone pointing litellm's Gemini provider at an AI gateway or a corporate proxy hits the same thing; it is a litellm bug rather than a cache limitation, and mounting Gemini is one line once it is fixed
Recordings are shared across workers and builds through dedicated Redis, separate from the candidate's own cache. They expire 86,400 seconds after capture starts, based on Redis time. Reads never extend expiry. There is no scheduled recapture: the next miss calls the provider again. Bounded coordination reduces duplicate concurrent calls, but slow or failed captures may lead to extra live calls after the wait expires
## Configuration
@ -18,16 +48,18 @@ The trusted runner receives:
- `E2E_PROVIDER_CACHE_NAMESPACE`: shared environment namespace, independent of build and candidate revision
- `E2E_PROVIDER_CACHE_METRICS_DIR`: optional per-process counter artifact directory
Do not give cache credentials to candidate deployments. Counter artifacts contain no recorded payloads or credentials. Hits count shared-cache responses; upstream attempts count actual forwards from the edge. Existing application-cache observations still count requests arriving at the edge, including shared-cache hits
Do not give cache credentials to candidate deployments. Counter artifacts contain no recorded payloads or credentials. Hits count shared-cache responses; upstream attempts count actual forwards from the edge. A rejection also counts its reason, one of `rejected_cut_short` (the consumer walked away mid-capture), `rejected_error_status` (the provider answered, with an error), `rejected_incomplete` (the body arrived whole with a success status and failed its endpoint's rule) or `rejected_unreachable` (the provider could not be reached at all). A mount whose rejections are nearly all of one kind is a different problem from one whose rejections are nearly all of another, and the flat count cannot tell them apart. Every counter is emitted twice, once as a flat total and once under `mount:{mount}:`, so a hit rate can be read per provider rather than only in aggregate. Existing application-cache observations still count requests arriving at the edge, including shared-cache hits
Tests that require real provider timing, limits or state use `@pytest.mark.provider_live`. The marker keeps newly registered models on live routes without weakening their assertions. The provider prompt-caching tests carry it because a replayed priming response reports cache creation rather than a cache read. Ordinary assertion failures still fail E2E. The shared cache does not modify provider response IDs or make the proxy aware of replay
Tests that require real provider timing, limits or state use `@pytest.mark.provider_live`. The marker keeps newly registered models on live routes without weakening their assertions. The provider prompt-caching tests carry it because a replayed priming response reports cache creation rather than a cache read.
One more class needs it, and it is the cost of normalizing the marker. A test that mints a fresh marker, sends it, and then asserts the provider's answer contains that exact value is asserting on the marker rather than using it as a salt. The key treats two such requests as the same identity, so a stale recording matches and answers with the marker from the run that recorded it. `TestOpenAIMessagesToolContinuation` is the one in the suite today: it sends a freshly minted receipt through a tool result and asserts the model echoes it back verbatim. If you add a test that asserts a provider echoed your own unique value, it belongs on the live path. Ordinary assertion failures still fail E2E. The shared cache does not modify provider response IDs or make the proxy aware of replay
## Recorded response semantics
Replay preserves the original response ID, usage and end-to-end headers. The proxy can therefore deduplicate repeated provider IDs when storing spend-log rows, just as it does when a live upstream returns the same ID twice. One spend-log row per invocation is not guaranteed for identical recorded responses. Existing spend reconciliation requests use distinct prompt markers and retain their distinct-ID and row-count assertions; accounting tests are not automatically excluded from caching
Replay preserves the original response ID, usage and end-to-end headers. The proxy can therefore deduplicate repeated provider IDs when storing spend-log rows, just as it does when a live upstream returns the same ID twice. One spend-log row per invocation is not guaranteed for identical recorded responses. Spend reconciliation keeps its distinct-ID and row-count assertions: its prompts differ by an index as well as a marker, so they stay distinct once markers are normalized, and calls that are canonically equal within one test take separate FIFO slots and separate recordings anyway. Accounting tests are not automatically excluded from caching
Provider remaining-quota headers describe the captured response. Metrics derived from them are historical on a cache hit, not a measurement of current provider capacity. Gateway-generated API-key quota headers are a separate contract. A test of fresh provider quota or timing must use the live-provider policy; replay can still exercise how the proxy processes the recorded headers
## Qualification
`tests/code_coverage_tests/test_provider_cache.py` exercises local HTTP providers and disposable real Redis. CI runs these checks with the existing provider-edge and replay harness tests. These component checks do not establish Buildkite deployment, full-suite cross-build reuse or a genuine 24-hour expiry observation; those require separate runtime evidence
`tests/code_coverage_tests/test_provider_cache.py` exercises local HTTP providers and disposable real Redis, including the marker-canonical key, the FIFO slot index, per-test isolation, SigV4 re-signing against a local upstream, and each endpoint's completeness rule. CI runs these checks with the existing provider-edge and replay harness tests. These component checks do not establish Buildkite deployment, full-suite cross-build reuse or a genuine 24-hour expiry observation; those require separate runtime evidence

162
tests/e2e/claude_code/_driver_unit_tests/test_request_determinism.py Normal file
View file

@ -0,0 +1,162 @@
"""The CLI must send the same request bytes from one build to the next.
Markerless harness test: it drives the real `claude` binary against a local
stub instead of a proxy, so it carries no `e2e` marker. The binary is a
prerequisite of this whole suite, so a missing one is a failure rather than a
skip.
Two builds differ in ways the driver does not control: a fresh pod, so no CLI
state survives, and a different candidate checked out at a different commit.
Both used to reach the request body, through the memory path the system prompt
names and through the git block the CLI adds for its working directory, so the
shared provider cache missed on every Claude Code cell. This replays those two
differences across a pair of invocations and holds the bytes equal.
A pinned session id is what makes the second test necessary. The matrix runs
its cells across xdist workers, and the CLI refuses to start a session id that
another live process already holds, so pinning one without also opting out of
session persistence turns most of a parallel run red.
"""
from __future__ import annotations
import json
import os
import shutil
import subprocess
import threading
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import List, Tuple
import pytest
from claude_code.cli_driver import _FIXED_CLI_USER_ID, _seed_cli_identity, _stable_cli_state, run_claude
from claude_code.rate_limiter import RateLimiter
pytestmark = pytest.mark.cli_determinism
_STUB_REPLY = {
"id": "msg_stub",
"type": "message",
"role": "assistant",
"model": "claude-haiku-4-5",
"content": [{"type": "text", "text": "ok"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 10, "output_tokens": 2},
}
def _make_repo(root: Path, subject: str) -> Path:
root.mkdir(parents=True, exist_ok=True)
identity = {"NAME": "t", "EMAIL": "t@e2e"}
env = dict(
os.environ,
**{f"GIT_{role}_{key}": value for role in ("AUTHOR", "COMMITTER") for key, value in identity.items()},
)
(root / "file.txt").write_text(subject, encoding="utf-8")
for args in (["init", "-q"], ["add", "."], ["commit", "-q", "-m", subject]):
subprocess.run(["git", *args], cwd=root, env=env, check=True, capture_output=True)
return root
@pytest.fixture(name="captured")
def _captured() -> Tuple[str, List[bytes]]:
bodies: List[bytes] = []
lock = threading.Lock()
class Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def do_POST(self) -> None:
raw = self.rfile.read(int(self.headers.get("content-length") or 0))
if "count_tokens" not in self.path:
with lock:
bodies.append(raw)
payload = json.dumps({"input_tokens": 10} if "count_tokens" in self.path else _STUB_REPLY).encode()
self.send_response(200)
self.send_header("content-type", "application/json")
self.send_header("content-length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
def log_message(self, *_args: object) -> None:
return
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
threading.Thread(target=server.serve_forever, daemon=True).start()
try:
yield f"http://127.0.0.1:{server.server_address[1]}", bodies
finally:
server.shutdown()
def test_two_builds_send_the_same_request_bytes(captured: Tuple[str, List[bytes]], tmp_path: Path) -> None:
base_url, bodies = captured
limiter = RateLimiter(state_dir=tmp_path / "limiter")
checkouts = (_make_repo(tmp_path / "build-1", "first"), _make_repo(tmp_path / "build-2", "second"))
origin = Path.cwd()
sent = []
for checkout in checkouts:
shutil.rmtree(Path(_stable_cli_state()[0]).parent, ignore_errors=True)
os.chdir(checkout)
try:
before = len(bodies)
run_claude(
prompt="say ok",
model="claude-haiku-4-5",
base_url=base_url,
api_key="stub",
extra_env={"CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1"},
rate_limiter=limiter,
)
sent.append(bodies[before:])
finally:
os.chdir(origin)
assert sent[0], "the CLI sent no request to the stub, so there is nothing to compare"
assert sent[0] == sent[1]
def test_concurrent_cells_do_not_collide_on_the_pinned_session(
captured: Tuple[str, List[bytes]], tmp_path: Path
) -> None:
base_url, bodies = captured
limiter = RateLimiter(state_dir=tmp_path / "limiter")
def one(_index: int) -> int:
return run_claude(
prompt="say ok",
model="claude-haiku-4-5",
base_url=base_url,
api_key="stub",
extra_env={"CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1"},
rate_limiter=limiter,
).exit_code
with ThreadPoolExecutor(max_workers=4) as pool:
codes = list(pool.map(one, range(4)))
assert codes == [0, 0, 0, 0]
assert bodies, "the CLI sent no request to the stub, so there is nothing to compare"
assert set(Counter(bodies).values()) == {4}
def test_seeding_the_device_id_survives_threads_racing_on_the_same_directory(tmp_path: Path) -> None:
"""`run_claude_models_parallel` drives several models from one process, so the
seed's staged file has to be unique per thread and not merely per process."""
config_dir = tmp_path / "config"
config_dir.mkdir()
seeded = config_dir / ".claude.json"
for _round in range(20):
seeded.unlink(missing_ok=True)
with ThreadPoolExecutor(max_workers=16) as pool:
for outcome in [pool.submit(_seed_cli_identity, str(config_dir)) for _ in range(16)]:
outcome.result()
assert json.loads(seeded.read_text(encoding="utf-8"))["userID"] == _FIXED_CLI_USER_ID
assert sorted(entry.name for entry in config_dir.iterdir()) == [".claude.json"]

62
tests/e2e/claude_code/cli_driver.py
View file

@ -132,6 +132,62 @@ def _make_isolated_home() -> str:
return tempfile.mkdtemp(prefix="claude-cli-home-")
_FIXED_CLI_USER_ID = "0" * 64
_FIXED_CLI_SESSION_ID = "00000000-0000-4000-8000-000000000000"
def _seed_cli_identity(config_dir: str) -> None:
"""Pin the device id the CLI would otherwise mint per config directory.
It mints 32 random bytes on first run, writes them to `.claude.json` as
`userID`, and sends them in `metadata.user_id` forever after, so the value
is stable for exactly as long as that file lives. Pinning it, and the
session id passed beside it, costs nothing: both feed abuse detection
rather than quota, caching or continuity.
The staged name has to be unique per *thread*, not per process:
`run_claude_models_parallel` drives several models from one process, so a
pid-suffixed name lets one thread rename the file another is still
writing, and the loser dies on a missing path."""
path = os.path.join(config_dir, ".claude.json")
try:
with open(path, encoding="utf-8") as handle:
if json.load(handle).get("userID") == _FIXED_CLI_USER_ID:
return
except (OSError, ValueError):
pass
handle_fd, staged = tempfile.mkstemp(dir=config_dir, prefix=".claude.json.")
with os.fdopen(handle_fd, "w", encoding="utf-8") as handle:
json.dump({"userID": _FIXED_CLI_USER_ID}, handle)
os.replace(staged, path)
def _stable_cli_state() -> Tuple[str, str]:
"""Config directory and working directory for the CLI, at fixed paths.
Both reach the request body. The memory directory the system prompt
names is `$CLAUDE_CONFIG_DIR/projects/<cwd slug>/memory`, and a working
directory inside a git repository also contributes its branch and recent
commits. So a per-invocation config directory rewrites every body, and
inheriting the checkout rewrites every body once per candidate, which is
why the shared provider cache could never serve a Claude Code cell.
Pinning both makes the bodies repeatable across builds.
This narrows what survives rather than widening it: HOME stays fresh and
empty per invocation, so the isolation `_make_isolated_home` describes is
unchanged, and the CLI's own state no longer outlives the pod either. The
working directory is deliberately not the checkout, so a model-directed
`Read` sees an empty directory instead of the repository.
"""
root = os.path.join(tempfile.gettempdir(), f"litellm-e2e-claude-{os.getuid()}")
config_dir = os.path.join(root, "config")
workspace = os.path.join(root, "workspace")
for path in (root, config_dir, workspace):
os.makedirs(path, mode=0o700, exist_ok=True)
_seed_cli_identity(config_dir)
return config_dir, workspace
class ClaudeCLIError(RuntimeError):
"""Raised when the `claude` CLI cannot be invoked or returns a fatal error."""
@ -222,6 +278,9 @@ def run_claude(
"--verbose",
"--model",
model,
"--session-id",
_FIXED_CLI_SESSION_ID,
"--no-session-persistence",
]
if extra_args:
cmd.extend(extra_args)
@ -244,6 +303,8 @@ def run_claude(
# regardless of how the subprocess exits.
isolated_home = _make_isolated_home()
env["HOME"] = isolated_home
config_dir, workspace = _stable_cli_state()
env["CLAUDE_CONFIG_DIR"] = config_dir
if extra_env:
env.update(extra_env)
@ -262,6 +323,7 @@ def run_claude(
completed = run_fn(
cmd,
env=env,
cwd=workspace,
input=stdin_input,
capture_output=True,
text=True,

13
tests/e2e/conftest.py
View file

@ -23,6 +23,7 @@ from typing import Final
import pytest
import requests
from e2e_config import (
CLI_DETERMINISM_OPT_IN_ENV,
CONTROL_PLANE_BASE_URL,
FIXTURE_DIR,
FIXTURE_MODE_RAW,
@ -53,6 +54,7 @@ OPT_IN_MARKERS: Final = MappingProxyType(
"managed_files": MANAGED_FILES_OPT_IN_ENV,
"prompt_caching_stack": PROMPT_CACHING_OPT_IN_ENV,
"redis_chaos": REDIS_CHAOS_OPT_IN_ENV,
"cli_determinism": CLI_DETERMINISM_OPT_IN_ENV,
}
)
@ -85,7 +87,11 @@ def jwt_identity(idp: Keycloak, resources: ResourceManager, proxy: ProxyClient)
def pytest_configure(config: pytest.Config) -> None:
config.addinivalue_line("markers", "provider_live: requires actual provider timing, limits or state; bypass shared cache")
config.addinivalue_line(
"markers",
"provider_live: requires actual provider timing, limits, state, or a response that echoes this"
" run's own unique value; bypass shared cache",
)
config.addinivalue_line(
"markers",
"e2e: live test that requires a running proxy and real provider keys",
@ -116,6 +122,11 @@ def pytest_configure(config: pytest.Config) -> None:
"prompt_caching_stack: needs a proxy running with router_settings.optional_pre_call_checks including "
"prompt_caching; deselected unless E2E_PROMPT_CACHING_STACK is set",
)
config.addinivalue_line(
"markers",
"cli_determinism: drives the real claude CLI for several seconds, which widens the window in which "
"another test's in-flight upstream call is attributed to it; deselected unless E2E_CLI_DETERMINISM is set",
)
config.addinivalue_line(
"markers",
"redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from "

1
tests/e2e/e2e_config.py
View file

@ -145,6 +145,7 @@ WEEKLY_ANOMALY_OPT_IN_ENV = "E2E_WEEKLY_ANOMALY"
MANAGED_FILES_OPT_IN_ENV = "E2E_MANAGED_FILES_STACK"
PROMPT_CACHING_OPT_IN_ENV = "E2E_PROMPT_CACHING_STACK"
REDIS_CHAOS_OPT_IN_ENV = "E2E_REDIS_CHAOS"
CLI_DETERMINISM_OPT_IN_ENV = "E2E_CLI_DETERMINISM"
ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6"))
ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6"))
ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3"))

5
tests/e2e/fixture_canonical.py
View file

@ -51,6 +51,9 @@ SECRET_FIELD_SUFFIXES: Final[tuple[str, ...]] = (
)
SECRET_PLACEHOLDER: Final = "<secret>"
MARKER_PATTERN: Final = re.compile(r"(?<![0-9a-fA-F])[0-9a-f]{12}(?![0-9a-fA-F])")
MARKER_PLACEHOLDER: Final = "<marker>"
PLACEHOLDER_RULES: Final[tuple[tuple[re.Pattern[str], str], ...]] = (
(re.compile(r"(?<![0-9a-fA-F])[0-9a-f]{64}(?![0-9a-fA-F])"), "<sha256>"),
(
@ -67,7 +70,7 @@ PLACEHOLDER_RULES: Final[tuple[tuple[re.Pattern[str], str], ...]] = (
re.compile(r"\b(?:chatcmpl|msgbatch|msg|resp|batch|call|req|ftjob|gen|file)[-_][A-Za-z0-9]{8,}\b"),
"<id>",
),
(re.compile(r"(?<![0-9a-fA-F])[0-9a-f]{12}(?![0-9a-fA-F])"), "<marker>"),
(MARKER_PATTERN, MARKER_PLACEHOLDER),
)

1
tests/e2e/llm_translation/test_messages_e2e.py
View file

@ -372,6 +372,7 @@ def _request_tool(
class TestOpenAIMessagesToolContinuation:
@pytest.mark.provider_live
@pytest.mark.parametrize("stream", [True, False], ids=["stream", "nonstream"])
def test_required_tool_arguments_and_correlated_result(
self, endpoints_client: EndpointsClient, resources: ResourceManager, stream: bool

1
tests/e2e/models.py
View file

@ -951,6 +951,7 @@ class LiteLLMParamsBody(BaseModel):
aws_access_key_id: str | None = None
aws_secret_access_key: str | None = None
aws_region_name: str | None = None
aws_bedrock_runtime_endpoint: str | None = None
vertex_project: str | None = None
vertex_location: str | None = None
vertex_credentials: str | None = None

459
tests/e2e/provider_cache.py
View file

@ -4,14 +4,18 @@ import base64
import hashlib
import hmac
import io
import json
import os
import threading
import time
from collections.abc import Callable, Generator, Mapping
from contextlib import closing
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import Final, Literal, Protocol
from urllib.parse import urlsplit
from botocore.eventstream import EventStreamBuffer, ParserError
from e2e_http import (
NetworkError,
StreamChunk,
@ -23,12 +27,36 @@ from e2e_http import (
prepare_forward,
primed_steps,
)
from fixture_canonical import MARKER_PATTERN, MARKER_PLACEHOLDER
from fixture_mode import SESSION_TEST_KEY, current_test_key
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError
LIFETIME_SECONDS: Final = 86_400
MAX_REQUEST_BYTES: Final = 256 * 1024
MAX_RESPONSE_BYTES: Final = 8 * 1024 * 1024
UNRECORDED_RESPONSE_HEADERS: Final = frozenset({"set-cookie"})
SIGNATURE_HEADERS: Final = frozenset(
{"authorization", "x-amz-date", "x-amz-security-token", "x-amz-content-sha256"}
)
BEDROCK_MOUNT_PREFIX: Final = "bedrock"
BEDROCK_CONVERSE_SUFFIX: Final = "/converse"
BEDROCK_INVOKE_SUFFIX: Final = "/invoke"
BEDROCK_CONVERSE_STREAM_SUFFIX: Final = "/converse-stream"
BEDROCK_INVOKE_STREAM_SUFFIX: Final = "/invoke-with-response-stream"
BEDROCK_SUFFIXES: Final = (
BEDROCK_CONVERSE_SUFFIX,
BEDROCK_INVOKE_SUFFIX,
BEDROCK_CONVERSE_STREAM_SUFFIX,
BEDROCK_INVOKE_STREAM_SUFFIX,
)
EVENTSTREAM_PRELUDE_BYTES: Final = 4
CUT_SHORT: Final = "cut_short"
INCOMPLETE: Final = "incomplete"
UNREACHABLE: Final = "unreachable"
ERROR_STATUS: Final = "error_status"
EVENT_TYPE_HEADER: Final = ":event-type"
EVENTSTREAM_HEADERS: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])
OPENAI_JSON_PATHS: Final = frozenset({"/v1/chat/completions", "/v1/messages", "/v1/embeddings", "/v1/responses"})
JSON_VALUE: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
@ -56,6 +84,24 @@ class CacheUnavailable:
type CacheLookup = CacheHit | CaptureLease | CacheBusy | CacheUnavailable
type RequestSigner = Callable[[str, str, Mapping[str, str], bytes | None], dict[str, str]]
@dataclass(frozen=True, slots=True)
class MountPolicy:
"""What a mount needs beyond plain forwarding.
``sign`` mints a fresh credential over the upstream URL, for providers whose
auth covers the Host the edge rewrote. ``unkeyed_headers`` names headers that
must stay out of the cache key because they change on every call and would
otherwise make the mount a permanent miss: a minted signature, or an OAuth
token the provider rotates. Naming one costs the guarantee that a recording
can never cross credentials, so a mount with a rotating token relies on the
environment holding one identity for that provider. Mounts with a static API
key name nothing here and keep the guarantee whole."""
sign: RequestSigner | None = None
unkeyed_headers: frozenset[str] = frozenset()
class ResponseStore(Protocol):
@ -83,28 +129,51 @@ class SignedResponse(BaseModel):
signature: str
def exact_key(secret: bytes, method: str, url: str, headers: Mapping[str, str], body: bytes | None) -> str:
def canonical_text(value: str) -> str:
return MARKER_PATTERN.sub(MARKER_PLACEHOLDER, value)
def canonical_body(body: bytes) -> bytes:
try:
return canonical_text(body.decode("utf-8")).encode("utf-8")
except UnicodeDecodeError:
return body
def request_identity(
secret: bytes, test_key: str, method: str, url: str, headers: Mapping[str, str], body: bytes | None,
) -> str:
fields: Final = (
b"provider-cache-exact-v1", method.encode(), url.encode(),
b"provider-cache-canonical-v2", test_key.encode(), method.encode(), canonical_text(url).encode(),
*(part.encode() for pair in sorted(headers.items()) for part in pair),
b"no-body" if body is None else b"body", b"" if body is None else body,
b"no-body" if body is None else b"body", b"" if body is None else canonical_body(body),
)
encoded: Final = b"".join(len(part).to_bytes(8, "big") + part for part in fields)
return hmac.new(secret, encoded, hashlib.sha256).hexdigest()
def cacheable_endpoint(method: str, url: str, body: bytes | None) -> bool:
return (
method == "POST"
and urlsplit(url).path in {"/v1/chat/completions", "/v1/messages"}
and body is not None
and len(body) <= MAX_REQUEST_BYTES
)
def slotted_key(secret: bytes, identity: str, slot: int) -> str:
return hmac.new(secret, f"{identity}:{slot}".encode(), hashlib.sha256).hexdigest()
def successful_response(url: str, status: int, headers: Mapping[str, str], body: bytes) -> bool:
def is_bedrock(mount: str) -> bool:
return mount.partition("/")[0] == BEDROCK_MOUNT_PREFIX
def cacheable_endpoint(mount: str, method: str, url: str, body: bytes | None) -> bool:
if method != "POST" or body is None or len(body) > MAX_REQUEST_BYTES:
return False
path: Final = urlsplit(url).path
if is_bedrock(mount):
return path.startswith("/model/") and path.endswith(BEDROCK_SUFFIXES)
return path in OPENAI_JSON_PATHS
def successful_response(mount: str, url: str, status: int, headers: Mapping[str, str], body: bytes) -> bool:
if not 200 <= status < 300 or len(body) > MAX_RESPONSE_BYTES:
return False
if is_bedrock(mount):
return complete_bedrock_response(url, body)
streaming: Final = "text/event-stream" in headers.get("content-type", "").lower()
if streaming:
try:
@ -118,28 +187,33 @@ def successful_response(url: str, status: int, headers: Mapping[str, str], body:
values: Final = tuple(JSON_VALUE.validate_json(event) for event in events if event != "[DONE]")
except (UnicodeDecodeError, ValidationError):
return False
if not values or any(not isinstance(value, dict) or "error" in value or value.get("type") == "error" for value in values):
if not values or any(
not isinstance(value, dict) or value.get("error") is not None or value.get("type") == "error"
for value in values
):
return False
if urlsplit(url).path == "/v1/responses":
return complete_responses_stream(values)
if urlsplit(url).path == "/v1/chat/completions":
return events[-1] == "[DONE]" and "[DONE]" not in events[:-1] and complete_chat_stream(values)
return (
"[DONE]" not in events
and isinstance(values[0], dict) and values[0].get("type") == "message_start"
and isinstance(values[-1], dict) and values[-1].get("type") == "message_stop"
and any(
isinstance(value, dict) and value.get("type") == "message_delta"
and isinstance(delta := value.get("delta"), dict) and isinstance(delta.get("stop_reason"), str)
for value in values
)
)
return "[DONE]" not in events and complete_anthropic_stream(values)
try:
value: Final = JSON_VALUE.validate_json(body)
except ValidationError:
return False
if not isinstance(value, dict) or "error" in value:
if not isinstance(value, dict) or value.get("error") is not None:
return False
if urlsplit(url).path == "/v1/messages":
path: Final = urlsplit(url).path
if path == "/v1/messages":
return value.get("type") == "message" and isinstance(value.get("content"), list) and isinstance(value.get("stop_reason"), str)
if path == "/v1/embeddings":
data: Final = value.get("data")
return isinstance(data, list) and bool(data) and isinstance(value.get("usage"), dict) and all(
isinstance(item, dict) and isinstance(item.get("embedding"), list) and bool(item["embedding"])
for item in data
)
if path == "/v1/responses":
return value.get("object") == "response" and value.get("status") == "completed"
choices: Final = value.get("choices")
return isinstance(choices, list) and bool(choices) and all(
isinstance(choice, dict) and isinstance(choice.get("message"), dict) and isinstance(choice.get("finish_reason"), str)
@ -147,6 +221,144 @@ def successful_response(url: str, status: int, headers: Mapping[str, str], body:
)
def complete_bedrock_response(url: str, body: bytes) -> bool:
"""Converse answers with ``output`` plus a ``stopReason``; InvokeModel on an
Anthropic model answers the Anthropic message shape. Either way a truncated
or error body is missing the terminator field, which is what makes it safe to
record."""
path: Final = urlsplit(url).path
if path.endswith(BEDROCK_CONVERSE_STREAM_SUFFIX):
return complete_converse_stream(body)
if path.endswith(BEDROCK_INVOKE_STREAM_SUFFIX):
return complete_invoke_stream(body)
try:
value: Final = JSON_VALUE.validate_json(body)
except ValidationError:
return False
if not isinstance(value, dict) or "message" in value:
return False
if path.endswith(BEDROCK_CONVERSE_SUFFIX):
return isinstance(value.get("output"), dict) and isinstance(value.get("stopReason"), str)
return (
value.get("type") == "message"
and isinstance(value.get("content"), list)
and isinstance(value.get("stop_reason"), str)
)
def whole_eventstream_messages(body: bytes) -> bool:
"""Whether the body is exactly a whole number of eventstream messages.
A dropped connection is the failure this catches, and it has to be caught
here: botocore yields the messages it did receive and silently discards a
trailing partial one, so a stream cut a single byte short parses clean. Each
message declares its own total length in its first four bytes, so walking
those is enough to tell a complete body from a cut one."""
offset = 0 # rebind-ok: a cursor walking the declared frame lengths
while offset + EVENTSTREAM_PRELUDE_BYTES <= len(body):
total: int = int.from_bytes(body[offset : offset + EVENTSTREAM_PRELUDE_BYTES], "big")
if total <= 0 or offset + total > len(body):
return False
offset += total
return offset == len(body)
def eventstream_events(body: bytes) -> tuple[tuple[str, JsonValue], ...] | None:
"""The stream's (event type, decoded payload) pairs, or None if it is not a
complete, uncorrupted stream.
botocore validates both CRCs and raises ``ParserError`` rather than decoding
corruption into something plausible. A failure that began after Bedrock had
already answered 200 arrives as an ``exception`` frame in place of the
terminator, so it is the terminator rules below that reject it and this does
not need to inspect ``:message-type`` as well."""
if not body or not whole_eventstream_messages(body):
return None
buffer: Final = EventStreamBuffer()
buffer.add_data(body)
try:
return tuple(
(event_type(event.headers), JSON_VALUE.validate_json(event.payload))
for event in buffer
)
except (ParserError, ValidationError, ValueError):
return None
def event_type(headers: object) -> str:
"""botocore's eventstream headers come back untyped, so the one header this
reads is validated into a string rather than trusted."""
parsed: Final = EVENTSTREAM_HEADERS.validate_python(headers)
return parsed.get(EVENT_TYPE_HEADER, "")
def complete_converse_stream(body: bytes) -> bool:
"""ConverseStream ends with ``metadata``, not with ``messageStop``.
Requiring the metadata frame rather than the stop frame is deliberate: it
carries the token usage litellm prices the call from, so a stream cut between
the two still names a stop reason but would replay as a free call."""
events: Final = eventstream_events(body)
if not events or events[-1][0] != "metadata":
return False
return any(
event_type == "messageStop" and isinstance(payload, dict) and isinstance(payload.get("stopReason"), str)
for event_type, payload in events
)
def complete_invoke_stream(body: bytes) -> bool:
"""InvokeModelWithResponseStream wraps the ordinary Anthropic event grammar
in ``chunk`` frames, one base64 payload each, so it is held to the same
terminator rule as the Anthropic SSE path. A frame Bedrock sends instead of a
chunk, an exception among them, carries no such payload and fails the rule
without the frame type needing to be read."""
events: Final = eventstream_events(body)
if not events:
return False
values: Final = tuple(invoke_chunk_value(payload) for _, payload in events)
return all(value is not None for value in values) and complete_anthropic_stream(values)
def invoke_chunk_value(payload: JsonValue) -> JsonValue | None:
"""The Anthropic event inside one ``chunk`` frame, or None for a frame that
carries no readable one."""
if not isinstance(payload, dict) or not isinstance(encoded := payload.get("bytes"), str):
return None
try:
return JSON_VALUE.validate_json(base64.b64decode(encoded, validate=True))
except (ValidationError, ValueError):
return None
def complete_anthropic_stream(values: tuple[JsonValue, ...]) -> bool:
"""The Anthropic event grammar, shared by the SSE mounts and by Bedrock's
invoke stream, which carries the same events inside eventstream frames. A
``message_delta`` naming a stop reason is what separates a finished turn from
one the connection cut short."""
if not values:
return False
first: Final = values[0]
last: Final = values[-1]
return (
isinstance(first, dict) and first.get("type") == "message_start"
and isinstance(last, dict) and last.get("type") == "message_stop"
and any(
isinstance(value, dict) and value.get("type") == "message_delta"
and isinstance(delta := value.get("delta"), dict) and isinstance(delta.get("stop_reason"), str)
for value in values
)
)
def complete_responses_stream(values: tuple[JsonValue, ...]) -> bool:
"""The Responses API streams typed events and ends with ``response.completed``.
A run that failed, was cancelled, or ran out of tokens ends with a different
terminal event, so requiring that one keeps a half-finished response out."""
last: Final = values[-1]
return isinstance(last, dict) and last.get("type") == "response.completed"
def complete_chat_stream(values: tuple[JsonValue, ...]) -> bool:
if any(not isinstance(value, dict) or not isinstance(value.get("choices"), list) for value in values):
return False
@ -172,7 +384,7 @@ def encode_response(secret: bytes, response: CachedResponse) -> bytes:
return SignedResponse(response=raw, signature=hmac.new(secret, raw.encode(), hashlib.sha256).hexdigest()).model_dump_json().encode()
def decode_response(secret: bytes, key: str, payload: bytes, url: str) -> CachedResponse | None:
def decode_response(secret: bytes, key: str, payload: bytes, mount: str, url: str) -> CachedResponse | None:
if len(payload) > 2 * MAX_RESPONSE_BYTES:
return None
try:
@ -183,11 +395,58 @@ def decode_response(secret: bytes, key: str, payload: bytes, url: str) -> Cached
chunks: Final = tuple(base64.b64decode(chunk, validate=True) for chunk in response.chunks)
except (ValidationError, ValueError):
return None
if response.request_key != key or not successful_response(url, response.status_code, response.headers, b"".join(chunks)):
if response.request_key != key or not successful_response(
mount, url, response.status_code, response.headers, b"".join(chunks)
):
return None
return response
def component_digests(
test_key: str, method: str, url: str, headers: Mapping[str, str], body: bytes | None,
) -> dict[str, str]:
"""Per-component digests of everything the key covers.
A mount whose corpus never converges is a mount where one of these moves
between builds, and the flat key cannot say which. Values are digested, so
no payload or credential is written, and a JSON body contributes one digest
per top-level field so the field that moved can be named."""
parts: dict[str, str] = { # rebind-ok: a report assembled from three differently shaped sources
"test_key": test_key,
"method": method,
"url": short_digest(canonical_text(url).encode()),
}
for name, value in sorted(headers.items()):
parts[f"header:{name.lower()}"] = short_digest(value.encode())
canonical: Final = b"" if body is None else canonical_body(body)
parts["body"] = short_digest(canonical)
try:
parsed: Final = JSON_VALUE.validate_json(canonical)
except ValidationError:
return parts
if isinstance(parsed, dict):
for name, value in sorted(parsed.items()):
parts[f"body:{name}"] = short_digest(json.dumps(value, sort_keys=True).encode())
return parts
def short_digest(value: bytes) -> str:
return hashlib.sha256(value).hexdigest()[:16]
@dataclass(slots=True)
class KeyProbe:
"""Every keyed request's components, when a metrics directory is configured."""
rows: tuple[tuple[tuple[str, str], ...], ...] = ()
lock: threading.Lock = field(default_factory=threading.Lock)
def observe(self, mount: str, outcome: str, parts: Mapping[str, str]) -> None:
row: Final = tuple({"mount": mount, "outcome": outcome, **parts}.items())
with self.lock:
self.rows = (*self.rows, row)
@dataclass(slots=True)
class CacheCounters:
counts: tuple[tuple[str, int], ...] = ()
@ -199,6 +458,24 @@ class CacheCounters:
self.counts = tuple((current | {name: current.get(name, 0) + 1}).items())
@dataclass(slots=True)
class SlotCounter:
"""FIFO position of a request among the canonically identical ones its test
has already sent. Two calls in one test that differ only by ``unique_marker``
canonicalize the same, so without this they would share one recording and the
second would replay the first's provider response id."""
counts: tuple[tuple[str, int], ...] = ()
lock: threading.Lock = field(default_factory=threading.Lock)
def take(self, identity: str) -> int:
with self.lock:
current: Final = dict(self.counts)
taken: Final = current.get(identity, 0)
self.counts = tuple((current | {identity: taken + 1}).items())
return taken
@dataclass(slots=True)
class ResponseCapture:
buffer: io.BytesIO = field(default_factory=io.BytesIO)
@ -226,14 +503,21 @@ def response_steps(response: CachedResponse) -> Generator[StreamStep, None, None
yield StreamChunk(base64.b64decode(chunk, validate=True))
NO_POLICIES: Final[Mapping[str, MountPolicy]] = MappingProxyType({})
@dataclass(frozen=True, slots=True)
class CacheEdge:
store: ResponseStore
secret: bytes = field(repr=False)
counters: CacheCounters = field(default_factory=CacheCounters)
probe: KeyProbe = field(default_factory=KeyProbe)
slots: SlotCounter = field(default_factory=SlotCounter)
policies: Mapping[str, MountPolicy] = NO_POLICIES
wait_seconds: float = 2.0
clock: Callable[[], float] = time.monotonic
sleep: Callable[[float], None] = time.sleep
test_key: Callable[[], str] = current_test_key
def lookup(self, key: str) -> CacheLookup:
deadline: Final = self.clock() + self.wait_seconds
@ -241,59 +525,122 @@ class CacheEdge:
self.sleep(min(0.05, max(0, deadline - self.clock())))
return result
def forward(self, method: str, url: str, headers: dict[str, str], body: bytes | None, timeout: float) -> StreamHead | NetworkError:
if not cacheable_endpoint(method, url, body):
self.counters.increment("bypass")
self.counters.increment("upstream_attempts")
return forward_stream(method, url, headers=headers, body=body, timeout=timeout)
prepared: Final = prepare_forward(method, url, headers, body)
def count(self, mount: str, name: str) -> None:
self.counters.increment(name)
self.counters.increment(f"mount:{mount}:{name}")
def record_key(
self, mount: str, outcome: str, test_key: str, method: str, url: str,
headers: Mapping[str, str], body: bytes | None,
) -> None:
if not os.environ.get("E2E_PROVIDER_CACHE_METRICS_DIR"):
return
self.probe.observe(mount, outcome, component_digests(test_key, method, url, headers, body))
def outbound(self, mount: str, method: str, url: str, headers: dict[str, str], body: bytes | None) -> dict[str, str]:
"""The headers actually sent upstream. A signing mount gets a signature
minted over the upstream URL, because the edge rewrote the Host the proxy
signed and the provider verifies it."""
signer: Final = self.policies.get(mount, MountPolicy()).sign
return headers if signer is None else signer(method, url, headers, body)
def keyed(self, mount: str, headers: Mapping[str, str]) -> Mapping[str, str]:
"""Headers the cache key is built from. A mount keeps its credentials in
the key unless its policy names them unkeyed, so by default one account
can never read another's recording."""
unkeyed: Final = self.policies.get(mount, MountPolicy()).unkeyed_headers
if not unkeyed:
return headers
return {name: value for name, value in headers.items() if name.lower() not in unkeyed}
def forward(
self, mount: str, method: str, url: str, headers: dict[str, str], body: bytes | None, timeout: float,
) -> StreamHead | NetworkError:
test_key: Final = self.test_key()
if test_key == SESSION_TEST_KEY or not cacheable_endpoint(mount, method, url, body):
self.count(mount, "bypass")
self.count(mount, "upstream_attempts")
return forward_stream(
method, url, headers=self.outbound(mount, method, url, headers, body), body=body, timeout=timeout,
)
prepared: Final = prepare_forward(method, url, self.outbound(mount, method, url, headers, body), body)
if isinstance(prepared, NetworkError):
self.counters.increment("rejected")
self.reject(mount, UNREACHABLE)
return prepared
key: Final = exact_key(self.secret, method, url, prepared.headers, body)
keyed_headers: Final = self.keyed(mount, prepared.headers)
identity: Final = request_identity(self.secret, test_key, method, url, keyed_headers, body)
key: Final = slotted_key(self.secret, identity, self.slots.take(identity))
found: Final = self.lookup(key)
if isinstance(found, CacheHit):
response: Final = decode_response(self.secret, key, found.payload, url)
response: Final = decode_response(self.secret, key, found.payload, mount, url)
if response is not None and self.clock() < found.valid_until:
self.counters.increment("hits")
self.count(mount, "hits")
self.record_key(mount, "hit", test_key, method, url, keyed_headers, body)
return StreamHead(response.status_code, response.headers, response_steps(response))
self.counters.increment("corrupt" if response is None else "expired")
self.count(mount, "corrupt" if response is None else "expired")
self.store.discard(key, found.payload)
capture_slot: Final = self.lookup(key) if isinstance(found, CacheHit) else found
self.counters.increment("misses")
self.count(mount, "misses")
self.record_key(mount, "miss", test_key, method, url, keyed_headers, body)
if isinstance(capture_slot, CacheUnavailable):
self.counters.increment("cache_errors")
self.counters.increment("upstream_attempts")
self.count(mount, "cache_errors")
self.count(mount, "upstream_attempts")
head: Final = forward_prepared_stream(prepared, timeout)
if not isinstance(capture_slot, CaptureLease):
return head
if isinstance(head, NetworkError):
self.store.release(key, capture_slot)
self.counters.increment("rejected")
self.reject(mount, UNREACHABLE)
return head
return StreamHead(head.status_code, head.headers, primed_steps(self.capture(key, capture_slot, url, head)))
return StreamHead(
head.status_code, head.headers, primed_steps(self.capture(mount, key, capture_slot, url, head)),
)
def capture(self, key: str, lease: CaptureLease, url: str, head: StreamHead) -> Generator[StreamStep, None, None]:
def capture(
self, mount: str, key: str, lease: CaptureLease, url: str, head: StreamHead,
) -> Generator[StreamStep, None, None]:
capture: Final = ResponseCapture()
reason = CUT_SHORT # rebind-ok: a consumer that walks away never reaches the settle call below
try:
with closing(head.steps):
yield StreamChunk(b"")
for step in head.steps:
yield step
capture.observe(step)
chunks: Final = capture.chunks() if capture.eligible else ()
headers: Final = {
name: value for name, value in head.headers.items() if name.lower() not in UNRECORDED_RESPONSE_HEADERS
}
if not capture.eligible or not successful_response(url, head.status_code, headers, b"".join(chunks)):
self.counters.increment("rejected")
return
response: Final = CachedResponse(
request_key=key, status_code=head.status_code, headers=headers,
chunks=tuple(base64.b64encode(chunk).decode("ascii") for chunk in chunks),
)
published: Final = self.store.publish(key, lease, encode_response(self.secret, response))
self.counters.increment("writes" if published else "write_failures")
reason = self.settle(mount, key, lease, url, head, capture)
finally:
self.reject(mount, reason)
self.store.release(key, lease)
capture.buffer.close()
def settle(
self, mount: str, key: str, lease: CaptureLease, url: str, head: StreamHead, capture: ResponseCapture,
) -> str | None:
"""None once the response is stored, otherwise the reason it was not."""
if not capture.eligible:
return CUT_SHORT
headers: Final = {
name: value for name, value in head.headers.items() if name.lower() not in UNRECORDED_RESPONSE_HEADERS
}
if not 200 <= head.status_code < 300:
return ERROR_STATUS
chunks: Final = capture.chunks()
if not successful_response(mount, url, head.status_code, headers, b"".join(chunks)):
return INCOMPLETE
response: Final = CachedResponse(
request_key=key, status_code=head.status_code, headers=headers,
chunks=tuple(base64.b64encode(chunk).decode("ascii") for chunk in chunks),
)
published: Final = self.store.publish(key, lease, encode_response(self.secret, response))
self.count(mount, "writes" if published else "write_failures")
return None
def reject(self, mount: str, reason: str | None) -> None:
"""A flat rejection count cannot separate a connection that went away from
a body the provider finished sending and the rules turned down, and the two
have opposite fixes. A mount whose rejections are nearly all one or the
other is a different problem, so the report has to be able to say which."""
if reason is None:
return
self.count(mount, "rejected")
self.count(mount, f"rejected_{reason}")

4
tests/e2e/provider_cache_redis.py
View file

@ -134,6 +134,10 @@ def write_metrics(cache: CacheEdge) -> None:
root: Final = Path(directory)
root.mkdir(parents=True, exist_ok=True)
(root / f"{os.getpid()}.json").write_text(report + "\n")
if cache.probe.rows:
(root / f"keys-{os.getpid()}.json").write_text(
json.dumps([dict(row) for row in cache.probe.rows]) + "\n"
)
except OSError:
logging.getLogger(__name__).warning("provider cache metrics artifact unavailable")
logging.getLogger(__name__).info("%s", report)

70
tests/e2e/provider_cache_routing.py
View file

@ -8,14 +8,80 @@ from models import LiteLLMParamsBody, ModelMode
LIVE_PROVIDER_REQUIRED: Final[ContextVar[bool]] = ContextVar("live_provider_required", default=False)
DEFAULT_BEDROCK_REGION: Final = "us-east-1"
BEDROCK_CROSS_REGION_PREFIX: Final = "us."
BEDROCK_EDGE_MODELS: Final = frozenset(
{
"us.anthropic.claude-haiku-4-5-20251001-v1:0",
"us.anthropic.claude-sonnet-4-5-20250929-v1:0",
"us.anthropic.claude-sonnet-5",
"us.anthropic.claude-opus-4-7",
}
)
ENV_REFERENCE_PREFIX: Final = "os.environ/"
def bedrock_region(declared: str | None) -> str:
"""The region whose edge mount a deployment belongs to.
Most Bedrock deployments declare `os.environ/AWS_REGION`, which only the
proxy can resolve from its own environment; the run pod does not share it.
Answering those with the default mount is correct because every model on the
edge allowlist is a `us.` inference profile, which fans out across the US
regions and is reachable from any of them. That invariant is enforced on the
allowlist itself rather than re-checked per call."""
if declared is None or declared.startswith(ENV_REFERENCE_PREFIX):
return DEFAULT_BEDROCK_REGION
return declared
def bedrock_mount(params: LiteLLMParamsBody) -> str | None:
"""The edge mount a Bedrock deployment belongs to, or None.
The allowlist mirrors the runner role's IAM policy, which names its models
one by one. A model outside it would be re-signed with an identity that
cannot invoke it and come back 403 from Bedrock, so an unlisted model keeps
its direct path and loses only caching. Adding a model is a policy edit in
litellm-ops and a line here."""
route: Final = params.model.partition("/")[2]
model: Final = route.partition("/")[2] or route
if model not in BEDROCK_EDGE_MODELS:
return None
return f"bedrock/{bedrock_region(params.aws_region_name)}"
def route_bedrock(
params: LiteLLMParamsBody, base_for: Callable[[str], str | None], mode: ModelMode | None,
) -> LiteLLMParamsBody:
"""Deployments that carry their own AWS identity stay off the edge. The edge
re-signs with the run pod's role, so routing an `aws_role_name` deployment
would quietly replace the very assume-role chain that test exists to prove."""
if mode is not None or params.aws_role_name is not None or params.aws_access_key_id is not None:
return params
if params.api_base is not None or params.aws_bedrock_runtime_endpoint is not None:
return params
mount: Final = bedrock_mount(params)
if mount is None:
return params
base: Final = base_for(mount)
if base is None:
return params
return params.model_copy(update={"aws_bedrock_runtime_endpoint": base})
def route_cache_model(
params: LiteLLMParamsBody, base_for: Callable[[str], str | None], *, enabled: bool, mode: ModelMode | None = None,
) -> LiteLLMParamsBody:
if not enabled or mode == "realtime" or LIVE_PROVIDER_REQUIRED.get() or params.api_base is not None or params.mock_response is not None:
if not enabled or LIVE_PROVIDER_REQUIRED.get() or params.mock_response is not None:
return params
if params.litellm_credential_name is not None:
return params
provider: Final = params.model.partition("/")[0]
if provider not in {"openai", "anthropic"} or params.litellm_credential_name is not None:
if provider == "bedrock":
return route_bedrock(params, base_for, mode)
if mode == "realtime" or params.api_base is not None:
return params
if provider not in {"openai", "anthropic"}:
return params
base: Final = base_for(provider)
if base is None:

67
tests/e2e/provider_edge.py
View file

@ -48,7 +48,7 @@ import threading
from collections import deque
from collections.abc import Generator, Mapping, Sequence
from contextlib import closing, contextmanager
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from itertools import islice
from pathlib import Path
@ -94,17 +94,41 @@ from fixture_mode import (
parse_fixture_mode,
)
from fixture_profile import IneligibleRequest, MatchProfile, match_profile, strict_identity
from provider_cache import CacheEdge
from provider_cache import SIGNATURE_HEADERS, CacheEdge, MountPolicy, is_bedrock
from provider_cache_routing import LIVE_PROVIDER_REQUIRED
from pydantic import JsonValue, TypeAdapter
BEDROCK_REGIONS: Final[tuple[str, ...]] = ("us-east-1",)
EDGE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType(
{
"openai": "https://api.openai.com",
"anthropic": "https://api.anthropic.com",
**{
f"bedrock/{region}": f"https://bedrock-runtime.{region}.amazonaws.com"
for region in BEDROCK_REGIONS
},
}
)
@dataclass(frozen=True, slots=True)
class ResolvedMount:
mount: str
upstream_base: str
upstream_path: str
def resolve_mount(path: str, mounts: Mapping[str, str]) -> ResolvedMount | None:
"""Longest mount prefix wins, so a region-qualified mount such as
``bedrock/us-east-1`` resolves whole instead of leaving the region as the
first segment of the upstream path."""
trimmed: Final = path.lstrip("/")
for mount in sorted(mounts, key=len, reverse=True):
if trimmed == mount or trimmed.startswith(f"{mount}/"):
return ResolvedMount(mount, mounts[mount], trimmed[len(mount):].lstrip("/"))
return None
REPLAY_MISS_STATUS: Final = 599
_HOP_BY_HOP_HEADERS: Final[frozenset[str]] = frozenset(
@ -754,14 +778,14 @@ def _handle_record(
def _handle_live(
method: str, url: str, headers: Mapping[str, str], body: bytes | None, timeout: float,
cache: CacheEdge | None = None,
cache: CacheEdge | None = None, mount: str = "",
) -> EdgeOutcome:
forwarded: Final = {
name: value for name, value in headers.items() if name.lower() not in _REQUEST_DROPPED_HEADERS
}
head: Final = (
forward_stream(method, url, headers=forwarded, body=body, timeout=timeout)
if cache is None else cache.forward(method, url, forwarded, body, timeout)
if cache is None else cache.forward(mount, method, url, forwarded, body, timeout)
)
match head:
case NetworkError(message=message):
@ -796,10 +820,13 @@ def handle_edge_request(
prefix, then record (forward + persist) or replay (serve from the bundle).
Socket-free so unit tests exercise every branch without a server."""
split: Final = urlsplit(raw_path)
mount, _, upstream_path = split.path.lstrip("/").partition("/")
upstream_base: Final = mounts.get(mount)
if upstream_base is None:
return _text_reply(404, f"unknown provider mount {mount!r}; known mounts: {', '.join(sorted(mounts))}")
resolved: Final = resolve_mount(split.path, mounts)
if resolved is None:
unknown: Final = split.path.lstrip("/").partition("/")[0]
return _text_reply(404, f"unknown provider mount {unknown!r}; known mounts: {', '.join(sorted(mounts))}")
mount: Final = resolved.mount
upstream_base: Final = resolved.upstream_base
upstream_path: Final = resolved.upstream_path
profile: Final = (
backend.recorder.profile
if isinstance(backend, RecordEdge)
@ -830,7 +857,8 @@ def handle_edge_request(
match backend:
case CacheEdge():
return _handle_live(
method, _upstream_url(upstream_base, upstream_path, split.query), headers, body, timeout, backend,
method, _upstream_url(upstream_base, upstream_path, split.query), headers, body, timeout,
backend, mount,
)
case LiveEdge():
return _handle_live(
@ -891,7 +919,7 @@ class _EdgeHandler(BaseHTTPRequestHandler):
)
if isinstance(edge_server.backend, CacheEdge) and duplicate_headers:
edge_server.backend.counters.increment("duplicate_header_bypass")
if urlsplit(self.path).path.lstrip("/").partition("/")[0] in edge_server.mounts:
if resolve_mount(urlsplit(self.path).path, edge_server.mounts) is not None:
edge_server.backend.counters.increment("upstream_attempts")
outcome: Final = handle_edge_request(
selected_backend,
@ -1079,6 +1107,8 @@ def provider_edge_api_base(
return _shared_cache_edge(bind_host, advertise_host, forward_timeout).api_base(mount)
return None
case "record" | "replay":
if is_bedrock(mount):
return None
if mount not in EDGE_MOUNTS:
raise ValueError(f"unknown provider mount {mount!r}; known mounts: {', '.join(sorted(EDGE_MOUNTS))}")
return _shared_edge(mode, bundle_dir, bind_host, advertise_host, forward_timeout, match_profile()).api_base(
@ -1108,7 +1138,22 @@ def configured_cache_backend() -> CacheEdge | None:
return None
from provider_cache_redis import configured_cache
return configured_cache()
cache: Final = configured_cache()
return None if cache is None else replace(cache, policies=bedrock_policies())
@functools.lru_cache(maxsize=1)
def bedrock_policies() -> Mapping[str, MountPolicy]:
"""One policy per mounted Bedrock region, built lazily so a run that never
mounts Bedrock neither imports botocore nor resolves an AWS identity."""
from provider_edge_bedrock import bedrock_signer
return MappingProxyType(
{
f"bedrock/{region}": MountPolicy(sign=bedrock_signer(region), unkeyed_headers=SIGNATURE_HEADERS)
for region in BEDROCK_REGIONS
}
)
@functools.lru_cache(maxsize=8)

72
tests/e2e/provider_edge_bedrock.py Normal file
View file

@ -0,0 +1,72 @@
"""SigV4 re-signing for Bedrock traffic routed through the provider edge.
Bedrock is the one provider the edge could never mount. SigV4 signs the Host
header, so rewriting ``api_base`` to point at the edge invalidates the proxy's
signature and Bedrock rejects the call before it reaches a model. The edge
therefore has to drop the proxy's signature and mint its own over the upstream
URL it is actually about to call.
The identity it signs with is the run pod's own, from the EKS Pod Identity
association on ServiceAccount ``buildkite-e2e-run``. That role carries Bedrock
invoke and converse on an allowlist of the Anthropic models the suite registers
and nothing else, so a re-signed call can reach exactly the models the suite
already uses. The proxy's own Bedrock credentials are not involved in a routed
deployment, which is why ``aws_role_name`` deployments stay off the edge: their
whole point is to prove the product's assume-role chain.
Signature headers are excluded from the cache key by the caller, and they have
to be: ``x-amz-date`` is a timestamp, so keying on it would make every Bedrock
request a permanent miss.
"""
from __future__ import annotations
import functools
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from typing import Final
from botocore.auth import SigV4Auth
from botocore.awsrequest import AWSRequest
from botocore.credentials import Credentials
from botocore.session import Session
from provider_cache import SIGNATURE_HEADERS
BEDROCK_SERVICE: Final = "bedrock"
class MissingAwsCredentials(RuntimeError):
"""No AWS identity is resolvable, so the edge cannot sign for Bedrock."""
@dataclass(frozen=True, slots=True)
class BedrockSigner:
region: str
credentials: Callable[[], Credentials]
def __call__(self, method: str, url: str, headers: Mapping[str, str], body: bytes | None) -> dict[str, str]:
unsigned: Final = {
name: value for name, value in headers.items() if name.lower() not in SIGNATURE_HEADERS
}
request: Final = AWSRequest(method=method, url=url, headers=unsigned, data=body or b"")
SigV4Auth(self.credentials(), BEDROCK_SERVICE, self.region).add_auth(request)
return dict(request.headers)
@functools.lru_cache(maxsize=1)
def pod_credentials() -> Credentials:
"""The run pod's own identity, resolved once per process through botocore's
ordinary chain, which reaches Pod Identity at the ``container-role`` link."""
resolved: Final = Session().get_credentials()
if resolved is None: # pyright: ignore[reportUnnecessaryComparison] # stubs miss the empty-chain None
raise MissingAwsCredentials(
"the provider edge is mounted for Bedrock but no AWS credentials resolve; "
"the run pod gets them from the Pod Identity association on buildkite-e2e-run"
)
return resolved
def bedrock_signer(region: str, credentials: Callable[[], Credentials] = pod_credentials) -> BedrockSigner:
"""Credentials are resolved on the first signed request, not here, so a run
that mounts Bedrock but never calls it needs no AWS identity at all."""
return BedrockSigner(region, credentials)

1
tests/e2e/pytest.ini
View file

@ -10,4 +10,5 @@ markers =
weekly: real-provider anomaly load test that spends real money; deselected unless E2E_WEEKLY_ANOMALY is set
managed_files: needs a proxy running with require_managed_files enabled; deselected unless E2E_MANAGED_FILES_STACK is set
prompt_caching_stack: needs a proxy running with router_settings.optional_pre_call_checks including prompt_caching; deselected unless E2E_PROMPT_CACHING_STACK is set
cli_determinism: drives the real claude CLI for several seconds, which widens the window in which another test's in-flight upstream call is attributed to it; deselected unless E2E_CLI_DETERMINISM is set
redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set

2
tests/e2e/quota_management/budgets/BUDGET_TEST_COVERAGE_MATRIX.md
View file

@ -21,7 +21,7 @@ on the shared lifecycle (every entity it creates is deleted on teardown).
| Entity | Unit | Pre-existing live | This suite (live) | Status |
|--------|------|-------------------|-------------------|--------|
| API key | `test_budget_reservation.py`, `test_max_budget_limiter.py` | `otel_tests` | `test_budget_enforcement_e2e::test_key_budget_blocks` | **covered** |
| API key | `test_budget_reservation.py` | `otel_tests` | `test_budget_enforcement_e2e::test_key_budget_blocks` | **covered** |
| Team | `test_team_budget_limits.py` | `otel_tests` | (org test builds a team) | **covered** |
| Internal user | auth unit tests | - | `test_internal_user_budget_blocks` | **covered (new)** |
| Team member | `test_team_member_budget.py` | - | `test_team_member_budget_blocks` | **covered (new)** |

19
tests/e2e/test_provider_edge.py
View file

@ -1279,15 +1279,30 @@ class TestApiBaseSeam:
)
def test_unknown_mount_raises_naming_the_known_mounts(self, tmp_path: Path) -> None:
with pytest.raises(ValueError, match="unknown provider mount 'bedrock'"):
with pytest.raises(ValueError, match="unknown provider mount 'cohere'"):
provider_edge_api_base(
"bedrock",
"cohere",
mode_raw="record",
bundle_dir=tmp_path / "bundle",
bind_host="127.0.0.1",
advertise_host="127.0.0.1",
)
@pytest.mark.parametrize("mode_raw", ["record", "replay"])
def test_bedrock_never_wires_a_bundle_because_the_edge_cannot_sign_into_one(
self, tmp_path: Path, mode_raw: str,
) -> None:
"""Record and replay serve from a bundle without re-signing, so a Bedrock
deployment pointed at that edge would send the proxy's signature over a
rewritten Host. It keeps its direct route in both modes."""
assert provider_edge_api_base(
"bedrock/us-east-1",
mode_raw=mode_raw,
bundle_dir=tmp_path / "bundle",
bind_host="127.0.0.1",
advertise_host="127.0.0.1",
) is None
def test_record_mode_boots_one_shared_edge_and_prepares_the_bundle(self, tmp_path: Path) -> None:
root = tmp_path / "bundle"
first = provider_edge_api_base(

7
tests/llm_translation/test_together_ai.py
View file

@ -3,6 +3,7 @@ Test TogetherAI LLM
"""
from base_llm_unit_tests import BaseLLMChatTest
from tests._live_test_helpers import cheapest_together_chat_model
import json
import os
from datetime import datetime
@ -16,7 +17,11 @@ import pytest
class TestTogetherAI(BaseLLMChatTest):
def get_base_completion_call_args(self) -> dict:
litellm.set_verbose = True
return {"model": "together_ai/openai/gpt-oss-20b"}
return {
"model": cheapest_together_chat_model(
function_calling=True, response_schema=True
)
}
def test_tool_call_no_arguments(self, tool_call_no_arguments):
"""Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""

74
tests/local_testing/test_completion.py
View file

@ -57,23 +57,6 @@ def test_response_model_none():
assert isinstance(x, litellm.ModelResponse)
def test_completion_custom_provider_model_name():
try:
litellm.cache = None
response = completion(
model="together_ai/openai/gpt-oss-20b",
messages=messages,
logger_fn=logger_fn,
)
# Add assertions here to check the-response
print(response)
print(response["choices"][0]["finish_reason"])
except litellm.Timeout as e:
pass
except Exception as e:
pytest.fail(f"Error occurred: {e}")
def _openai_mock_response(*args, **kwargs) -> litellm.ModelResponse:
new_response = MagicMock()
new_response.headers = {"hello": "world"}
@ -2803,41 +2786,6 @@ def test_completion_together_ai_llama():
# test_completion_together_ai()
def test_customprompt_together_ai():
try:
litellm.set_verbose = False
litellm.num_retries = 0
print("in test_customprompt_together_ai")
print(litellm.success_callback)
print(litellm._async_success_callback)
response = completion(
model="together_ai/openai/gpt-oss-20b",
messages=messages,
roles={
"system": {
"pre_message": "<|im_start|>system\n",
"post_message": "<|im_end|>",
},
"assistant": {
"pre_message": "<|im_start|>assistant\n",
"post_message": "<|im_end|>",
},
"user": {
"pre_message": "<|im_start|>user\n",
"post_message": "<|im_end|>",
},
},
)
print(response)
except litellm.exceptions.Timeout as e:
print(f"Timeout Error")
pass
except Exception as e:
print(f"ERROR TYPE {type(e)}")
pytest.fail(f"Error occurred: {e}")
# test_customprompt_together_ai()
def response_format_tests(response: litellm.ModelResponse):
@ -3644,28 +3592,6 @@ async def test_acompletion_stream_watsonx():
# test_maritalk()
def test_completion_together_ai_stream():
litellm.set_verbose = True
user_message = "Write 1pg about YC & litellm"
messages = [{"content": user_message, "role": "user"}]
try:
response = completion(
model="together_ai/openai/gpt-oss-20b",
messages=messages,
stream=True,
max_tokens=5,
)
print(response)
for chunk in response:
print(chunk)
# print(string_response)
except Exception as e:
pytest.fail(f"Error occurred: {e}")
# test_completion_together_ai_stream()
def test_moderation():
response = litellm.moderation(input="i'm ishaan cto of litellm")
print(response)

3
tests/local_testing/test_text_completion.py
View file

@ -12,6 +12,7 @@ from unittest.mock import MagicMock, patch
import pytest
import litellm
from tests._live_test_helpers import cheapest_together_chat_model
from litellm import (
RateLimitError,
TextCompletionResponse,
@ -4030,7 +4031,7 @@ def test_async_text_completion_together_ai():
async def test_get_response():
try:
response = await litellm.atext_completion(
model="together_ai/openai/gpt-oss-20b",
model=cheapest_together_chat_model(),
prompt="good morning",
max_tokens=10,
)

26
tests/logging_callback_tests/test_generic_api_callback.py
View file

@ -10,6 +10,7 @@ import httpx
import json
import logging
import time
from typing import Final
from unittest.mock import AsyncMock, patch
import pytest
@ -98,8 +99,15 @@ async def test_generic_api_callback():
assert isinstance(actual_request, list), "Request body should be a list"
assert len(actual_request) > 0, "Request body list should not be empty"
# Validate the first payload item
payload_item: StandardLoggingPayload = StandardLoggingPayload(**actual_request[0])
this_test_messages: Final = [{"role": "user", "content": "Hello, world!"}]
mine: Final = [
item for item in actual_request if item.get("messages") == this_test_messages
]
assert (
len(mine) == 1
), f"Expected this test's single call in the batch, got {len(mine)} of {len(actual_request)}"
payload_item: StandardLoggingPayload = StandardLoggingPayload(**mine[0])
print("##########\n")
print(json.dumps(payload_item, indent=4))
print("##########\n")
@ -448,11 +456,17 @@ async def test_generic_api_callback_sumologic_uses_ndjson():
assert isinstance(ndjson_data, str), "Data should be a string for NDJSON"
lines = ndjson_data.strip().split("\n")
assert len(lines) == 2, f"Expected 2 lines of NDJSON, got {len(lines)}"
records: Final = [json.loads(line) for line in lines]
# Each line should be valid JSON
for line in lines:
json.loads(line) # Will raise if invalid JSON
this_test_messages: Final = [
[{"role": "user", "content": f"Test {i}"}] for i in range(2)
]
mine: Final = [
record for record in records if record.get("messages") in this_test_messages
]
assert (
len(mine) == 2
), f"Expected this test's 2 calls as NDJSON lines, got {len(mine)} of {len(records)}"
@pytest.mark.asyncio

12
tests/logging_callback_tests/test_unit_test_litellm_logging.py
View file

@ -8,8 +8,8 @@ from typing import Literal
import pytest
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy.hooks.max_budget_limiter import _PROXY_MaxBudgetLimiter
from litellm.proxy.hooks.cache_control_check import _PROXY_CacheControlCheck
from litellm.proxy.hooks.max_iterations_limiter import _PROXY_MaxIterationsHandler
from litellm._service_logger import ServiceLogging
import asyncio
@ -58,11 +58,11 @@ def test_is_internal_litellm_proxy_callback():
"""
Ensure we can determine if a callback is an internal litellm proxy callback
eg. `_PROXY_MaxBudgetLimiter`, `_PROXY_CacheControlCheck`
eg. `_PROXY_MaxIterationsHandler`, `_PROXY_CacheControlCheck`
"""
logging = setup_logging()
assert logging._is_internal_litellm_proxy_callback(_PROXY_MaxBudgetLimiter) == True
assert logging._is_internal_litellm_proxy_callback(_PROXY_MaxIterationsHandler) == True
# Test non-internal callbacks
def regular_callback():
@ -95,7 +95,7 @@ def test_should_run_sync_callbacks_for_async_calls():
assert logging._should_run_sync_callbacks_for_async_calls() == True
# Test with internal callback only
litellm.success_callback = [_PROXY_MaxBudgetLimiter]
litellm.success_callback = [_PROXY_MaxIterationsHandler]
assert logging._should_run_sync_callbacks_for_async_calls() == False
@ -107,7 +107,7 @@ def test_remove_internal_litellm_callbacks():
callbacks = [
regular_callback,
_PROXY_MaxBudgetLimiter,
_PROXY_MaxIterationsHandler,
_PROXY_CacheControlCheck,
"string_callback",
]
@ -116,5 +116,5 @@ def test_remove_internal_litellm_callbacks():
assert len(filtered) == 2 # Should only keep regular_callback and string_callback
assert regular_callback in filtered
assert "string_callback" in filtered
assert _PROXY_MaxBudgetLimiter not in filtered
assert _PROXY_MaxIterationsHandler not in filtered
assert _PROXY_CacheControlCheck not in filtered

1
tests/proxy_unit_tests/test_proxy_server.py
View file

@ -809,6 +809,7 @@ def test_img_gen(mock_aimage_generation, client_no_auth):
n=1,
size="1024x1024",
imageConfig={"aspectRatio": "9:16", "imageSize": "1K"},
litellm_call_id=mock.ANY,
metadata=mock.ANY,
proxy_server_request=mock.ANY,
secret_fields=mock.ANY,

Some files were not shown because too many files have changed in this diff Show more