Merge remote-tracking branch 'origin/main' into pr-44290-fixes

This commit is contained in:
Ritik Raj 2026-10-03 22:28:06 +00:00
commit 91d970a689
No known key found for this signature in database
119 changed files with 12084 additions and 751 deletions

View file

@ -4469,6 +4469,19 @@ dependencies = [
"time",
]
[[package]]
name = "litellm-traces-cache"
version = "0.1.0"
dependencies = [
"litellm-traces",
"moka",
"rstest",
"serde_json",
"sha2 0.10.9",
"thiserror 2.0.19",
"tokio",
]
[[package]]
name = "litellm-traces-clickhouse"
version = "0.1.0"
@ -4484,6 +4497,7 @@ dependencies = [
"litellm-migrate",
"litellm-storage-clickhouse",
"litellm-traces",
"litellm-traces-cache",
"macro_rules_attribute",
"moka",
"rstest",

View file

@ -13,6 +13,7 @@ litellm-config = { path = "crates/config" }
litellm-router = { path = "crates/router" }
litellm-tracing = { path = "crates/tracing" }
litellm-traces = { path = "crates/traces" }
litellm-traces-cache = { path = "crates/traces-cache" }
litellm-traces-clickhouse = { path = "crates/traces-clickhouse" }
litellm-storage-clickhouse = { path = "crates/storage-clickhouse" }
litellm-migrate = { path = "crates/migrate" }

View file

@ -0,0 +1,5 @@
Own resolved-trace snapshot storage, cache identity, weighting, and expiry
Depend on trace domain types, never storage, HTTP, or Python
Preserve the full source and authorization scope in every cache key
Keep snapshots immutable and expose borrowed data
Keep cursor formats and database reads in their existing owners

View file

@ -0,0 +1,17 @@
[package]
name = "litellm-traces-cache"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
litellm-traces.workspace = true
moka.workspace = true
serde_json.workspace = true
sha2.workspace = true
thiserror.workspace = true
[dev-dependencies]
rstest.workspace = true
tokio.workspace = true

View file

@ -0,0 +1,7 @@
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("trace snapshot serialization failed")]
Serialization(#[from] serde_json::Error),
#[error("trace snapshot exceeds the size limit")]
ReadTooLarge,
}

View file

@ -0,0 +1,166 @@
use std::{sync::Arc, time::Duration};
use litellm_traces::{Trace, query::named::ReadAccessParams};
use moka::future::Cache;
use sha2::{Digest, Sha256};
mod error;
pub use error::Error;
#[derive(Clone, Eq, Hash, PartialEq)]
pub struct SnapshotKey(String);
impl SnapshotKey {
pub fn new(
source: &str,
access: &ReadAccessParams,
trace_id: &str,
trace_ref: &str,
snapshot_ms: u64,
) -> Result<Self, Error> {
let encoded = serde_json::to_vec(&(source, access, trace_id, trace_ref, snapshot_ms))?;
Ok(Self(format!("{:x}", Sha256::digest(encoded))))
}
}
pub struct Snapshot {
trace: Trace,
version: String,
weight: u32,
}
impl Snapshot {
pub fn trace(&self) -> &Trace {
&self.trace
}
pub fn version(&self) -> &str {
&self.version
}
}
pub struct SnapshotCache {
entries: Cache<SnapshotKey, Arc<Snapshot>>,
max_graph_bytes: usize,
}
impl SnapshotCache {
pub fn new(max_graph_bytes: usize, ttl: Duration) -> Self {
Self {
entries: Cache::builder()
.max_capacity((max_graph_bytes as u64).saturating_mul(2))
.weigher(|_: &SnapshotKey, snapshot: &Arc<Snapshot>| snapshot.weight)
.time_to_live(ttl)
.build(),
max_graph_bytes,
}
}
pub async fn get(&self, key: &SnapshotKey) -> Option<Arc<Snapshot>> {
self.entries.get(key).await
}
pub async fn insert(&self, key: SnapshotKey, trace: Trace) -> Result<Arc<Snapshot>, Error> {
let encoded = serde_json::to_vec(&trace)?;
if encoded.len() > self.max_graph_bytes {
return Err(Error::ReadTooLarge);
}
let span_ids: Vec<&str> = trace
.spans
.iter()
.map(|span| span.span_id.as_str())
.collect();
let version = format!("{:x}", Sha256::digest(serde_json::to_vec(&span_ids)?));
let snapshot = Arc::new(Snapshot {
trace,
version,
weight: u32::try_from(encoded.len().saturating_mul(2)).unwrap_or(u32::MAX),
});
self.entries.insert(key, Arc::clone(&snapshot)).await;
Ok(snapshot)
}
}
#[cfg(test)]
mod tests {
use litellm_traces::{
SpanStatus,
query::named::{SpendByResponseIdsRow, TraceSpansRow},
resolve_trace,
};
use super::*;
fn trace(span_id: &str) -> Trace {
let rows = [TraceSpansRow {
trace_id: String::new(),
span_id: span_id.into(),
parent_span_id: String::new(),
name: "run".into(),
kind: litellm_traces::ObservationType::Agent,
wrapper_candidate: false,
agent: "agent".into(),
framework: String::new(),
status: SpanStatus::Ok,
status_message: String::new(),
error_truncated: false,
start_ns: 1_790_742_989_000_000_000,
duration_ns: 10_000_000,
service: "agent-demo".into(),
input_preview: format!("input of {span_id}"),
model: String::new(),
input_tokens: 0,
output_tokens: 0,
litellm_request_id: String::new(),
call_keys: Vec::new(),
call_evidence: None,
tool_call_id: String::new(),
team_id: String::new(),
api_key_hash: String::new(),
user_id: String::new(),
}];
resolve_trace("trace", "ref", &rows, &[] as &[SpendByResponseIdsRow])
.expect("fixture should resolve")
}
fn key(suffix: &str) -> SnapshotKey {
SnapshotKey::new(
"source",
&ReadAccessParams {
all_teams: false,
user_id: String::new(),
team_ids: vec!["team".into()],
},
suffix,
"ref",
100,
)
.unwrap()
}
#[tokio::test]
async fn weighted_capacity_bounds_retained_snapshots() {
let limit = ["first", "second", "third"]
.iter()
.map(|span_id| serde_json::to_vec(&trace(span_id)).unwrap().len())
.max()
.unwrap();
let cache = SnapshotCache::new(limit, Duration::from_secs(120));
for (key, span_id) in [
(key("a"), "first"),
(key("b"), "second"),
(key("c"), "third"),
] {
cache.insert(key, trace(span_id)).await.unwrap();
}
cache.entries.run_pending_tasks().await;
assert!(cache.entries.weighted_size() <= (limit as u64) * 2);
}
}

View file

@ -0,0 +1,191 @@
use std::time::Duration;
use litellm_traces::{
SpanStatus, Trace,
query::named::{ReadAccessParams, SpendByResponseIdsRow, TraceSpansRow},
resolve_trace,
};
use litellm_traces_cache::{Error, SnapshotCache, SnapshotKey};
use rstest::{fixture, rstest};
const T0: i64 = 1_790_742_989_000_000_000;
const MS: i64 = 1_000_000;
const TTL: Duration = Duration::from_secs(120);
fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> TraceSpansRow {
TraceSpansRow {
trace_id: String::new(),
span_id: span_id.into(),
parent_span_id: parent.into(),
name: name.into(),
kind: kind.parse().unwrap(),
wrapper_candidate: false,
agent: agent.into(),
framework: String::new(),
status: SpanStatus::Ok,
status_message: String::new(),
error_truncated: false,
start_ns: T0,
duration_ns: 10 * MS as u64,
service: "agent-demo".into(),
input_preview: format!("input of {name}"),
model: String::new(),
input_tokens: 0,
output_tokens: 0,
litellm_request_id: String::new(),
call_keys: Vec::new(),
call_evidence: None,
tool_call_id: String::new(),
team_id: String::new(),
api_key_hash: String::new(),
user_id: String::new(),
}
}
fn access() -> ReadAccessParams {
ReadAccessParams {
all_teams: false,
user_id: String::new(),
team_ids: vec!["team".into()],
}
}
fn key(
source: &str,
access: &ReadAccessParams,
trace_id: &str,
trace_ref: &str,
ms: u64,
) -> SnapshotKey {
SnapshotKey::new(source, access, trace_id, trace_ref, ms).unwrap()
}
#[fixture]
fn trace() -> Trace {
resolve_trace(
"trace",
"ref",
&[row("root", "", "run", "agent", "agent")],
&[] as &[SpendByResponseIdsRow],
)
.expect("fixture should resolve")
}
#[rstest]
#[case::different_team(false, "", "other-team")]
#[case::different_user(false, "other-user", "team")]
#[case::different_scope(true, "", "team")]
#[tokio::test]
async fn cached_trace_is_isolated_by_access_scope(
trace: Trace,
#[case] all_teams: bool,
#[case] user_id: &str,
#[case] team_id: &str,
) {
let cache = SnapshotCache::new(1024 * 1024, TTL);
let stored = key("source", &access(), "trace", "ref", 100);
cache.insert(stored.clone(), trace.clone()).await.unwrap();
let other_access = ReadAccessParams {
all_teams,
user_id: user_id.into(),
team_ids: vec![team_id.into()],
};
let other = key("source", &other_access, "trace", "ref", 100);
assert!(cache.get(&other).await.is_none());
let cached = cache.get(&stored).await.unwrap();
assert_eq!(cached.trace(), &trace);
}
#[rstest]
#[case::different_source("other-source", "trace", "ref", 100)]
#[case::different_trace_id("source", "other-trace", "ref", 100)]
#[case::different_trace_ref("source", "trace", "other-ref", 100)]
#[case::different_snapshot_ms("source", "trace", "ref", 200)]
#[tokio::test]
async fn cached_trace_is_isolated_by_key_fields(
trace: Trace,
#[case] source: &str,
#[case] trace_id: &str,
#[case] trace_ref: &str,
#[case] snapshot_ms: u64,
) {
let cache = SnapshotCache::new(1024 * 1024, TTL);
let stored = key("source", &access(), "trace", "ref", 100);
cache.insert(stored.clone(), trace.clone()).await.unwrap();
let other = key(source, &access(), trace_id, trace_ref, snapshot_ms);
assert!(cache.get(&other).await.is_none());
assert!(cache.get(&stored).await.is_some());
}
#[rstest]
#[tokio::test]
async fn snapshot_at_the_size_limit_is_accepted(trace: Trace) {
let size = serde_json::to_vec(&trace).unwrap().len();
let cache = SnapshotCache::new(size, TTL);
let stored = key("source", &access(), "trace", "ref", 100);
cache.insert(stored.clone(), trace).await.unwrap();
assert!(cache.get(&stored).await.is_some());
}
#[rstest]
#[tokio::test]
async fn snapshot_one_byte_over_the_size_limit_is_rejected(trace: Trace) {
let size = serde_json::to_vec(&trace).unwrap().len();
let cache = SnapshotCache::new(size - 1, TTL);
let stored = key("source", &access(), "trace", "ref", 100);
assert!(matches!(
cache.insert(stored.clone(), trace).await,
Err(Error::ReadTooLarge)
));
assert!(cache.get(&stored).await.is_none());
}
#[rstest]
#[case::same_ids(&["root", "child"], &["root", "child"], true)]
#[case::different_ids(&["root", "child"], &["root", "other"], false)]
#[tokio::test]
async fn snapshot_version_tracks_the_ordered_span_ids(
#[case] first_ids: &[&str],
#[case] second_ids: &[&str],
#[case] equal: bool,
) {
let build = |ids: &[&str]| -> Trace {
let rows: Vec<TraceSpansRow> = ids
.iter()
.map(|span_id| row(span_id, "", "run", "agent", "agent"))
.collect();
resolve_trace("trace", "ref", &rows, &[] as &[SpendByResponseIdsRow])
.expect("fixture should resolve")
};
let cache = SnapshotCache::new(1024 * 1024, TTL);
let first = cache
.insert(key("source", &access(), "a", "ref", 100), build(first_ids))
.await
.unwrap();
let second = cache
.insert(key("source", &access(), "b", "ref", 100), build(second_ids))
.await
.unwrap();
assert_eq!(first.version() == second.version(), equal);
}
#[rstest]
#[tokio::test]
async fn snapshots_expire_after_the_ttl(trace: Trace) {
let cache = SnapshotCache::new(1024 * 1024, Duration::from_millis(50));
let stored = key("source", &access(), "trace", "ref", 100);
cache.insert(stored.clone(), trace).await.unwrap();
tokio::time::sleep(Duration::from_millis(200)).await;
assert!(cache.get(&stored).await.is_none());
}

View file

@ -21,6 +21,7 @@ litellm-http.workspace = true
litellm-migrate.workspace = true
litellm-storage-clickhouse.workspace = true
litellm-traces.workspace = true
litellm-traces-cache.workspace = true
moka.workspace = true
serde.workspace = true
serde_json.workspace = true

View file

@ -0,0 +1,14 @@
SELECT t.TraceId, t.SpanId, s.request_id, s.spend, s.metadata
FROM otel_traces AS t
INNER JOIN (
SELECT *
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
) AS s
ON t.LiteLLMRequestId = s.response_id
AND t.TeamId = s.team_id
AND ((t.UserId != '' AND t.UserId = s.user)
OR (t.ApiKeyHash != '' AND t.ApiKeyHash = s.api_key))
WHERE t.Timestamp >= now() - INTERVAL 1 DAY
AND t.LiteLLMRequestId != ''
LIMIT 100

View file

@ -0,0 +1,8 @@
SELECT
request_id, response_id, model, spend, JSONExtractString(metadata, 'project') AS project
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
AND JSONHas(metadata, 'project')
AND JSONExtractString(metadata, 'project') = 'example'
ORDER BY start_time DESC
LIMIT 100

View file

@ -0,0 +1,6 @@
SELECT
DISTINCT arrayJoin(JSONExtractKeys(metadata)) AS key
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 30 DAY
ORDER BY key
LIMIT 200

View file

@ -0,0 +1,7 @@
SELECT TeamId AS team, ApiKeyHash AS api_key, TraceId AS trace_id,
SpanId AS span_id, StatusMessage AS message
FROM otel_traces
WHERE Timestamp >= now() - INTERVAL 1 DAY
AND StatusCode = 'STATUS_CODE_ERROR'
ORDER BY Timestamp DESC, team, api_key, trace_id, span_id
LIMIT 100

View file

@ -1,5 +1,8 @@
SELECT team_id AS team, api_key, request_id, spend,
JSONExtractString(metadata, 'labels', 'priority') AS priority
FROM spend_logs FINAL
WHERE JSONExtractString(metadata, 'labels', 'priority') = 'high'
WHERE start_time >= now() - INTERVAL 1 DAY
AND JSONHas(metadata, 'labels', 'priority')
AND JSONExtractString(metadata, 'labels', 'priority') = 'high'
ORDER BY team, api_key, request_id
LIMIT 100

View file

@ -0,0 +1,17 @@
SELECT
team_id, model, requests, unknown_cost_requests,
if(unknown_cost_requests = 0, recorded_spend, NULL) AS spend,
input_tokens, output_tokens
FROM (
SELECT
team_id, model, count() AS requests,
countIf(isNull(spend) OR NOT isFinite(spend)) AS unknown_cost_requests,
sum(spend) AS recorded_spend,
sum(prompt_tokens) AS input_tokens,
sum(completion_tokens) AS output_tokens
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
GROUP BY team_id, model
)
ORDER BY team_id, model
LIMIT 100

View file

@ -0,0 +1,8 @@
SELECT
request_id,
JSONType(metadata, 'labels', 'priority') AS type,
JSONExtractRaw(metadata, 'labels', 'priority') AS value
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
AND JSONHas(metadata, 'labels', 'priority')
LIMIT 100

View file

@ -0,0 +1,8 @@
SELECT
TraceId, SpanId, Model, InputTokens, OutputTokens,
Duration / 1000000 AS duration_ms
FROM otel_traces
WHERE Timestamp >= now() - INTERVAL 1 DAY
AND ObservationType = 'llm'
ORDER BY Timestamp DESC
LIMIT 100

View file

@ -0,0 +1,8 @@
SELECT
request_id, response_id, trace_id, span_id, model, spend,
prompt_tokens, completion_tokens, status,
JSONExtractBool(metadata, 'synthetic_spend') AS synthetic_spend
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
ORDER BY start_time DESC, request_id
LIMIT 100

View file

@ -0,0 +1,10 @@
SELECT
team_id, api_key, trace_id, count() AS requests,
countIf(isNull(spend) OR NOT isFinite(spend)) AS unknown_cost_requests,
if(unknown_cost_requests = 0, sum(spend), NULL) AS recorded_spend
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
AND trace_id != ''
GROUP BY team_id, api_key, trace_id
ORDER BY team_id, api_key, trace_id
LIMIT 100

View file

@ -7,4 +7,6 @@ SELECT TeamId AS team, ApiKeyHash AS api_key, TraceId AS trace_id,
toUInt32(sum(OutputTokens)) AS output_tokens
FROM agent_traces_by_key
GROUP BY TeamId, ApiKeyHash, TraceId
HAVING min(StartTs) >= now() - INTERVAL 1 DAY
ORDER BY team, api_key, trace_id
LIMIT 100

View file

@ -0,0 +1,18 @@
SELECT
t.TraceId, t.SpanId, t.Model, t.LiteLLMRequestId,
t.InputTokens, t.OutputTokens
FROM otel_traces AS t
LEFT ANTI JOIN (
SELECT *
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
) AS s
ON t.TeamId = s.team_id
AND ((t.UserId != '' AND t.UserId = s.user)
OR (t.ApiKeyHash != '' AND t.ApiKeyHash = s.api_key))
AND t.LiteLLMRequestId != ''
AND (t.LiteLLMRequestId = s.response_id OR t.LiteLLMRequestId = s.request_id)
WHERE t.Timestamp >= now() - INTERVAL 1 DAY
AND t.ObservationType = 'llm'
ORDER BY t.Timestamp DESC, t.SpanId
LIMIT 100

View file

@ -47,3 +47,12 @@ pub enum Error {
#[error(transparent)]
Cached(#[from] std::sync::Arc<Error>),
}
impl From<litellm_traces_cache::Error> for Error {
fn from(error: litellm_traces_cache::Error) -> Self {
match error {
litellm_traces_cache::Error::Serialization(_) => Self::InvalidResponse,
litellm_traces_cache::Error::ReadTooLarge => Self::ReadTooLarge,
}
}
}

View file

@ -297,7 +297,7 @@ pub struct QueryHelp {
attributes: Vec<AttributeCatalog>,
relationships: &'static [Relationship],
#[cfg_attr(feature = "schema", schemars(with = "Vec<Example>"))]
examples: [Example; 9],
examples: [Example; 12],
#[cfg_attr(feature = "schema", schemars(with = "Vec<String>"))]
gotchas: [String; 13],
guide: String,

View file

@ -28,6 +28,12 @@ use crate::{Error, NormalizedFieldDefinition, query_access::ReaderLimits};
"trace_spend_sql",
"unmatched_spans_name",
"unmatched_spans_sql",
"trace_summary_name",
"trace_summary_sql",
"failed_spans_name",
"failed_spans_sql",
"metadata_filter_name",
"metadata_filter_sql",
"missing_spend",
"partial_spend",
"time_window",
@ -60,7 +66,7 @@ impl QueryGuide<'_> {
])
}
pub fn examples(&self) -> Result<[Example; 9], Error> {
pub fn examples(&self) -> Result<[Example; 12], Error> {
Ok([
Example {
name: render(&self.as_recent_spans_name())?,
@ -98,6 +104,18 @@ impl QueryGuide<'_> {
name: render(&self.as_unmatched_spans_name())?,
sql: render(&self.as_unmatched_spans_sql())?,
},
Example {
name: render(&self.as_trace_summary_name())?,
sql: render(&self.as_trace_summary_sql())?,
},
Example {
name: render(&self.as_failed_spans_name())?,
sql: render(&self.as_failed_spans_sql())?,
},
Example {
name: render(&self.as_metadata_filter_name())?,
sql: render(&self.as_metadata_filter_sql())?,
},
])
}

View file

@ -1,6 +1,6 @@
//! Scoped trace reads: the trace list, one trace resolved with its spend, and span payloads.
use std::sync::{Arc, LazyLock};
use std::sync::LazyLock;
use std::time::Duration;
use base64::{Engine, engine::general_purpose::URL_SAFE};
@ -12,9 +12,8 @@ use litellm_traces::{
SpanDetail, SpanErrorPage, SpendLookup, Trace, TracePage, listed_summary,
query::named as contracts, resolve_trace, to_ui_content,
};
use moka::future::Cache;
use litellm_traces_cache::{SnapshotCache, SnapshotKey};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use crate::{
Connection, Error,
@ -38,17 +37,11 @@ impl Query for RunCandidates {
}
// Cursor pages share a bounded snapshot so advancing does not resolve the whole graph again.
static TRACE_SNAPSHOTS: LazyLock<Cache<String, Arc<Trace>>> = LazyLock::new(|| {
Cache::builder()
.max_capacity((2 * crate::span_batches::MAX_GRAPH_BYTES) as u64)
.weigher(|_: &String, trace: &Arc<Trace>| {
serde_json::to_vec(trace.as_ref())
.ok()
.and_then(|bytes| u32::try_from(bytes.len().saturating_mul(2)).ok())
.unwrap_or(u32::MAX)
})
.time_to_live(Duration::from_secs(120))
.build()
static TRACE_SNAPSHOTS: LazyLock<SnapshotCache> = LazyLock::new(|| {
SnapshotCache::new(
crate::span_batches::MAX_GRAPH_BYTES,
Duration::from_secs(120),
)
});
const NANOS_PER_MS: i64 = 1_000_000;
@ -339,74 +332,57 @@ pub async fn get_trace_page(
version: String::new(),
},
};
let key_bytes = serde_json::to_vec(&(
let key = SnapshotKey::new(
connection.url().as_str(),
access,
trace_id,
&trace_ref,
position.snapshot_ms,
))
)
.map_err(|_| Error::InvalidParameters)?;
let key = format!("{:x}", Sha256::digest(key_bytes));
let snapshot = if let Some(trace) = TRACE_SNAPSHOTS.get(&key).await {
trace
} else {
let params = TraceSpansParams {
access: access.clone(),
trace_id: trace_id.to_owned(),
trace_ref: trace_ref.clone(),
};
let rows =
crate::span_batches::read_spans(client, connection, params, position.snapshot_ms)
.await?;
let spend_rows = spend(client, connection, access, &rows).await;
let Some(trace) = resolve_trace(trace_id, &trace_ref, &rows, &spend_rows) else {
return Ok(None);
};
if serde_json::to_vec(&trace)
.map_err(|_| Error::InvalidResponse)?
.len()
> crate::span_batches::MAX_GRAPH_BYTES
{
return Err(Error::ReadTooLarge);
let snapshot = match TRACE_SNAPSHOTS.get(&key).await {
Some(snapshot) => snapshot,
None => {
let params = TraceSpansParams {
access: access.clone(),
trace_id: trace_id.to_owned(),
trace_ref: trace_ref.clone(),
};
let rows =
crate::span_batches::read_spans(client, connection, params, position.snapshot_ms)
.await?;
let spend_rows = spend(client, connection, access, &rows).await;
let Some(trace) = resolve_trace(trace_id, &trace_ref, &rows, &spend_rows) else {
return Ok(None);
};
TRACE_SNAPSHOTS.insert(key, trace).await?
}
let trace = Arc::new(trace);
TRACE_SNAPSHOTS.insert(key, Arc::clone(&trace)).await;
trace
};
let span_ids: Vec<&str> = snapshot
.spans
.iter()
.map(|span| span.span_id.as_str())
.collect();
let version = format!(
"{:x}",
Sha256::digest(serde_json::to_vec(&span_ids).map_err(|_| Error::InvalidResponse)?)
);
if cursor.is_some() && position.version != version {
let spans = &snapshot.trace().spans;
if cursor.is_some() && position.version != snapshot.version() {
return Err(Error::TraceChanged);
}
let mut trace = Trace {
summary: snapshot.summary.clone(),
agents: snapshot.agents.clone(),
summary: snapshot.trace().summary.clone(),
agents: snapshot.trace().agents.clone(),
spans: Vec::new(),
next_cursor: None,
};
if position.offset > snapshot.spans.len() {
if position.offset > spans.len() {
return Err(Error::InvalidCursor("span"));
}
let end = position
.offset
.saturating_add(page_size as usize)
.min(snapshot.spans.len());
trace.next_cursor = (end < snapshot.spans.len()).then(|| {
.min(spans.len());
trace.next_cursor = (end < spans.len()).then(|| {
encode_cursor(&SpanPosition {
offset: end,
version: version.clone(),
version: snapshot.version().to_owned(),
..position
})
});
trace.spans = snapshot.spans[position.offset..end].to_vec();
trace.spans = spans[position.offset..end].to_vec();
while serde_json::to_vec(&trace)
.map_err(|_| Error::InvalidResponse)?
.len()
@ -420,7 +396,7 @@ pub async fn get_trace_page(
trace_ref: trace_ref.clone(),
snapshot_ms: position.snapshot_ms,
offset: position.offset + trace.spans.len(),
version: version.clone(),
version: snapshot.version().to_owned(),
}));
}
Ok(Some(trace))

View file

@ -60,14 +60,7 @@ Recent normalized LLM spans
{%- endblock %}
{% block recent_spans_sql -%}
SELECT
TraceId, SpanId, Model, InputTokens, OutputTokens,
Duration / 1000000 AS duration_ms
FROM otel_traces
WHERE Timestamp >= now() - INTERVAL 1 DAY
AND ObservationType = 'llm'
ORDER BY Timestamp DESC
LIMIT 100
{% include "../query/help/recent_spans.sql" %}
{%- endblock %}
{% block custom_metadata_name -%}
@ -75,14 +68,7 @@ Find calls by custom metadata
{%- endblock %}
{% block custom_metadata_sql -%}
SELECT
request_id, response_id, model, spend, JSONExtractString(metadata, 'project') AS project
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
AND JSONHas(metadata, 'project')
AND JSONExtractString(metadata, 'project') = 'example'
ORDER BY start_time DESC
LIMIT 100
{% include "../query/help/custom_metadata.sql" %}
{%- endblock %}
{% block nested_metadata_name -%}
@ -90,14 +76,7 @@ Nested metadata with unknown types
{%- endblock %}
{% block nested_metadata_sql -%}
SELECT
request_id,
JSONType(metadata, 'labels', 'priority') AS type,
JSONExtractRaw(metadata, 'labels', 'priority') AS value
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
AND JSONHas(metadata, 'labels', 'priority')
LIMIT 100
{% include "../query/help/nested_metadata.sql" %}
{%- endblock %}
{% block correlated_calls_name -%}
@ -105,20 +84,7 @@ Traces correlated with LLM call metadata
{%- endblock %}
{% block correlated_calls_sql -%}
SELECT t.TraceId, t.SpanId, s.request_id, s.spend, s.metadata
FROM otel_traces AS t
INNER JOIN (
SELECT *
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
) AS s
ON t.LiteLLMRequestId = s.response_id
AND t.TeamId = s.team_id
AND ((t.UserId != '' AND t.UserId = s.user)
OR (t.ApiKeyHash != '' AND t.ApiKeyHash = s.api_key))
WHERE t.Timestamp >= now() - INTERVAL 1 DAY
AND t.LiteLLMRequestId != ''
LIMIT 100
{% include "../query/help/correlated_calls.sql" %}
{%- endblock %}
{% block discover_keys_name -%}
@ -126,12 +92,7 @@ Discover metadata keys over a different window
{%- endblock %}
{% block discover_keys_sql -%}
SELECT
DISTINCT arrayJoin(JSONExtractKeys(metadata)) AS key
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 30 DAY
ORDER BY key
LIMIT 200
{% include "../query/help/discover_keys.sql" %}
{%- endblock %}
{% block recent_spend_name -%}
@ -139,14 +100,7 @@ Recent spend records
{%- endblock %}
{% block recent_spend_sql -%}
SELECT
request_id, response_id, trace_id, span_id, model, spend,
prompt_tokens, completion_tokens, status,
JSONExtractBool(metadata, 'synthetic_spend') AS synthetic_spend
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
ORDER BY start_time DESC, request_id
LIMIT 100
{% include "../query/help/recent_spend.sql" %}
{%- endblock %}
{% block model_spend_name -%}
@ -154,23 +108,7 @@ Spend and tokens by model
{%- endblock %}
{% block model_spend_sql -%}
SELECT
team_id, model, requests, unknown_cost_requests,
if(unknown_cost_requests = 0, recorded_spend, NULL) AS spend,
input_tokens, output_tokens
FROM (
SELECT
team_id, model, count() AS requests,
countIf(isNull(spend) OR NOT isFinite(spend)) AS unknown_cost_requests,
sum(spend) AS recorded_spend,
sum(prompt_tokens) AS input_tokens,
sum(completion_tokens) AS output_tokens
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
GROUP BY team_id, model
)
ORDER BY team_id, model
LIMIT 100
{% include "../query/help/model_spend.sql" %}
{%- endblock %}
{% block trace_spend_name -%}
@ -178,16 +116,7 @@ Recorded spend by trace
{%- endblock %}
{% block trace_spend_sql -%}
SELECT
team_id, api_key, trace_id, count() AS requests,
countIf(isNull(spend) OR NOT isFinite(spend)) AS unknown_cost_requests,
if(unknown_cost_requests = 0, sum(spend), NULL) AS recorded_spend
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
AND trace_id != ''
GROUP BY team_id, api_key, trace_id
ORDER BY team_id, api_key, trace_id
LIMIT 100
{% include "../query/help/trace_spend.sql" %}
{%- endblock %}
{% block unmatched_spans_name -%}
@ -195,24 +124,31 @@ LLM spans without a direct spend match
{%- endblock %}
{% block unmatched_spans_sql -%}
SELECT
t.TraceId, t.SpanId, t.Model, t.LiteLLMRequestId,
t.InputTokens, t.OutputTokens
FROM otel_traces AS t
LEFT ANTI JOIN (
SELECT *
FROM spend_logs FINAL
WHERE start_time >= now() - INTERVAL 1 DAY
) AS s
ON t.TeamId = s.team_id
AND ((t.UserId != '' AND t.UserId = s.user)
OR (t.ApiKeyHash != '' AND t.ApiKeyHash = s.api_key))
AND t.LiteLLMRequestId != ''
AND (t.LiteLLMRequestId = s.response_id OR t.LiteLLMRequestId = s.request_id)
WHERE t.Timestamp >= now() - INTERVAL 1 DAY
AND t.ObservationType = 'llm'
ORDER BY t.Timestamp DESC, t.SpanId
LIMIT 100
{% include "../query/help/unmatched_spans.sql" %}
{%- endblock %}
{% block trace_summary_name -%}
Trace summaries with tokens and errors
{%- endblock %}
{% block trace_summary_sql -%}
{% include "../query/help/trace_summary.sql" %}
{%- endblock %}
{% block failed_spans_name -%}
Recent failed spans
{%- endblock %}
{% block failed_spans_sql -%}
{% include "../query/help/failed_spans.sql" %}
{%- endblock %}
{% block metadata_filter_name -%}
Filter calls by nested metadata
{%- endblock %}
{% block metadata_filter_sql -%}
{% include "../query/help/metadata_filter.sql" %}
{%- endblock %}
{% block time_window -%}

View file

@ -1683,7 +1683,15 @@ async fn query_help_discovers_live_schema_and_runs_its_examples(
let values: serde_json::Value = serde_json::from_str(&body)?;
assert_eq!(
values["data"].as_array().ok_or("missing data")?.is_empty(),
!populated || example["name"] == "LLM spans without a direct spend match",
!populated
|| matches!(
example["name"].as_str(),
Some(
"LLM spans without a direct spend match"
| "Recent failed spans"
| "Filter calls by nested metadata"
)
),
"{sql}"
);
if populated && example["name"] == "Traces correlated with LLM call metadata" {

View file

@ -5,7 +5,7 @@ use litellm_traces::query::named as contracts;
use litellm_traces_clickhouse::{
QueryScope,
query::named::{ListTraces, ListTracesParams, TraceSpans, TraceSpansParams},
query_sql,
query_help, query_sql,
};
use rstest::{fixture, rstest};
use serde::Deserialize;
@ -48,12 +48,9 @@ struct QueryResult {
}
#[rstest]
#[case::rollups(include_str!("queries/rollups.sql"), include_str!("queries/rollups.expected.json"))]
#[case::costs(include_str!("queries/trace_costs.sql"), include_str!("queries/trace_costs.expected.json"))]
#[case::errors(include_str!("queries/failed_spans.sql"), include_str!("queries/failed_spans.expected.json"))]
#[case::metadata(include_str!("queries/metadata_filters.sql"), include_str!("queries/metadata_filters.expected.json"))]
#[tokio::test]
async fn curated_queries_return_expected_rows(
async fn storage_queries_return_expected_rows(
#[future(awt)] seeded_database: TestResult<SeededDatabase>,
#[case] sql: &str,
#[case] expected_json: &str,
@ -78,6 +75,64 @@ async fn curated_queries_return_expected_rows(
Ok(())
}
#[rstest]
#[case::trace_summary("Trace summaries with tokens and errors", include_str!("../query/help/trace_summary.sql"), include_str!("queries/rollups.expected.json"))]
#[case::failed_spans("Recent failed spans", include_str!("../query/help/failed_spans.sql"), include_str!("queries/failed_spans.expected.json"))]
#[case::metadata_filter("Filter calls by nested metadata", include_str!("../query/help/metadata_filter.sql"), include_str!("queries/metadata_filters.expected.json"))]
#[tokio::test]
async fn documented_queries_render_and_return_expected_rows(
#[future(awt)] seeded_database: TestResult<SeededDatabase>,
fixture_clock: TestResult<u64>,
#[case] name: &str,
#[case] expected_sql: &str,
#[case] expected_json: &str,
#[values(ScopeCase::Admin, ScopeCase::Team, ScopeCase::OtherTeam)] scope: ScopeCase,
) -> TestResult {
let fixture = seeded_database?;
let reader = fixture
.readers
.connection(&fixture.database.client, &scope.scope(), "fixture-secret")
.await?;
let help = serde_json::to_value(query_help(&fixture.database.client, &reader).await?)?;
let example = help["examples"]
.as_array()
.ok_or("missing examples")?
.iter()
.find(|example| example["name"] == name)
.ok_or("missing documented query")?;
let sql = example["sql"].as_str().ok_or("missing example SQL")?;
assert_eq!(sql.trim(), expected_sql.trim());
assert!(
help["guide"]
.as_str()
.ok_or("missing guide")?
.contains(&format!("{name}\n{sql}"))
);
let sql_at_fixture_time = sql.replace("now()", &format!("toDateTime({})", fixture_clock?));
let result: QueryResult = serde_json::from_str(
&query_sql(&fixture.database.client, &reader, &sql_at_fixture_time).await?,
)?;
let expected: BTreeMap<String, Vec<Value>> = serde_json::from_str(expected_json)?;
assert_eq!(
&result.data,
expected
.get(scope.as_ref())
.ok_or("missing expected scope")?,
"{}: {sql}",
scope.as_ref()
);
Ok(())
}
#[fixture]
fn fixture_clock() -> TestResult<u64> {
let spans = litellm_traces::decode_otlp(
include_bytes!("../../traces/tests/fixtures/query_root.json"),
Some("application/json"),
)?;
Ok(spans.first().ok_or("missing fixture root")?.start_ns / 1_000_000_000)
}
#[fixture]
fn admin_access() -> TestResult<contracts::ReadAccessParams> {
Ok(serde_json::from_str(include_str!(

View file

@ -1,5 +0,0 @@
SELECT TeamId AS team, ApiKeyHash AS api_key, TraceId AS trace_id,
SpanId AS span_id, StatusMessage AS message
FROM otel_traces
WHERE StatusCode = 'STATUS_CODE_ERROR'
ORDER BY team, api_key, trace_id, span_id

View file

@ -927,6 +927,12 @@ class MCPClient:
async def _call_tool_operation(session: ClientSession):
verbose_logger.debug("MCP client sending tool call to session")
if self.protocol_version == "2026-07-28":
tools: Final = await list_tools_with_pagination(
session, listing_deadline=max(self.timeout, MCP_TOOL_LISTING_TIMEOUT)
)
if not any(tool.name == call_tool_request_params.name for tool in tools):
raise MCPError(code=-32603, message="Tool schema is unavailable from the bounded upstream catalog")
return await session.call_tool(
name=call_tool_request_params.name,
arguments=call_tool_request_params.arguments,

View file

@ -22,6 +22,7 @@ from typing import TYPE_CHECKING, Any, Final, Protocol
import httpx
import openai
from pydantic import ConfigDict, TypeAdapter
import litellm
from litellm.constants import (
@ -46,6 +47,9 @@ if TYPE_CHECKING:
from uvicorn import Server
verbose_logger: Final = logging.getLogger("LiteLLM")
_JSON_VALUE_ADAPTER: Final = TypeAdapter(object, config=ConfigDict(strict=True))
_JSON_OBJECT_ADAPTER: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
_PORT_ADAPTER: Final = TypeAdapter(int, config=ConfigDict(strict=True))
MISSING_DEPS_MESSAGE = "litellm.harness needs starlette and uvicorn: pip install starlette uvicorn"
@ -207,11 +211,11 @@ class SSEUsageParser:
if not payload or payload == b"[DONE]":
return
try:
event = json.loads(payload)
event: Final[object] = _JSON_VALUE_ADAPTER.validate_python(json.loads(payload))
except ValueError:
return
if isinstance(event, Mapping):
self.absorb(event)
self.absorb(_JSON_OBJECT_ADAPTER.validate_python(event))
def absorb(self, event: Mapping[str, object]) -> None:
event_type = event.get("type")
@ -434,7 +438,7 @@ class ModelEndpoint:
except BaseException:
await self.stop()
raise
self.port = self._server.servers[0].sockets[0].getsockname()[1]
self.port = _PORT_ADAPTER.validate_python(self._server.servers[0].sockets[0].getsockname()[1])
async def stop(self) -> None:
if self._server is not None:
@ -540,11 +544,12 @@ class ModelEndpoint:
if not self._authorized(request):
return self._unauthorized()
try:
body = json.loads(await request.body())
parsed_body: Final[object] = _JSON_VALUE_ADAPTER.validate_python(json.loads(await request.body()))
except ValueError as e:
return self._error(e, 400)
if not isinstance(body, dict):
if not isinstance(parsed_body, dict):
return self._error(ValueError("request body must be a JSON object"), 400)
body: Final = _JSON_OBJECT_ADAPTER.validate_python(parsed_body)
route = route_of(request.url.path)
if self.gateway is not None:
return await self._forward(request, route, body)
@ -615,7 +620,8 @@ class ModelEndpoint:
tokens = (parser.input_tokens, parser.output_tokens)
else:
try:
tokens = usage_from_body(json.loads(collected))
body: Final[object] = _JSON_VALUE_ADAPTER.validate_python(json.loads(collected))
tokens = usage_from_body(body)
except ValueError:
tokens = (0, 0)
self._record(model, tokens[0], tokens[1], header_cost(upstream.headers))

View file

@ -186,7 +186,7 @@ class Session:
def history(
self,
) -> list[dict[str, Any]]: # mutable-ok: public API returns OpenAI-format message dicts from the handler
) -> list[dict[str, object]]: # mutable-ok: public API returns OpenAI-format message dicts from the handler
return run_sync(self._inner.history(), "history")
@property

View file

@ -492,7 +492,7 @@ class GalileoObserve(CustomLogger):
@staticmethod
def _get_chat_content_for_galileo(response_obj: litellm.ModelResponse) -> object:
if response_obj.choices and len(response_obj.choices) > 0:
message: Final = response_obj["choices"][0]["message"]
message: Final = response_obj.choices[0].message
if hasattr(message, "json"):
message_json: Final[object] = message.json()
if isinstance(message_json, str):

View file

@ -27,7 +27,7 @@ from litellm.types.llms.openai import (
ResponsesAPIResponse,
)
from litellm.types.prompts.init_prompts import PromptSpec
from litellm.types.utils import CallTypes, StandardCallbackDynamicParams
from litellm.types.utils import CallTypes, LLMResponseTypes, ModelResponse, StandardCallbackDynamicParams
from litellm.types.vector_stores import (
LiteLLM_ManagedVectorStore,
VectorStoreSearchFailure,
@ -46,6 +46,7 @@ else:
SEARCH_FAILURES_FIELD: Final = "vector_store_search_failures"
_DEFAULT_FAILURE_MODE: Final[VectorStoreSearchFailureMode] = "annotate"
_FAILURE_MODE_ADAPTER: Final = TypeAdapter(VectorStoreSearchFailureMode)
_OBJECT_ADAPTER: Final = TypeAdapter(object)
_STR_KEYED_ADAPTER: Final = TypeAdapter(dict[str, object])
_GUARDRAIL_KEYS_THE_PROXY_MERGES_INTO_METADATA: Final = frozenset(
{"guardrails", "guardrail_config", "policies", "include_guardrail_response"}
@ -269,7 +270,9 @@ class VectorStorePreCallHook(CustomLogger):
verbose_logger.debug("No query found in messages for vector store search")
return None
request_litellm_params: Final = litellm_logging_obj.model_call_details.get("litellm_params", {})
request_litellm_params: Final = _OBJECT_ADAPTER.validate_python(
litellm_logging_obj.model_call_details.get("litellm_params", {})
)
request_metadata: Final = (
request_litellm_params.get("metadata", {}) if isinstance(request_litellm_params, dict) else {}
)
@ -415,9 +418,9 @@ class VectorStorePreCallHook(CustomLogger):
async def async_post_call_success_deployment_hook(
self,
request_data: dict,
response: Any,
response: LLMResponseTypes,
call_type: CallTypes | None,
) -> Any | None:
) -> LLMResponseTypes | None:
"""
Add search results to the response after successful LLM call.
@ -451,7 +454,7 @@ class VectorStorePreCallHook(CustomLogger):
return response
# Add search results to response object
if hasattr(response, "choices") and response.choices:
if isinstance(response, ModelResponse) and response.choices:
for choice in response.choices:
if hasattr(choice, "message") and choice.message:
provider_fields = getattr(choice.message, "provider_specific_fields", None) or {}

View file

@ -3,6 +3,7 @@ from typing import Final, Literal
import litellm
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantle_chat_config
from litellm.types.utils import LlmProviders, LlmProvidersSet
@ -104,7 +105,7 @@ def get_supported_openai_params(
elif custom_llm_provider == "groq":
return litellm.GroqChatConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "bedrock_mantle":
return litellm.BedrockMantleChatConfig().get_supported_openai_params(model=model)
return bedrock_mantle_chat_config(model).get_supported_openai_params(model=model)
elif custom_llm_provider == "hosted_vllm":
return litellm.HostedVLLMChatConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "vllm":

View file

@ -1741,11 +1741,14 @@ def _find_server_tool_result(
)
_AnthropicToolInvokeItem: TypeAlias = AnthropicMessagesToolUseParam | dict[str, Any]
def convert_to_anthropic_tool_invoke(
tool_calls: list[ChatCompletionAssistantToolCall],
web_search_results: Sequence[object] | None = None,
tool_results: Sequence[object] | None = None,
) -> list[AnthropicMessagesToolUseParam | dict[str, Any]]:
) -> list[_AnthropicToolInvokeItem]:
"""
OpenAI tool invokes:
{
@ -2565,9 +2568,9 @@ def anthropic_messages_pt(
# Group tool invoke results into (server_tool_use, result) pairs
# and separate regular tool_use blocks
server_tool_groups: list[list[Any]] = []
regular_tool_uses: list[Any] = []
_current_group: list[Any] = []
server_tool_groups: list[list[_AnthropicToolInvokeItem]] = []
regular_tool_uses: list[_AnthropicToolInvokeItem] = []
_current_group: list[_AnthropicToolInvokeItem] = []
for item in tool_invoke_results:
item_type = item.get("type", "") if isinstance(item, dict) else getattr(item, "type", "")
if item_type == "server_tool_use":

View file

@ -0,0 +1,33 @@
from litellm.llms.base_llm.chat.transformation import BaseConfig
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.chat.mantle.transformation import AmazonMantleConfig
from litellm.llms.bedrock_mantle.chat.transformation import BedrockMantleChatConfig
from litellm.llms.bedrock_mantle.common_utils import BedrockMantleAuthMixin, is_mantle_claude_model
from litellm.llms.bedrock_mantle.messages.transformation import build_mantle_native_messages_url
class BedrockMantleClaudeChatConfig(BedrockMantleAuthMixin, AmazonMantleConfig):
def __init__(self, aws_signer: BaseAWSLLM | None = None) -> None:
AmazonMantleConfig.__init__(self)
self._aws_signer = aws_signer or self
@property
def custom_llm_provider(self) -> str | None:
return "bedrock_mantle"
def get_complete_url(
self,
api_base: str | None,
api_key: str | None,
model: str,
optional_params: dict[str, object],
litellm_params: dict[str, object],
stream: bool | None = None,
) -> str:
return build_mantle_native_messages_url(api_base=api_base, litellm_params=litellm_params)
def bedrock_mantle_chat_config(model: str) -> BaseConfig:
if is_mantle_claude_model(model):
return BedrockMantleClaudeChatConfig()
return BedrockMantleChatConfig()

View file

@ -127,6 +127,10 @@ class BedrockMantleAuthMixin(SignsRequestsWithAWS):
) from e
def is_mantle_claude_model(model: str) -> bool:
return "claude" in model.lower()
def mantle_supports_responses(model: str | None, model_cost: dict) -> bool:
"""Whether a Bedrock Mantle model can serve the native Responses API.

View file

@ -129,7 +129,7 @@ class ClaudeCodeStreamState:
result_text: str | None = None
is_error: bool = False
errors: Sequence[str] = ()
structured_output: Any | None = None
structured_output: object = None
@property
def final_text(self) -> str:
@ -157,7 +157,7 @@ def _stringify_block(block: object) -> str:
return json.dumps(block, ensure_ascii=False)
def _message_blocks(event: Mapping[str, Any]) -> Sequence[Any]:
def _message_blocks(event: Mapping[str, object]) -> Sequence[Any]:
message: Final = event.get("message")
content: Final = message.get("content") if isinstance(message, Mapping) else None
if isinstance(content, str):
@ -187,7 +187,7 @@ def _assistant_block_events(block: Mapping[str, Any], state: ClaudeCodeStreamSta
return ()
def _assistant_events(event: Mapping[str, Any], state: ClaudeCodeStreamState) -> Sequence[Event]:
def _assistant_events(event: Mapping[str, object], state: ClaudeCodeStreamState) -> Sequence[Event]:
if event.get("parent_tool_use_id"):
return event_list() # subagent traffic
message: Final = event.get("message")
@ -201,7 +201,7 @@ def _is_tool_result(block: object) -> bool:
return isinstance(block, dict) and block.get("type") == "tool_result"
def _user_events(event: Mapping[str, Any]) -> Sequence[Event]:
def _user_events(event: Mapping[str, object]) -> Sequence[Event]:
if event.get("parent_tool_use_id"):
return event_list()
return event_list(
@ -217,7 +217,7 @@ def _user_events(event: Mapping[str, Any]) -> Sequence[Event]:
)
def _system_events(event: Mapping[str, Any], state: ClaudeCodeStreamState) -> Sequence[Event]:
def _system_events(event: Mapping[str, object], state: ClaudeCodeStreamState) -> Sequence[Event]:
subtype = event.get("subtype")
if subtype == "init" and event.get("session_id"):
state.session_id = str(event["session_id"])
@ -253,7 +253,7 @@ def turn_error_message(state: ClaudeCodeStreamState, exit_code: int, stderr_tail
return f"{message}\nstderr:\n{tail}" if tail else message
def build_system_prompt(instructions: str | None, output_schema: Mapping[str, Any] | None) -> str | None:
def build_system_prompt(instructions: str | None, output_schema: Mapping[str, object] | None) -> str | None:
schema_part: Final = (
STRUCTURED_OUTPUT_INSTRUCTION.format(schema=json.dumps(output_schema)) if output_schema is not None else None
)
@ -355,7 +355,7 @@ class ClaudeCodeHarnessConfig(BaseCLIHarnessConfig):
def create_stream_state(self) -> ClaudeCodeStreamState:
return ClaudeCodeStreamState()
def transform_stream_line(self, line: Mapping[str, Any], state: ClaudeCodeStreamState) -> Sequence[Event]:
def transform_stream_line(self, line: Mapping[str, object], state: ClaudeCodeStreamState) -> Sequence[Event]:
kind = line.get("type")
if kind == "assistant":
return _assistant_events(line, state)

View file

@ -117,7 +117,7 @@ class OllamaChatConfig(BaseConfig):
system: str | None = None,
template: str | None = None,
) -> None:
locals_: Final = locals().copy()
locals_: Final[dict[str, object]] = locals().copy()
for key, value in locals_.items():
if key != "self" and value is not None:
setattr(self.__class__, key, value)

View file

@ -121,6 +121,7 @@ from litellm.llms.bedrock.common_utils import (
bedrock_route_for_request,
without_bedrock_route_prefix,
)
from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantle_chat_config
from litellm.llms.cohere.common_utils import CohereModelInfo
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
@ -2192,7 +2193,7 @@ def _complete_bedrock_mantle(
api_base = api_base or litellm.api_base or get_secret("BEDROCK_MANTLE_API_BASE")
api_key = api_key or litellm.api_key or get_secret("BEDROCK_MANTLE_API_KEY")
headers = headers or litellm.headers
config: Final = litellm.BedrockMantleChatConfig.get_config()
config: Final = bedrock_mantle_chat_config(model).get_config()
for k, v in _provider_config_items(config):
if k not in optional_params:
optional_params[k] = v

View file

@ -16085,6 +16085,28 @@
"input_cost_per_image_token": 4.7e-07,
"source": "https://cohere.com/pricing"
},
"cohere/embed-v5.0-pro": {
"input_cost_per_token": 1.2e-07,
"litellm_provider": "cohere",
"max_input_tokens": 128000,
"max_tokens": 128000,
"mode": "embedding",
"output_cost_per_token": 0.0,
"output_vector_size": 2048,
"supports_embedding_image_input": true,
"source": "https://cohere.com/pricing"
},
"cohere/embed-v5.0-fast": {
"input_cost_per_token": 8e-08,
"litellm_provider": "cohere",
"max_input_tokens": 128000,
"max_tokens": 128000,
"mode": "embedding",
"output_cost_per_token": 0.0,
"output_vector_size": 2048,
"supports_embedding_image_input": true,
"source": "https://cohere.com/pricing"
},
"cohere/parse-v5.0": {
"litellm_provider": "cohere",
"mode": "ocr",
@ -61056,6 +61078,9 @@
"input_cost_per_audio_token": 2e-06,
"input_cost_per_token": 2e-06,
"litellm_provider": "gemini",
"max_input_tokens": 98304,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "audio_transcription",
"output_cost_per_token": 1.2e-05,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
@ -61078,6 +61103,9 @@
"input_cost_per_audio_token": 3.5e-06,
"input_cost_per_token": 3.5e-06,
"litellm_provider": "gemini",
"max_input_tokens": 131072,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "audio_transcription",
"output_cost_per_token": 2.1e-05,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
@ -64092,6 +64120,23 @@
"video"
]
},
"xai/grok-imagine-video-1.5-lite": {
"input_cost_per_image": 0.01,
"litellm_provider": "xai",
"mode": "video_generation",
"output_cost_per_second": 0.02,
"output_cost_per_second_1080p": 0.14,
"output_cost_per_second_480p": 0.02,
"output_cost_per_second_720p": 0.03,
"source": "https://docs.x.ai/developers/pricing",
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"video"
]
},
"xai/grok-imagine-video-1.5-2026-05-30": {
"input_cost_per_image": 0.01,
"litellm_provider": "xai",
@ -75511,6 +75556,42 @@
"supports_vision": true,
"supports_web_search": true
},
"openrouter/openai/gpt-image-2": {
"input_cost_per_image_token": 8e-06,
"input_cost_per_token": 5e-06,
"litellm_provider": "openrouter",
"mode": "image_generation",
"output_cost_per_image_token": 3e-05,
"source": "https://openrouter.ai/api/v1/images/models/openai/gpt-image-2/endpoints",
"supported_endpoints": [
"/v1/images/generations"
],
"supports_vision": true
},
"openrouter/openai/gpt-image-2.5-flare": {
"input_cost_per_image_token": 8e-06,
"input_cost_per_token": 5e-06,
"litellm_provider": "openrouter",
"mode": "image_generation",
"output_cost_per_image_token": 3e-05,
"source": "https://openrouter.ai/api/v1/images/models/openai/gpt-image-2.5-flare/endpoints",
"supported_endpoints": [
"/v1/images/generations"
],
"supports_vision": true
},
"openrouter/openai/gpt-image-2.5-sunburst": {
"input_cost_per_image_token": 8e-06,
"input_cost_per_token": 5e-06,
"litellm_provider": "openrouter",
"mode": "image_generation",
"output_cost_per_image_token": 3e-05,
"source": "https://openrouter.ai/api/v1/images/models/openai/gpt-image-2.5-sunburst/endpoints",
"supported_endpoints": [
"/v1/images/generations"
],
"supports_vision": true
},
"openrouter/openai/gpt-5.4-mini:batch": {
"cache_read_input_token_cost": 3.75e-08,
"input_cost_per_token": 3.75e-07,

View file

@ -555,7 +555,7 @@ if MCP_AVAILABLE:
verbose_logger.warning("_prefetch_user_oauth_creds: failed to prefetch for user=%s: %s", user_id, e)
return {}
def _create_tool_response_objects(tools, server: MCPServer):
def _create_tool_response_objects(tools: Sequence[MCPTool], server: MCPServer):
"""Helper function to create tool response objects.
Enriches the server's ``mcp_info`` with ``server_id`` and ``alias`` so
@ -569,9 +569,7 @@ if MCP_AVAILABLE:
}
return [
ListMCPToolsRestAPIResponseObject(
name=tool.name,
description=tool.description,
inputSchema=tool.input_schema,
**tool.model_dump(by_alias=True, exclude={"mcp_info"}),
mcp_info=enriched_mcp_info,
)
for tool in tools

View file

@ -2,13 +2,19 @@ from __future__ import annotations
import hashlib
import json
from collections.abc import Mapping, Sequence
import re
from collections import deque
from collections.abc import Iterator, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime
from itertools import chain, islice
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, TypedDict
from pydantic import ValidationError
import anyio
from anyio import to_process
from anyio.lowlevel import RunVar
from pydantic import JsonValue, ValidationError
from typing_extensions import ReadOnly, Required, assert_never
import litellm
@ -44,6 +50,77 @@ VIRTUAL_TOOL_NAMES: Final = frozenset(
(MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME, AGENT_SEARCH_TOOL_NAME, SKILL_SEARCH_TOOL_NAME)
)
_SCHEMA_VALIDATION_LIMITER: Final = RunVar[anyio.CapacityLimiter]("mcp_schema_validation_limiter")
_MAX_VALIDATION_DEPTH: Final = 64
_MAX_VALIDATION_NODES: Final = 10_000
_MAX_VALIDATION_CHARACTERS: Final = 1_048_576
_VALIDATION_TIMEOUT_SECONDS: Final = 30
def _validation_nodes(value: JsonValue | Mapping[str, JsonValue]) -> Iterator[tuple[int, int]]:
pending: Final[deque[Iterator[JsonValue | Mapping[str, JsonValue]]]] = deque((iter((value,)),))
while pending:
try:
node, depth = next(pending[-1]), len(pending) - 1
except StopIteration:
pending.pop()
continue
yield depth, len(node) if isinstance(node, str) else 0
if depth > _MAX_VALIDATION_DEPTH:
continue
if isinstance(node, Mapping):
pending.append(chain(node.keys(), node.values()))
elif isinstance(node, list):
pending.append(iter(node))
def _validation_limit_error(schema: Mapping[str, JsonValue], arguments: Mapping[str, JsonValue]) -> str | None:
nodes: Final = tuple(
islice(chain(_validation_nodes(schema), _validation_nodes(arguments)), _MAX_VALIDATION_NODES + 1)
)
if len(nodes) > _MAX_VALIDATION_NODES or any(depth > _MAX_VALIDATION_DEPTH for depth, _ in nodes):
return "Tool schema or arguments exceed validation size or depth limits"
if sum(size for _, size in nodes) > _MAX_VALIDATION_CHARACTERS:
return "Tool schema or arguments exceed validation size or depth limits"
return None
def _validate_tool_arguments(schema: Mapping[str, JsonValue], arguments: Mapping[str, JsonValue]) -> str | None:
from jsonschema import validate
from jsonschema.exceptions import SchemaError
from jsonschema.exceptions import ValidationError as JsonSchemaValidationError
from referencing import Registry
from referencing.exceptions import Unresolvable
try:
validate(instance=arguments, schema=dict(schema), registry=Registry())
except JsonSchemaValidationError as exc:
return f"Invalid arguments: {exc.message}"
except (SchemaError, Unresolvable, RecursionError, re.error):
return "Unable to validate tool arguments against the supplied schema"
return None
async def _tool_argument_validation_error(
schema: Mapping[str, JsonValue], arguments: Mapping[str, JsonValue]
) -> str | None:
limit_error: Final = _validation_limit_error(schema, arguments)
if limit_error is not None:
return limit_error
existing: Final = _SCHEMA_VALIDATION_LIMITER.get(None)
limiter: Final = existing if existing is not None else anyio.CapacityLimiter(4)
if existing is None:
_SCHEMA_VALIDATION_LIMITER.set(limiter)
try:
with anyio.fail_after(_VALIDATION_TIMEOUT_SECONDS):
return await to_process.run_sync(
_validate_tool_arguments, schema, arguments, cancellable=True, limiter=limiter
)
except TimeoutError:
return "Tool argument validation exceeded its time limit"
except anyio.BrokenWorkerProcess:
return "Tool argument validation worker failed"
def coerce_top_k(value: Any, default: int = 5) -> int:
try:
@ -502,7 +579,7 @@ async def handle_mcp_tool_search(
async def handle_mcp_proxy_tool(
name: str,
arguments: dict[str, object], # mutable-ok: MCP dispatcher passes mutable call arguments
arguments: dict[str, JsonValue], # mutable-ok: MCP dispatcher passes mutable JSON call arguments
user_api_key_dict: UserAPIKeyAuth,
client_ip: str | None = None,
mcp_servers: list[str] | None = None, # mutable-ok: preserve MCP scope container for existing resolver
@ -513,8 +590,6 @@ async def handle_mcp_proxy_tool(
litellm_logging_obj: LiteLLMLoggingObj | None = None,
) -> CallToolResult:
from fastapi import HTTPException
from jsonschema import ValidationError as JsonSchemaValidationError
from jsonschema import validate
from litellm.proxy import proxy_server
from litellm.proxy._experimental.mcp_server.operations import (
@ -571,10 +646,9 @@ async def handle_mcp_proxy_tool(
tool_arguments: Final = arguments.get("arguments", {})
if not isinstance(tool_arguments, dict):
return _text_tool_result("arguments must be an object", is_error=True)
try:
validate(instance=tool_arguments, schema=tool.input_schema)
except JsonSchemaValidationError as exc:
return _text_tool_result(f"Invalid arguments: {exc.message}", is_error=True)
validation_error: Final = await _tool_argument_validation_error(tool.input_schema, tool_arguments)
if validation_error is not None:
return _text_tool_result(validation_error, is_error=True)
return await handle_mcp_tool_call(
tool_name=_mcp_proxy_identity(tool)["tool_name"],

View file

@ -13,7 +13,7 @@ import re
import secrets
from collections.abc import Mapping
from datetime import datetime, timezone
from typing import Any, Final, NamedTuple, Protocol, Union, cast
from typing import Final, NamedTuple, Protocol, Union, cast
import fastapi
import orjson
@ -215,7 +215,7 @@ def _get_model_from_request_context(
request_data: dict,
route: str,
request: Request | None,
llm_router: Any | None = None,
llm_router: litellm.Router | None = None,
team_id: str | None = None,
) -> str | list[str] | None:
return get_model_from_request(
@ -520,8 +520,8 @@ def _get_bearer_token_or_received_api_key(api_key: str) -> str:
def _routing_selector_matches_claim(
selector_value: Any | None,
claim_value: Any | None,
selector_value: object,
claim_value: object,
*,
split_space_delimited: bool = False,
) -> bool:
@ -661,7 +661,7 @@ async def user_api_key_auth_websocket_for_model(websocket: WebSocket, model: str
# ``websocket.url``, which Starlette reconstructs from the (poisonable)
# Host header. Carry the ASGI scope's path / root_path so the lookup
# never reaches the fallback.
synthetic_scope: Final[dict[str, Any]] = {
synthetic_scope: Final[dict[str, object]] = {
"type": "http",
"method": "GET",
"query_string": ws_scope.get("query_string", b""),
@ -3211,7 +3211,7 @@ async def _reserve_budget_after_common_checks(
user_api_key_auth_obj: UserAPIKeyAuth,
request_data: dict,
route: str,
llm_router: Any | None,
llm_router: litellm.Router | None,
team_object: LiteLLM_TeamTableCachedObj | None,
user_object: LiteLLM_UserTable | None,
prisma_client: PrismaClient | None,
@ -3257,7 +3257,7 @@ def _should_skip_budget_checks(
request_data: dict,
route: str,
request: Request | None,
llm_router: Any | None,
llm_router: litellm.Router | None,
team_id: str | None = None,
) -> bool:
model: Final = _get_model_from_request_context(
@ -3760,7 +3760,7 @@ async def _enforce_key_and_fallback_model_access(
route: str,
request: Request | None,
llm_model_list: list | None,
llm_router: Any | None,
llm_router: litellm.Router | None,
) -> None:
"""
Key-level model allowlist and client fallbacks (same as standard auth).

View file

@ -9,11 +9,19 @@ from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from types import MappingProxyType
from typing import Final, Literal, TypeAlias
from typing import Final, TypeAlias
import click
import requests
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from pydantic import BaseModel, TypeAdapter, ValidationError
from litellm.proxy.common_utils.codex_model_catalog import (
CODEX_BASE_INSTRUCTIONS_PATH,
CodexCatalogRow,
CodexStockCatalog,
CodexStockModel,
codex_models_response_json,
)
from .auth import CliContextObj, context_secret_vault, get_stored_api_key, login
from .claude_settings import ClaudeSettingsError, install_statusline_script
@ -68,7 +76,6 @@ _HIDDEN_AGENTS: Final = frozenset({"pi"})
CODEX_PROXY_PROVIDER: Final = "litellm"
CODEX_HOME_ENV: Final = "CODEX_HOME"
CODEX_MODEL_CATALOG_FILENAME: Final = "litellm-models.json"
_CODEX_BASE_INSTRUCTIONS_PATH: Final = Path(__file__).with_name("codex_base_instructions.md")
_CODEX_PREFLIGHT_TIMEOUT_SECONDS: Final = 10.0
@ -390,96 +397,8 @@ def opencode_model_sync_env(
return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing)})
class _CodexTruncationPolicy(BaseModel):
mode: Literal["bytes"] = "bytes"
limit: int = 10_000
class _CodexModel(BaseModel):
"""One `ModelInfo` entry of a Codex model catalog for a model the installed Codex does not know.
Every field that some Codex release since `model_catalog_json` appeared
(0.105.0) deserializes without a default is spelled out here, so one catalog
parses on all of them; the values match the fallback metadata Codex uses for
a model slug it does not know, so picking such a proxy model behaves the
same as `codex -m` did.
"""
slug: str
display_name: str
description: None = None
supported_reasoning_levels: tuple[()] = ()
shell_type: Literal["unified_exec"] = "unified_exec"
visibility: Literal["list"] = "list"
supported_in_api: Literal[True] = True
priority: int
availability_nux: None = None
upgrade: None = None
support_verbosity: Literal[False] = False
supports_reasoning_summaries: Literal[False] = False
supports_parallel_tool_calls: Literal[False] = False
default_verbosity: None = None
apply_patch_tool_type: None = None
truncation_policy: _CodexTruncationPolicy = _CodexTruncationPolicy()
experimental_supported_tools: tuple[()] = ()
context_window: int | None
base_instructions: str
class _StockCodexUpgrade(BaseModel):
model_config = ConfigDict(extra="allow")
model: str
class _StockCodexModel(BaseModel):
"""One `ModelInfo` entry as the installed Codex prints it from `codex debug models`.
Only the fields the sync rewrites are named; everything else that release
knows about the model (its reasoning levels, prompt, tool support) rides
along untouched, whatever the release's schema.
"""
model_config = ConfigDict(extra="allow")
slug: str
priority: int
visibility: str
supported_in_api: bool = True
upgrade: _StockCodexUpgrade | None = None
class _StockCodexCatalog(BaseModel):
models: tuple[_StockCodexModel, ...]
class _CodexCatalog(BaseModel):
models: tuple[_CodexModel | _StockCodexModel, ...]
def _codex_catalog_entry(
priority: int,
listed: ListedModel,
stock: _StockCodexModel | None,
served: frozenset[str],
instructions: str,
) -> _CodexModel | _StockCodexModel:
if stock is None:
return _CodexModel(
slug=listed.id,
display_name=listed.id,
priority=priority,
context_window=listed.max_input_tokens,
base_instructions=instructions,
)
upgrade: Final = stock.upgrade if stock.upgrade is not None and stock.upgrade.model in served else None
return stock.model_copy(
update={"priority": priority, "visibility": "list", "supported_in_api": True, "upgrade": upgrade}
)
def codex_model_catalog(
models: Sequence[ListedModel], stock: Sequence[_StockCodexModel], instructions: str
models: Sequence[ListedModel], stock: Sequence[CodexStockModel], instructions: str
) -> str | None:
"""The `model_catalog_json` body listing the proxy's chat models, or None if there are none.
@ -490,19 +409,14 @@ def codex_model_catalog(
Codex hides it or keeps it off the API, and keeps Codex's upgrade nudge only
when the model it points at is served too. A model Codex does not know gets the fallback
entry, with the same base instructions Codex itself uses so the agent never
runs without a system prompt.
runs without a system prompt. The catalog is the one `/v1/models?client_version=...`
serves, built from the installed Codex's own entries instead of the proxy's vendored ones.
"""
chat_models: Final = _chat_models(models)
if not chat_models:
return None
served: Final = frozenset(m.id for m in chat_models)
known: Final = MappingProxyType({m.slug: m for m in stock})
catalog: Final = _CodexCatalog(
models=tuple(
_codex_catalog_entry(index, m, known.get(m.id), served, instructions) for index, m in enumerate(chat_models)
)
rows: Final = tuple(CodexCatalogRow(id=m.id, mode=m.mode, max_input_tokens=m.max_input_tokens) for m in models)
body: Final = codex_models_response_json(
rows, stock=MappingProxyType({m.slug: m for m in stock}), instructions=instructions, byte_limit=None
)
return catalog.model_dump_json()
return body.json if body.listed else None
def codex_model_catalog_path(env: Mapping[str, str], *, home: Callable[[], Path] = Path.home) -> Path:
@ -558,12 +472,12 @@ def _codex_debug_models(
def _stock_codex_models(
binary: str, env: Mapping[str, str], *, run: Callable[..., subprocess.CompletedProcess[str]]
) -> tuple[_StockCodexModel, ...] | ModelSyncSkipped:
) -> tuple[CodexStockModel, ...] | ModelSyncSkipped:
printed: Final = _codex_debug_models(binary, (), env, run=run)
if isinstance(printed, ModelSyncSkipped):
return printed
try:
return _StockCodexCatalog.model_validate_json(printed).models
return CodexStockCatalog.model_validate_json(printed).models
except ValidationError as e:
name: Final = os.path.basename(binary)
return ModelSyncSkipped(f"`{name} debug models` printed no model catalog: {e.errors()[0]['msg']}")
@ -578,7 +492,7 @@ def codex_model_sync_args(
get: Callable[..., requests.Response] = requests.get,
run: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run,
home: Callable[[], Path] = Path.home,
instructions_path: Path = _CODEX_BASE_INSTRUCTIONS_PATH,
instructions_path: Path = CODEX_BASE_INSTRUCTIONS_PATH,
) -> ModelSyncArgs | ModelSyncSkipped:
"""`-c model_catalog_json=...` pointing Codex at the proxy's model list, or why it was skipped.

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1,420 @@
"""Codex-native model catalog for the model listing endpoints.
Codex CLI discovers a provider's models with ``GET <model_catalog_url>?client_version=<its version>``
(the provider's ``model_catalog_url`` pointed at ``/v1/models``, with its ``api_key_model_discovery``
feature on) and decodes the body as its own ``ModelsResponse``, ``{"models": [ModelInfo, ...]}``,
never the OpenAI list shape. This module builds that body from the rows ``/v1/models`` already
lists: a model Codex knows keeps Codex's own entry for it, any other model gets the fallback entry
Codex itself uses for an unknown slug, and ``model_info.service_tiers`` becomes the entry's
``service_tiers``, which Codex turns into slash commands (``/ultrafast``) that send ``service_tier``
upstream.
The stock entries are Codex 0.159.3's bundled catalog, vendored unchanged from
https://github.com/openai/codex/blob/rust-v0.159.3/codex-rs/models-manager/models.json (Apache-2.0)
as ``codex_bundled_models_0.159.3.json``, and ``codex_base_instructions.md`` is
``codex-rs/models-manager/prompt.md`` at the same tag. Refresh both by downloading those two paths at
a newer tag.
"""
from __future__ import annotations
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from functools import cache
from itertools import accumulate, pairwise
from pathlib import Path
from types import MappingProxyType, NoneType
from typing import TYPE_CHECKING, Annotated, Final, Literal
from pydantic import BaseModel, ConfigDict, StringConstraints, TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
if TYPE_CHECKING:
from litellm.router import Router
from litellm.types.proxy.model_listing import ModelInfoResponse
CODEX_BASE_INSTRUCTIONS_PATH: Final = Path(__file__).with_name("codex_base_instructions.md")
CODEX_BUNDLED_MODELS_PATH: Final = Path(__file__).with_name("codex_bundled_models_0.159.3.json")
CODEX_CATALOG_BYTE_LIMIT: Final = 1024 * 1024
CODEX_CHAT_MODES: Final = frozenset({"chat", "responses"})
_TierId = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)]
_BODY_PREFIX: Final = '{"models":['
_BODY_SUFFIX: Final = "]}"
class CodexServiceTier(BaseModel):
"""A `ModelServiceTier` as Codex reads it: the slash command is `name` lowercased, and toggling
it sends `id` as the request's `service_tier`."""
model_config = ConfigDict(frozen=True)
id: _TierId
name: str
description: str
class _ConfiguredTier(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
id: _TierId
name: str | None = None
description: str | None = None
_CONFIGURED_TIERS: Final = TypeAdapter(tuple[_TierId | _ConfiguredTier, ...])
class CodexTruncationPolicy(BaseModel):
mode: Literal["bytes"] = "bytes"
limit: int = 10_000
class CodexFallbackModel(BaseModel):
"""One `ModelInfo` entry of a Codex model catalog for a model Codex does not know.
Every field that some Codex release since `model_catalog_json` appeared
(0.105.0) deserializes without a default is spelled out here, so one catalog
parses on all of them; the values match the fallback metadata Codex uses for
a model slug it does not know, so picking such a proxy model behaves the
same as `codex -m` did.
"""
slug: str
display_name: str
description: NoneType = None
supported_reasoning_levels: tuple[()] = ()
shell_type: Literal["unified_exec"] = "unified_exec"
visibility: Literal["list"] = "list"
supported_in_api: Literal[True] = True
priority: int
service_tiers: tuple[CodexServiceTier, ...] = ()
availability_nux: NoneType = None
upgrade: NoneType = None
support_verbosity: Literal[False] = False
supports_reasoning_summaries: Literal[False] = False
supports_parallel_tool_calls: Literal[False] = False
default_verbosity: NoneType = None
apply_patch_tool_type: NoneType = None
truncation_policy: CodexTruncationPolicy = CodexTruncationPolicy()
experimental_supported_tools: tuple[()] = ()
context_window: int | None
base_instructions: str
class CodexStockUpgrade(BaseModel):
model_config = ConfigDict(extra="allow")
model: str
class CodexStockModel(BaseModel):
"""One `ModelInfo` entry as Codex ships it or prints it from `codex debug models`.
Only the fields the listing rewrites are named; everything else that release
knows about the model (its reasoning levels, prompt, tool support) rides
along untouched, whatever the release's schema.
"""
model_config = ConfigDict(extra="allow")
slug: str
display_name: str
priority: int
visibility: str
supported_in_api: bool = True
upgrade: CodexStockUpgrade | None = None
service_tiers: tuple[CodexServiceTier, ...] = ()
default_service_tier: str | None = None
class CodexStockCatalog(BaseModel):
models: tuple[CodexStockModel, ...]
CodexCatalogEntry = CodexFallbackModel | CodexStockModel
@dataclass(frozen=True, slots=True)
class CodexCatalogRow:
"""What the catalog needs to know about one listed model: its public id, the listing's mode and
input limit, the upstream model behind it, and the `model_info` values the operator set;
`service_tiers` holds one raw `model_info.service_tiers` value per deployment behind the id, None
where a deployment sets none."""
id: str
mode: str | None = None
max_input_tokens: int | None = None
upstream_model: str | None = None
display_name: str | None = None
service_tiers: tuple[object, ...] = ()
@cache
def bundled_codex_models() -> Mapping[str, CodexStockModel]:
"""Codex's bundled catalog by slug, read once from the vendored file."""
catalog: Final = CodexStockCatalog.model_validate_json(CODEX_BUNDLED_MODELS_PATH.read_text(encoding="utf-8"))
return MappingProxyType({model.slug: model for model in catalog.models})
@cache
def codex_base_instructions() -> str:
return CODEX_BASE_INSTRUCTIONS_PATH.read_text(encoding="utf-8")
def _tier(item: str | _ConfiguredTier, known: Mapping[str, CodexServiceTier]) -> CodexServiceTier:
if isinstance(item, str) and item in known:
return known[item]
configured: Final = _ConfiguredTier(id=item) if isinstance(item, str) else item
return CodexServiceTier(
id=configured.id,
name=configured.name if configured.name is not None else configured.id.capitalize(),
description=(
configured.description
if configured.description is not None
else f"Sends service_tier={configured.id} upstream"
),
)
def _first_by_id(tiers: Sequence[CodexServiceTier]) -> tuple[CodexServiceTier, ...]:
return tuple(tier for index, tier in enumerate(tiers) if tier.id not in {seen.id for seen in tiers[:index]})
_NO_KNOWN_TIERS: Final[Mapping[str, CodexServiceTier]] = MappingProxyType({})
def _deployment_tiers(
raw: object, model_id: str, known: Mapping[str, CodexServiceTier]
) -> tuple[CodexServiceTier, ...]:
try:
items: Final = _CONFIGURED_TIERS.validate_python(raw)
except ValidationError as e:
verbose_proxy_logger.warning(
"model_info.service_tiers for %s offers no tier, expected a list of tier ids or {id, name, description} objects: %s",
model_id,
e.errors()[0]["msg"],
)
return ()
return _first_by_id(tuple(_tier(item, known) for item in items))
def configured_service_tiers(
per_deployment: Sequence[object], model_id: str, known: Mapping[str, CodexServiceTier] = _NO_KNOWN_TIERS
) -> tuple[CodexServiceTier, ...] | None:
"""The tiers `model_info.service_tiers` configures for `model_id`, given each of its deployments'
raw value (None where unset), or None when no deployment sets one.
Accepts tier id strings and `{id, name, description}` objects. A string naming one of the
model's `known` tiers (the ones Codex ships for it) keeps that tier's name and description, so
`"priority"` stays Codex's `/fast`; any other string `x` is the command `/x` described as
"Sends service_tier=x upstream". A duplicate id keeps its first entry. A request to `model_id`
can route to any of its deployments, so a tier is offered only when every deployment lists it,
in the first deployment's order; a deployment that sets none leaves nothing to offer. Each of
those cases is logged, and an invalid value on any deployment is logged and offers no tier for
the whole model, stock tiers included, so one typo never fails the listing and never offers a
tier nobody declared.
"""
declared: Final = tuple(raw for raw in per_deployment if raw is not None)
if not declared:
return None
parsed: Final = tuple(_deployment_tiers(raw, model_id, known) for raw in declared)
if len(parsed) < len(per_deployment):
verbose_proxy_logger.warning(
"model_info.service_tiers for %s is set on %d of its %d deployments, so no tier is offered: "
"a tier is offered only when every deployment lists it",
model_id,
len(parsed),
len(per_deployment),
)
return ()
tier_ids: Final = tuple(frozenset(tier.id for tier in tiers) for tiers in parsed)
shared: Final = tier_ids[0].intersection(*tier_ids[1:])
offered: Final = tuple(tier for tier in parsed[0] if tier.id in shared)
if len(offered) < len(parsed[0]):
verbose_proxy_logger.warning(
"model_info.service_tiers for %s differs between its deployments, offering only the tiers every one lists: %s",
model_id,
", ".join(tier.id for tier in offered) or "none",
)
return offered
def _codex_can_drive(row: CodexCatalogRow) -> bool:
return "*" not in row.id and (row.mode is None or row.mode in CODEX_CHAT_MODES)
def _stock_for(row: CodexCatalogRow, stock: Mapping[str, CodexStockModel]) -> CodexStockModel | None:
listed: Final = stock.get(row.id)
if listed is not None or row.upstream_model is None:
return listed
upstream: Final = stock.get(row.upstream_model)
return upstream if upstream is not None else stock.get(row.upstream_model.partition("/")[2])
def _entry(
index: int,
row: CodexCatalogRow,
stock: Mapping[str, CodexStockModel],
served: frozenset[str],
instructions: str,
) -> CodexCatalogEntry:
stock_model: Final = _stock_for(row, stock)
stock_tiers: Final = MappingProxyType(
{tier.id: tier for tier in (stock_model.service_tiers if stock_model is not None else ())}
)
configured: Final = configured_service_tiers(row.service_tiers, row.id, stock_tiers)
if stock_model is None:
return CodexFallbackModel(
slug=row.id,
display_name=row.display_name if row.display_name is not None else row.id,
priority=index,
service_tiers=configured if configured is not None else (),
context_window=row.max_input_tokens,
base_instructions=instructions,
)
upgrade: Final = (
stock_model.upgrade if stock_model.upgrade is not None and stock_model.upgrade.model in served else None
)
stock_name: Final = stock_model.display_name if row.id == stock_model.slug else row.id
listing: Final = {
"slug": row.id,
"display_name": row.display_name if row.display_name is not None else stock_name,
"priority": index,
"visibility": "list",
"supported_in_api": True,
"upgrade": upgrade,
}
if configured is None:
return stock_model.model_copy(update=listing)
configured_ids: Final = frozenset(tier.id for tier in configured)
return stock_model.model_copy(
update={
**listing,
"service_tiers": configured,
"default_service_tier": (
stock_model.default_service_tier if stock_model.default_service_tier in configured_ids else None
),
}
)
def _serialized(entry: CodexCatalogEntry) -> str:
"""A stock entry keeps exactly the fields its source printed (an older Codex may know no
`service_tiers`), a fallback entry spells out every field Codex requires."""
return entry.model_dump_json(exclude_unset=isinstance(entry, CodexStockModel))
def _bytes_left_after(left: int, size: int) -> int:
return left - size if size <= left else left
def _fits(serialized: Sequence[str], byte_limit: int | None) -> tuple[bool, ...]:
"""Whether each entry, taken in order, fits in what is left of a body of at most `byte_limit`
bytes: an entry larger than the bytes left is passed over and a smaller one after it is still
taken, each entry counted with its separating comma and the last one's comma given back."""
if byte_limit is None:
return (True,) * len(serialized)
budget: Final = byte_limit - len(_BODY_PREFIX) - len(_BODY_SUFFIX) + 1
left: Final = tuple(
accumulate((len(entry.encode()) + 1 for entry in serialized), _bytes_left_after, initial=budget)
)
return tuple(after < before for before, after in pairwise(left))
def _kept_positions(
entries: Sequence[CodexCatalogEntry], serialized: Sequence[str], byte_limit: int | None
) -> frozenset[int]:
"""The listing positions that survive the byte limit: an entry offering a service tier is taken
ahead of one offering none, each group in listing order, since a model without a tier is driven
by name just as well off Codex's bundled catalog; an entry too large for the bytes left never
costs the smaller ones after it their place."""
survival: Final = sorted(range(len(entries)), key=lambda position: (not entries[position].service_tiers, position))
fits: Final = _fits(tuple(serialized[position] for position in survival), byte_limit)
return frozenset(position for position, fit in zip(survival, fits, strict=True) if fit)
@dataclass(frozen=True, slots=True)
class CodexCatalogBody:
"""A `ModelsResponse` body, the ids it lists, and the chat-capable ids the byte limit left out."""
json: str
listed: tuple[str, ...]
left_out: tuple[str, ...]
def codex_models_response_json(
rows: Sequence[CodexCatalogRow],
*,
stock: Mapping[str, CodexStockModel] | None = None,
instructions: str | None = None,
byte_limit: int | None = CODEX_CATALOG_BYTE_LIMIT,
) -> CodexCatalogBody:
"""Codex's `ModelsResponse` body for `rows`, in listing order, cut to Codex's byte limit.
Only chat-capable rows are listed (Codex cannot drive an embedding or image model, and a
wildcard id is no model), their order is their `priority`. Codex drops a catalog over its
byte limit silently and keeps its bundled one, so the body holds only the entries that fit,
the ones offering a service tier taken first and the rest in listing order, an entry too large
for the bytes left passed over, every kept entry at its listing position; `byte_limit=None`
keeps every entry.
"""
known: Final = bundled_codex_models() if stock is None else stock
prompt: Final = codex_base_instructions() if instructions is None else instructions
drivable: Final = tuple(row for row in rows if _codex_can_drive(row))
served: Final = frozenset(row.id for row in drivable)
entries: Final = tuple(_entry(index, row, known, served, prompt) for index, row in enumerate(drivable))
serialized: Final = tuple(_serialized(entry) for entry in entries)
kept: Final = _kept_positions(entries, serialized, byte_limit)
return CodexCatalogBody(
json=f"{_BODY_PREFIX}{','.join(serialized[position] for position in sorted(kept))}{_BODY_SUFFIX}",
listed=tuple(row.id for position, row in enumerate(drivable) if position in kept),
left_out=tuple(row.id for position, row in enumerate(drivable) if position not in kept),
)
def _catalog_row(
row: ModelInfoResponse, lookup_id: str, llm_router: Router | None, team_id: str | None
) -> CodexCatalogRow:
return CodexCatalogRow(
id=row["id"],
mode=row.get("mode"),
max_input_tokens=row.get("max_input_tokens"),
upstream_model=llm_router.get_routable_upstream_model(lookup_id, team_id) if llm_router is not None else None,
display_name=llm_router.get_configured_display_name(lookup_id) if llm_router is not None else None,
service_tiers=llm_router.get_configured_service_tiers(lookup_id, team_id) if llm_router is not None else (),
)
def codex_catalog_rows(
rows: Sequence[ModelInfoResponse],
entries: Sequence[tuple[str, str]],
llm_router: Router | None,
team_id: str | None = None,
) -> tuple[CodexCatalogRow, ...]:
"""`rows` joined with the router's configured metadata, looked up by each entry's internal id so
team-scoped rows resolve the way the Anthropic listing's display names do, a `model_group_alias`
reading its target's deployments under its own name; `team_id` is the requesting key's team, so the
upstream model that picks Codex's stock entry and the tiers are read only off the deployments its
requests can route to."""
lookup_ids: Final = MappingProxyType(dict(entries))
return tuple(_catalog_row(row, lookup_ids.get(row["id"], row["id"]), llm_router, team_id) for row in rows)
def codex_model_list_body(
rows: Sequence[ModelInfoResponse],
entries: Sequence[tuple[str, str]],
llm_router: Router | None,
team_id: str | None = None,
) -> str:
"""The `/v1/models?client_version=...` body, logging the models Codex's byte limit left out."""
body: Final = codex_models_response_json(codex_catalog_rows(rows, entries, llm_router, team_id))
if body.left_out:
verbose_proxy_logger.warning(
"Codex model catalog cut at %d bytes, left out: %s. Models offering a service tier are kept first, "
"then model_list order; list the models Codex users need first in model_list",
CODEX_CATALOG_BYTE_LIMIT,
", ".join(body.left_out),
)
return body.json

View file

@ -254,6 +254,40 @@ def translate_libpq_ssl_params(url: str, resolve_root_cert: RootCertResolver = p
return urllib.parse.urlunsplit(parsed._replace(query=query))
def postgres_connection_budget_message(writer_limit: str, reader_limit: str | None, num_workers: str) -> str:
"""The startup line that states this pod's worst-case Postgres connection demand.
Prisma's ``connection_limit`` is per query engine, and every uvicorn worker
owns one engine per configured database (writer, plus the reader when
``DATABASE_URL_READ_REPLICA`` is set). The limits are read off the final URLs,
so a reader that pins its own ``connection_limit`` or an operator override in
``database_extra_connection_params`` is counted at its real value. The
server-side cap is shared by every pod, so the number an operator has to keep
under ``max_connections`` minus ``superuser_reserved_connections`` is
pods x workers x the per-worker sum, not one engine's limit.
"""
try:
workers: Final = max(1, int(num_workers))
writer: Final = int(writer_limit)
reader: Final = int(reader_limit) if reader_limit is not None else 0
except ValueError:
return (
"LiteLLM Proxy: Postgres connection budget per pod = workers x (writer connection_limit + reader "
f"connection_limit) (workers={num_workers!r}, writer={writer_limit!r}, reader={reader_limit!r}); "
"keep pods x that figure under max_connections minus superuser_reserved_connections"
)
engines: Final = (
f"(writer connection_limit {writer} + reader connection_limit {reader})"
if reader_limit is not None
else f"writer connection_limit {writer}"
)
per_pod: Final = workers * (writer + reader)
return (
f"LiteLLM Proxy: Postgres connection budget per pod = {workers} worker(s) x {engines} = up to {per_pod} "
"connections; keep pods x that figure under max_connections minus superuser_reserved_connections"
)
def reader_shareable_params(params: Mapping[str, str | int | float]) -> Mapping[str, str | int | float]:
"""Return the subset of ``params`` the read replica is allowed to inherit."""
return MappingProxyType({key: value for key, value in params.items() if key in CONNECTION_PARAM_KEYS})
@ -266,16 +300,26 @@ def connection_params_from_url(url: str) -> Mapping[str, str | int | float]:
)
# A re-minted token URL replaces a URL to the same database, so unlike the
# reader allowlist it may also carry ``options``: that is where the server-side
# timeouts (statement, lock, idle-in-transaction) live, and a refresh that
# dropped them would leave the replacement engine's sessions unbounded.
TOKEN_REFRESH_PARAM_KEYS: Final[frozenset[str]] = CONNECTION_PARAM_KEYS | PRISMA_TLS_PARAM_KEYS | frozenset({"options"})
def token_refresh_params_from_url(url: str) -> Mapping[str, str | int | float]:
"""Return the params a re-minted token URL carries over from the URL it replaces.
The pool and timeout params plus Prisma's TLS params (already translated from
libpq spelling), so a refreshed URL keeps verifying the server the way the
first one did.
The pool and timeout params, Prisma's TLS params (already translated from
libpq spelling) and the ``options`` string, so a refreshed URL keeps verifying
the server and bounding its sessions the way the first one did.
"""
kept: Final = CONNECTION_PARAM_KEYS | PRISMA_TLS_PARAM_KEYS
return MappingProxyType(
{key: value for key, value in urllib.parse.parse_qsl(urllib.parse.urlsplit(url).query) if key in kept}
{
key: value
for key, value in urllib.parse.parse_qsl(urllib.parse.urlsplit(url).query)
if key in TOKEN_REFRESH_PARAM_KEYS
}
)

View file

@ -24,6 +24,13 @@ _TRANSIENT_DB_UNAVAILABLE_MESSAGE: Final = (
_DATABASE_ERROR_META: Final = TypeAdapter(dict[str, object])
_BATCH_POSTGRES_ERROR_CODE: Final = re.compile(r'PostgresError \{ code: "([0-9A-Z]{5})"')
_CONNECTION_CAPACITY_PHRASES: Final = (
"too many clients already",
"too many connections for role",
"too many connections for database",
"remaining connection slots are reserved",
)
_PRISMA_POOL_TIMEOUT_CODE: Final = "P2024"
def _exception_chain(e: BaseException) -> Iterator[BaseException]:
@ -109,6 +116,8 @@ class PrismaDBExceptionHandler:
return True
if isinstance(e, _exception_types(prisma.engine.errors.EngineConnectionError)):
return True
if PrismaDBExceptionHandler.is_database_capacity_error(e):
return True
return isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection
@staticmethod
@ -192,6 +201,8 @@ class PrismaDBExceptionHandler:
),
):
return True
if PrismaDBExceptionHandler.is_database_capacity_error(e):
return False
if isinstance(e, _exception_types(prisma.errors.PrismaError)):
error_message: Final = str(e).lower()
connection_keywords: Final = (
@ -236,6 +247,25 @@ class PrismaDBExceptionHandler:
or "write conflict or a deadlock" in error_message
)
@staticmethod
def is_database_capacity_error(e: Exception) -> bool:
"""True iff Postgres refused the pool a new connection (SQLSTATE 53300:
``too many clients already``, a reserved slot, or a per-role limit) or
prisma's own pool timed out handing one over (P2024). The server is up
but full, so the failure is neither a transport error (which would tear
the engine down and open yet more connections against it) nor a
rejection of the rows being written."""
import prisma
if not isinstance(e, _exception_types(prisma.errors.PrismaError)):
return False
if isinstance(e, prisma.errors.DataError) and e.code == _PRISMA_POOL_TIMEOUT_CODE:
return True
if PrismaDBExceptionHandler.postgres_sqlstate(e) == "53300":
return True
error_message: Final = str(e).lower()
return any(phrase in error_message for phrase in _CONNECTION_CAPACITY_PHRASES)
@staticmethod
def postgres_sqlstate(e: Exception) -> str | None:
"""The SQLSTATE Postgres attached to a failed statement, as prisma surfaces it, or None."""
@ -325,6 +355,8 @@ class PrismaDBExceptionHandler:
return True
if PrismaDBExceptionHandler.is_database_transport_error(e):
return True
if PrismaDBExceptionHandler.is_database_capacity_error(e):
return True
if PrismaDBExceptionHandler.is_prisma_engine_internal_error(e):
return True
if "cached plan must not change result type" in str(e).lower():

View file

@ -37,18 +37,24 @@ from litellm.proxy.lens.models import (
RunRequest,
Sample,
Scope,
Step,
WatchAllResult,
WatchSkipped,
Worker,
WorkerCreated,
)
from litellm.proxy.lens.repository import LensRepository, WriterDatabase
from litellm.proxy.lens.sources import ActivityAvailability, SourceReader, Storage, parse_execution
from litellm.proxy.lens.state import (
add_step,
can_access,
claim_job,
current_job,
merge_finding,
next_scan_start,
queue_job,
replace_job,
scheduled_window,
snapshot_finding,
)
from litellm.proxy.tracing_runtime import provide_storage
@ -227,6 +233,40 @@ async def list_agents(auth: Auth, storage: StorageDep) -> tuple[str, ...]:
return await source_reader(storage).agents(scope) if storage is not None else ()
def watching(lens: Lens) -> Lens:
if lens.settings.enabled:
return lens
return lens.model_copy(
update=MappingProxyType(
{
"settings": lens.settings.model_copy(update=MappingProxyType({"enabled": True})),
"revision": lens.revision + 1,
}
)
)
async def watchable(lens: Lens, auth: UserAPIKeyAuth) -> WatchSkipped | None:
try:
await validate_model(lens.settings.model_copy(update=MappingProxyType({"enabled": True})), auth)
except HTTPException as exc:
return WatchSkipped(id=lens.id, name=lens.settings.name, reason=str(exc.detail))
return None
@router.post("/watch-all", response_model=WatchAllResult)
async def watch_all(auth: Auth) -> WatchAllResult:
scope: Final = user_scope(auth, write=True)
paused: Final = tuple(
e for e in await repository().lenses() if can_access(scope, e.scope) and not e.settings.enabled
)
checks: Final = tuple([(lens, await watchable(lens, auth)) for lens in paused])
skipped: Final = tuple(skip for _, skip in checks if skip is not None)
ready: Final = tuple(lens for lens, skip in checks if skip is None)
updated: Final = tuple([await repository().update(lens.id, watching) for lens in ready])
return WatchAllResult(watching=tuple(u.id for u in updated if u is not None), skipped=skipped)
@router.put("/{lens_id}", response_model=Lens)
async def update_lens(lens_id: str, settings: LensSettings, auth: Auth) -> Lens:
lens: Final = await get_lens(lens_id, user_scope(auth, write=True))
@ -248,6 +288,20 @@ async def update_lens(lens_id: str, settings: LensSettings, auth: Auth) -> Lens:
)
def run_window(lens: Lens, body: RunRequest, now: datetime) -> tuple[datetime, datetime] | None:
if body.start is not None and body.end is not None:
return body.start, body.end
if body.lookback_hours is None and body.settings is None:
return scheduled_window(lens, now)
return None
def run_settings(lens: Lens, body: RunRequest) -> LensSettings | None:
if body.agent_name is None:
return body.settings
return (body.settings or lens.settings).model_copy(update=MappingProxyType({"agent_name": body.agent_name}))
@router.post("/{lens_id}/runs", response_model=Lens)
async def run_lens(lens_id: str, body: RunRequest, auth: Auth) -> Lens:
lens: Final = await get_lens(lens_id, user_scope(auth, write=True))
@ -257,7 +311,18 @@ async def run_lens(lens_id: str, body: RunRequest, auth: Auth) -> Lens:
now: Final = datetime.now(timezone.utc)
job_id: Final = str(uuid4())
return required(
await repository().update(lens_id, lambda e: queue_job(e, now, job_id, body.lookback_hours, body.settings))
await repository().update(
lens_id,
lambda e: queue_job(
e,
now,
job_id,
body.lookback_hours,
run_settings(e, body),
run_window(e, body, now),
"manual",
),
)
)
@ -425,13 +490,14 @@ async def progress(lens_id: str, job_id: str, body: Progress, worker: WorkerAuth
job: Final = current_job(e)
if job is None or job.id != job_id or job.worker_id != worker.id:
return e
renewed: Final = job.model_copy(
update=MappingProxyType(
{"stage": body.stage, "coverage": body.coverage, "lease_until": now + timedelta(minutes=5)}
)
)
return replace_job(
e,
job.model_copy(
update=MappingProxyType(
{"stage": body.stage, "coverage": body.coverage, "lease_until": now + timedelta(minutes=5)}
)
),
renewed if body.stage == job.stage else add_step(renewed, Step(at=now, kind="stage", label=body.stage)),
)
required(await repository().update(lens_id, renew))
@ -575,7 +641,7 @@ async def result(lens_id: str, job_id: str, body: Result, worker: WorkerAuth, st
update=MappingProxyType(
{
"findings": (*merged, *(f for f in e.findings if f.id not in merged_ids)),
"last_scan_at": e.last_scan_at if body.error else max(e.last_scan_at or job.end, job.end),
"last_scan_at": next_scan_start(e, job, failed=bool(body.error)),
"next_run_at": now + timedelta(minutes=e.settings.interval_minutes),
}
)

View file

@ -13,9 +13,9 @@ from litellm.integrations.clickhouse.context import lens_analysis
from litellm.litellm_core_utils.initialize_dynamic_callback_params import inherit_message_logging_privacy
from litellm.litellm_core_utils.token_counter import get_modified_max_tokens
from litellm.proxy.lens.billing import complete, validate_key
from litellm.proxy.lens.models import Job, Lens, ModelRequest, ModelResult, Worker
from litellm.proxy.lens.models import Job, Lens, ModelRequest, ModelResult, Step, Worker
from litellm.proxy.lens.repository import LensRepository
from litellm.proxy.lens.state import current_job, renew_budget, replace_job
from litellm.proxy.lens.state import add_step, current_job, renew_budget, replace_job
from litellm.types.utils import CostPerToken, ModelResponse
@ -192,20 +192,17 @@ async def analyze(
current, active.model_copy(update=MappingProxyType({"cost": active.cost + estimate}))
).model_copy(update=MappingProxyType({"spent": current.spent + estimate}))
def settle(e: Lens, cost: float) -> Lens:
def settle(e: Lens, cost: float, step: Step | None) -> Lens:
charged: Final = next((j for j in e.jobs if j.id == job.id), None)
adjusted: Final = (
e.model_copy(update=MappingProxyType({"spent": max(0, e.spent - estimate + cost)}))
if e.budget_month == now.strftime("%Y-%m")
else e
)
return (
replace_job(
adjusted, charged.model_copy(update=MappingProxyType({"cost": max(0, charged.cost - estimate + cost)}))
)
if charged
else adjusted
)
if charged is None:
return adjusted
refunded: Final = charged.model_copy(update=MappingProxyType({"cost": max(0, charged.cost - estimate + cost)}))
return replace_job(adjusted, add_step(refunded, step) if step is not None else refunded)
@asynccontextmanager
async def reserve_budget() -> AsyncIterator[None]:
@ -214,7 +211,7 @@ async def analyze(
try:
yield
except BaseException:
await repo.update(lens.id, lambda e: settle(e, 0))
await repo.update(lens.id, lambda e: settle(e, 0, None))
raise
data: Final[dict[str, object]] = { # mutable-ok: proxy processing enriches request data
@ -241,7 +238,8 @@ async def analyze(
response, billed_cost = await complete(worker.analysis_key_id, data, reserve_budget, request)
cost: Final = billed_cost if billed_cost is not None else completion_charge(deployments, response, estimate)
await repo.update(lens.id, lambda e: settle(e, cost))
step: Final = model_step(response, body, job.settings.model, cost)
await repo.update(lens.id, lambda e: settle(e, cost, step))
parsed: Final = Completion.model_validate_json(response.model_dump_json())
choice: Final = parsed.choices[0]
return ModelResult(
@ -253,6 +251,37 @@ async def analyze(
)
class Usage(BaseModel):
model_config = ConfigDict(extra="ignore")
prompt_tokens: int | None = None
completion_tokens: int | None = None
class UsageEnvelope(BaseModel):
model_config = ConfigDict(extra="ignore")
model: str | None = None
usage: Usage | None = None
_PURPOSE_LABELS: Final = MappingProxyType(
{"extract": "Reviewed a run", "cluster": "Compared observations", "investigate": "Checked a pattern"}
)
def model_step(response: ModelResponse, body: ModelRequest, requested: str, cost: float) -> Step:
envelope: Final = UsageEnvelope.model_validate_json(response.model_dump_json())
return Step(
at=datetime.now(timezone.utc),
kind="model",
label=_PURPOSE_LABELS[body.purpose],
model=envelope.model or requested,
purpose=body.purpose,
prompt_tokens=(envelope.usage.prompt_tokens if envelope.usage else None) or 0,
completion_tokens=(envelope.usage.completion_tokens if envelope.usage else None) or 0,
cost=cost,
)
def completion_charge(deployments: tuple[Deployment, ...], response: ModelResponse, estimate: float) -> float:
custom: Final = deployments[0].litellm_params if len(deployments) == 1 else None
if custom and custom.input_cost_per_token is not None and custom.output_cost_per_token is not None:

View file

@ -190,6 +190,20 @@ class RunAssessment(Record):
cannot_assess: bool = False
MAX_STEPS = 200
class Step(Record):
at: datetime
kind: Literal["stage", "model", "error"]
label: str = Field(max_length=200)
model: str = Field(default="", max_length=200)
purpose: str = Field(default="", max_length=40)
prompt_tokens: int = 0
completion_tokens: int = 0
cost: float = 0
class Job(Record):
id: str
status: Literal["queued", "running", "completed", "failed", "cancelled"] = "queued"
@ -209,6 +223,8 @@ class Job(Record):
cost: float = 0
findings: tuple[Finding, ...] | None = None
assessments: tuple[RunAssessment, ...] = ()
steps: tuple[Step, ...] = ()
trigger: Literal["schedule", "manual"] = "schedule"
class Lens(Record):
@ -249,6 +265,28 @@ class LensList(Record):
class RunRequest(Record):
settings: LensSettings | None = None
lookback_hours: LookbackHours | None = None
start: datetime | None = None
end: datetime | None = None
agent_name: str | None = Field(default=None, max_length=200)
@model_validator(mode="after")
def ordered_window(self) -> "RunRequest":
if (self.start is None) != (self.end is None):
raise ValueError("Choose both a start and an end time")
if self.start is not None and self.end is not None and self.start >= self.end:
raise ValueError("Start time must be before end time")
return self
class WatchSkipped(Record):
id: str
name: str
reason: str
class WatchAllResult(Record):
watching: tuple[str, ...]
skipped: tuple[WatchSkipped, ...] = ()
class FindingUpdate(Record):

View file

@ -1,9 +1,19 @@
import hashlib
from datetime import datetime, timedelta
from types import MappingProxyType
from typing import Final
from typing import Final, Literal
from litellm.proxy.lens.models import Finding, FindingDraft, Job, Lens, LensSettings, Scope, Worker
from litellm.proxy.lens.models import (
MAX_STEPS,
Finding,
FindingDraft,
Job,
Lens,
LensSettings,
Scope,
Step,
Worker,
)
def can_access(viewer: Scope, target: Scope) -> bool:
@ -24,27 +34,54 @@ def replace_job(lens: Lens, job: Job) -> Lens:
)
SETTLE_DELAY = timedelta(minutes=2)
def scheduled_window(lens: Lens, now: datetime) -> tuple[datetime, datetime]:
end: Final = now - SETTLE_DELAY
floor: Final = now - timedelta(hours=lens.settings.lookback_hours)
start: Final = max(lens.last_scan_at, floor) if lens.last_scan_at else floor
return min(start, end), end
def next_scan_start(lens: Lens, job: Job, failed: bool) -> datetime | None:
if failed or job.trigger == "manual":
return lens.last_scan_at
return max(lens.last_scan_at or job.end, job.end)
def queue_job(
lens: Lens,
now: datetime,
job_id: str,
lookback_hours: int | None = None,
settings: LensSettings | None = None,
window: tuple[datetime, datetime] | None = None,
trigger: Literal["schedule", "manual"] = "schedule",
) -> Lens:
if current_job(lens):
return lens
selected: Final = settings or lens.settings
hours: Final = lookback_hours if lookback_hours is not None else (selected.lookback_hours if settings else None)
start, end = window or (
(now - timedelta(hours=hours), now - SETTLE_DELAY) if hours is not None else scheduled_window(lens, now)
)
job: Final = Job(
id=job_id,
created_at=now,
start=now - timedelta(hours=lookback_hours if lookback_hours is not None else selected.lookback_hours),
end=now - timedelta(minutes=2),
start=start,
end=end,
settings=selected,
revision=lens.revision,
trigger=trigger,
)
return lens.model_copy(update=MappingProxyType({"jobs": (job,)}))
def add_step(job: Job, step: Step) -> Job:
return job.model_copy(update=MappingProxyType({"steps": (*job.steps, step)[-MAX_STEPS:]}))
def claim_job(lens: Lens, worker: Worker, now: datetime) -> Lens:
job: Final = current_job(lens)
if job is None or not can_access(worker.scope, lens.scope):

View file

@ -108,12 +108,14 @@ class DatabaseTimeoutSettings(BaseModel):
database_statement_timeout: float | None = None
database_lock_timeout: float | None = None
database_idle_in_transaction_session_timeout: float | None = None
def _pg_options_with_timeouts(
existing_options: str,
statement_timeout: float | None,
lock_timeout: float | None,
idle_in_transaction_session_timeout: float | None = None,
) -> str:
"""Return the Postgres ``options`` value carrying the configured timeouts.
@ -143,6 +145,7 @@ def _pg_options_with_timeouts(
for name, seconds in (
("statement_timeout", statement_timeout),
("lock_timeout", lock_timeout),
("idle_in_transaction_session_timeout", idle_in_transaction_session_timeout),
)
if seconds is not None and not re.search(rf"(?:-c\s*|--){re.escape(name)}=", existing_options)
)
@ -1160,6 +1163,7 @@ def run_server(
db_extra_connection_params: dict | None = None
db_statement_timeout: float | None = None
db_lock_timeout: float | None = None
db_idle_in_transaction_timeout: float | None = None
general_settings = {}
### GET DB TOKEN FOR RDS IAM / AZURE ENTRA AUTH ###
@ -1270,6 +1274,7 @@ def run_server(
db_timeouts: Final = DatabaseTimeoutSettings.model_validate(general_settings)
db_statement_timeout = db_timeouts.database_statement_timeout
db_lock_timeout = db_timeouts.database_lock_timeout
db_idle_in_transaction_timeout = db_timeouts.database_idle_in_transaction_session_timeout
if database_url and database_url.startswith("os.environ/"):
original_dir: Final = os.getcwd()
# set the working directory to where this script is
@ -1303,6 +1308,7 @@ def run_server(
DISABLE_PREPARED_STATEMENTS_ENV_VAR,
add_missing_query_params,
idle_lifetime_params,
postgres_connection_budget_message,
reader_shareable_params,
translate_libpq_ssl_params,
unsupported_db_scheme,
@ -1343,6 +1349,7 @@ def run_server(
_url_query_value(resolved_url, "options"),
db_statement_timeout,
db_lock_timeout,
db_idle_in_transaction_timeout,
)
writer_url: Final = (
_with_query_value(resolved_url, "options", pg_options)
@ -1373,6 +1380,7 @@ def run_server(
_url_query_value(read_replica_url, "options"),
db_statement_timeout,
db_lock_timeout,
db_idle_in_transaction_timeout,
)
os.environ["DATABASE_URL_READ_REPLICA"] = translate_libpq_ssl_params(
add_missing_query_params(
@ -1385,6 +1393,19 @@ def run_server(
lifetime_params,
)
)
print(
postgres_connection_budget_message(
writer_limit=_url_query_value(os.getenv("DATABASE_URL"), "connection_limit")
or f"{db_connection_pool_limit}",
reader_limit=(
_url_query_value(os.getenv("DATABASE_URL_READ_REPLICA"), "connection_limit")
or f"{db_connection_pool_limit}"
)
if read_replica_url
else None,
num_workers=f"{1 if run_hypercorn else num_workers}",
)
)
from litellm_proxy_extras.prisma_toolchain import prisma_cli_available
is_prisma_runnable: Final = prisma_cli_available()

View file

@ -410,6 +410,7 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
AuthCacheInvalidationSubscriber,
)
from litellm.proxy.common_utils.callback_utils import initialize_callbacks_on_proxy
from litellm.proxy.common_utils.codex_model_catalog import codex_model_list_body
from litellm.proxy.common_utils.config_includes import resolve_include_file_path, resolve_includes
from litellm.proxy.common_utils.config_sync_pubsub import ConfigSyncSubscriber
from litellm.proxy.common_utils.debug_utils import init_verbose_loggers
@ -9443,7 +9444,7 @@ def _format_fallback_metadata_sse_event(
def _restamp_streaming_chunk_model(
*,
chunk: Any,
chunk: object,
requested_model_from_client: str,
request_data: dict,
model_mismatch_logged: bool,
@ -11596,6 +11597,7 @@ async def model_list(
fallback_type: str | None = None,
scope: str | None = None,
healthy_only: bool | None = False,
client_version: str | None = None,
):
"""
Use `/model/info` - to get detailed model information, example - pricing, mode, etc.
@ -11603,6 +11605,11 @@ async def model_list(
This is just for compatibility with openai projects like aider.
Query Parameters:
- client_version: Sent by Codex CLI (`?client_version=0.159.3`) when it fetches a
provider's model catalog. When present, the response is Codex's own
catalog shape (`{"models": [...]}`) built from the same listing, with
each model's `model_info.service_tiers` as its service tiers; absent,
the OpenAI shape below is returned
- include_metadata: Include additional metadata in the response with fallback information
- fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy")
Defaults to "general" when include_metadata=true
@ -11745,8 +11752,16 @@ async def model_list(
model_info["id"] = response_id
model_data.append(model_info)
admin_listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above
if client_version is not None:
return Response(
content=codex_model_list_body(
admin_listing, admin_entries, llm_router, team_id or user_api_key_dict.team_id
),
media_type="application/json",
)
if wants_anthropic_format:
admin_listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above
return create_anthropic_model_list_response(
admin_listing,
display_names=configured_display_names(admin_entries, llm_router),
@ -11805,8 +11820,14 @@ async def model_list(
model_info["id"] = response_id
model_data.append(model_info)
listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above
if client_version is not None:
return Response(
content=codex_model_list_body(listing, entries, llm_router, team_id or user_api_key_dict.team_id),
media_type="application/json",
)
if wants_anthropic_format:
listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above
return create_anthropic_model_list_response(
listing,
display_names=configured_display_names(entries, llm_router),

View file

@ -7661,8 +7661,9 @@ async def _run_spend_logs_job(
)
# Tool usage tracking: drain the request-time queue into the tool index and the
# LiteLLM_DailyToolSpend rollup. Never retried; a dropped batch is permanently
# absent from the rollup, so failures log at error.
# LiteLLM_DailyToolSpend rollup. A batch Postgres had no connection for was never
# sent, so it is requeued and the job stops; any other failure is dropped because
# a replay could double-count the rollup.
async with prisma_client._tool_usage_transactions_lock:
tool_usage_to_process: Final = prisma_client.tool_usage_transactions[:MAX_LOGS_PER_INTERVAL]
prisma_client.tool_usage_transactions = prisma_client.tool_usage_transactions[len(tool_usage_to_process) :]
@ -7674,11 +7675,25 @@ async def _run_spend_logs_job(
transactions=tool_usage_to_process,
)
except Exception as tool_tracking_err:
verbose_proxy_logger.error(
"Spend tracking - tool usage flush failed; %s tool usage transactions dropped: %s",
len(tool_usage_to_process),
tool_tracking_err,
)
if PrismaDBExceptionHandler.is_database_capacity_error(tool_tracking_err):
async with prisma_client._tool_usage_transactions_lock:
prisma_client.tool_usage_transactions = [
*tool_usage_to_process,
*prisma_client.tool_usage_transactions,
]
verbose_proxy_logger.warning(
"Spend tracking - database out of connections during tool usage flush, "
"requeued %s tool usage transactions for the next flush: %s",
len(tool_usage_to_process),
tool_tracking_err,
)
raise
else:
verbose_proxy_logger.error(
"Spend tracking - tool usage flush failed; %s tool usage transactions dropped: %s",
len(tool_usage_to_process),
tool_tracking_err,
)
async with prisma_client._model_usage_transactions_lock:
model_usage_to_process: Final = prisma_client.model_usage_transactions

View file

@ -479,6 +479,7 @@ def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream])
_NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType({})
_SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str])
_ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
_DEPLOYMENT_SELECTED_EVENT: Final = "litellm.request.deployment_selected"
@ -503,6 +504,10 @@ def _deployment_pick_attributes(model: str, request_kwargs: Mapping[str, object]
)
def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, object]:
return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {})
def _as_retry_skipped_deployment_ids(value: object) -> tuple[str, ...]:
return tuple(item for item in value if isinstance(item, str)) if isinstance(value, tuple) else ()
@ -10650,6 +10655,63 @@ class Router:
return display_name
return None
def routable_model_group(self, model_name: str) -> str:
"""
The model group a request to model_name routes to: its target when
model_name is a `model_group_alias`, else model_name itself.
"""
target: Final = self._get_model_from_alias(model_name)
return target if target is not None else model_name
def _routable_deployments(self, model_name: str, team_id: str | None) -> tuple[DeploymentTypedDict, ...]:
"""
The deployments a request from team_id to model_name can route to, in
model_list order and via the same O(1) index lookup as
get_configured_display_name, selected the way routing selects them: a
`model_group_alias` reads its target's deployments, another team's
deployment of the name is left out, a caller with no team reads the
deployments no team owns (every deployment of the name when a team owns
each one, which is what an admin's request routes to), and one an admin
paused via `LiteLLM_ProxyModelTable.blocked` is left out.
Returns an empty tuple for wildcard-expanded or unknown names.
"""
named: Final = self._get_all_deployments(model_name=self.routable_model_group(model_name), team_id=team_id)
usable: Final = tuple(
deployment for deployment in named if self._deployment_usable_by_team(deployment, team_id)
)
return tuple(
deployment
for deployment in (usable or named)
if _configured_model_info(deployment).get("blocked") is not True
)
def get_configured_service_tiers(self, model_name: str, team_id: str | None = None) -> tuple[object, ...]:
"""
Return the service_tiers value each deployment a request from team_id
to model_name can route to (see _routable_deployments) configures in
its model_info, unvalidated and in model_list order, None for a
deployment that sets none; the caller validates the shape it expects
and decides how the deployments combine.
Returns an empty tuple for wildcard-expanded or unknown names.
"""
return tuple(
_configured_model_info(deployment).get("service_tiers")
for deployment in self._routable_deployments(model_name, team_id)
)
def get_routable_upstream_model(self, model_name: str, team_id: str | None = None) -> str | None:
"""
Return the `litellm_params.model` of the first deployment a request
from team_id to model_name can route to (see _routable_deployments).
Returns None for wildcard-expanded or unknown names, and when the team
can route to no deployment of the name.
"""
deployment: Final = next(iter(self._routable_deployments(model_name, team_id)), None)
return deployment["litellm_params"].get("model") if deployment is not None else None
def get_credential_deployment(self, model_id: str, team_id: str | None = None) -> Deployment | None:
"""
The deployment a passthrough endpoint (files, batches, etc.) resolves for a

View file

@ -4897,7 +4897,7 @@ def get_optional_params(
drop_params=bool(drop_params),
)
elif custom_llm_provider == "bedrock_mantle":
optional_params = litellm.BedrockMantleChatConfig().map_openai_params(
optional_params = ProviderConfigManager._get_bedrock_mantle_config(model).map_openai_params(
non_default_params=non_default_params,
optional_params=optional_params,
model=model,
@ -5669,7 +5669,7 @@ def _get_model_cost_key(potential_key: str) -> str | None:
return None
def _get_model_info_from_model_cost(key: str) -> dict:
def _get_model_info_from_model_cost(key: str) -> dict[str, Any]:
return litellm.model_cost[key]
@ -5724,12 +5724,26 @@ from typing_extensions import ReadOnly, TypedDict
class PotentialModelNamesAndCustomLLMProvider(TypedDict):
split_model: str
combined_model_name: str
region_free_combined_model_name: ReadOnly[str]
stripped_model_name: str
combined_stripped_model_name: str
provider_prefixed_model_name: ReadOnly[str]
custom_llm_provider: str
def _first_registered_match(
candidates: Sequence[str], custom_llm_provider: str | None
) -> tuple[str | None, dict[str, Any] | None]:
registered_keys: Final = (key for key in map(_get_model_cost_key, candidates) if key is not None)
entries: Final = ((key, _get_model_info_from_model_cost(key=key)) for key in registered_keys)
matches: Final = (
(key, info)
for key, info in entries
if _check_provider_match(model_info=info, custom_llm_provider=custom_llm_provider)
)
return next(matches, (None, None))
def _get_model_info_from_generalization(
model: str,
potential_model_names: PotentialModelNamesAndCustomLLMProvider,
@ -5751,6 +5765,7 @@ def _get_model_info_from_generalization(
candidates: Final = (
potential_model_names["combined_model_name"],
model,
potential_model_names["region_free_combined_model_name"],
potential_model_names["split_model"],
potential_model_names["combined_stripped_model_name"],
potential_model_names["stripped_model_name"],
@ -5828,6 +5843,11 @@ def _get_potential_model_names(model: str, custom_llm_provider: str | None) -> P
return PotentialModelNamesAndCustomLLMProvider(
split_model=region_free_split_model,
combined_model_name=combined_model_name,
region_free_combined_model_name=(
f"bedrock_mantle/{region_free_split_model}"
if custom_llm_provider == "bedrock_mantle"
else combined_model_name
),
stripped_model_name=stripped_model_name,
combined_stripped_model_name=region_free_combined_stripped_model_name,
provider_prefixed_model_name=provider_cost_key or provider_prefixed_model_name,
@ -6007,78 +6027,29 @@ def _get_model_info_helper(
Check if: (in order of specificity)
1. 'custom_llm_provider/model' in litellm.model_cost. Checks "groq/llama3-8b-8192" if model="llama3-8b-8192" and custom_llm_provider="groq"
2. 'model' in litellm.model_cost. Checks "gemini-1.5-pro-002" in litellm.model_cost if model="gemini-1.5-pro-002" and custom_llm_provider=None
3. 'split_model' in litellm.model_cost. Checks "au.anthropic.claude-opus-4-8" in litellm.model_cost if model="bedrock/au.anthropic.claude-opus-4-8"
4. 'combined_stripped_model_name' in litellm.model_cost. Checks if 'gemini/gemini-1.5-flash' in model map, if 'gemini/gemini-1.5-flash-001' given.
5. 'stripped_model_name' in litellm.model_cost. Checks if 'ft:gpt-3.5-turbo' in model map, if 'ft:gpt-3.5-turbo:my-org:custom_suffix:id' given.
6. 'provider_prefixed_model_name' in litellm.model_cost, for providers whose own model ids repeat the
3. 'region_free_combined_model_name' in litellm.model_cost. Checks "bedrock_mantle/anthropic.claude-opus-5-5" if
model="bedrock_mantle/us-east-1/anthropic.claude-opus-5-5", before 4 reaches the bare Bedrock row. Same as 1 for every other provider.
4. 'split_model' in litellm.model_cost. Checks "au.anthropic.claude-opus-4-8" in litellm.model_cost if model="bedrock/au.anthropic.claude-opus-4-8"
5. 'combined_stripped_model_name' in litellm.model_cost. Checks if 'gemini/gemini-1.5-flash' in model map, if 'gemini/gemini-1.5-flash-001' given.
6. 'stripped_model_name' in litellm.model_cost. Checks if 'ft:gpt-3.5-turbo' in model map, if 'ft:gpt-3.5-turbo:my-org:custom_suffix:id' given.
7. 'provider_prefixed_model_name' in litellm.model_cost, for providers whose own model ids repeat the
litellm provider name. Checks "perplexity/perplexity/glm-5.2" if model="perplexity/glm-5.2" and
custom_llm_provider="perplexity", where 1-5 all read the leading "perplexity/" as the litellm prefix
and strip it. Tried last so no model that already resolves through 1-5 can change.
custom_llm_provider="perplexity", where 1-6 all read the leading "perplexity/" as the litellm prefix
and strip it. Tried last so no model that already resolves through 1-6 can change.
"""
_model_info: dict[str, Any] | None = None
key: str | None = None
# Use case-insensitive lookup for all model name checks
_matched_key = _get_model_cost_key(combined_model_name)
if _matched_key is not None:
key = _matched_key
_model_info = _get_model_info_from_model_cost(key=cast(str, key))
if not _check_provider_match(
model_info=_model_info,
custom_llm_provider=model_cost_custom_llm_provider,
):
_model_info = None
if _model_info is None:
_matched_key = _get_model_cost_key(model)
if _matched_key is not None:
key = _matched_key
_model_info = _get_model_info_from_model_cost(key=cast(str, key))
if not _check_provider_match(
model_info=_model_info,
custom_llm_provider=model_cost_custom_llm_provider,
):
_model_info = None
if _model_info is None:
_matched_key = _get_model_cost_key(split_model)
if _matched_key is not None:
key = _matched_key
_model_info = _get_model_info_from_model_cost(key=cast(str, key))
if not _check_provider_match(
model_info=_model_info,
custom_llm_provider=model_cost_custom_llm_provider,
):
_model_info = None
if _model_info is None:
_matched_key = _get_model_cost_key(combined_stripped_model_name)
if _matched_key is not None:
key = _matched_key
_model_info = _get_model_info_from_model_cost(key=cast(str, key))
if not _check_provider_match(
model_info=_model_info,
custom_llm_provider=model_cost_custom_llm_provider,
):
_model_info = None
if _model_info is None:
_matched_key = _get_model_cost_key(stripped_model_name)
if _matched_key is not None:
key = _matched_key
_model_info = _get_model_info_from_model_cost(key=cast(str, key))
if not _check_provider_match(
model_info=_model_info,
custom_llm_provider=model_cost_custom_llm_provider,
):
_model_info = None
if _model_info is None:
_matched_key = _get_model_cost_key(provider_prefixed_model_name)
if _matched_key is not None:
key = _matched_key
_model_info = _get_model_info_from_model_cost(key=cast(str, key))
if not _check_provider_match(
model_info=_model_info,
custom_llm_provider=model_cost_custom_llm_provider,
):
_model_info = None
lookup_order: Final = (
combined_model_name,
model,
potential_model_names["region_free_combined_model_name"],
split_model,
combined_stripped_model_name,
stripped_model_name,
provider_prefixed_model_name,
)
lookup: Final = _first_registered_match(lookup_order, model_cost_custom_llm_provider)
key: str | None = lookup[0]
_model_info: dict[str, Any] | None = lookup[1]
if _model_info is not None and key is not None and _model_info.get("mode", "chat") in _BACKFILL_MODES:
fill_missing: Final = match_fill_missing_generalizations(key, _model_info.get("litellm_provider", ""))
@ -8478,10 +8449,7 @@ class ProviderConfigManager:
LlmProviders.DEEPSEEK: (lambda: litellm.DeepSeekChatConfig(), False),
LlmProviders.TENCENT: (lambda: litellm.TencentChatConfig(), False),
LlmProviders.GROQ: (lambda: litellm.GroqChatConfig(), False),
LlmProviders.BEDROCK_MANTLE: (
lambda: litellm.BedrockMantleChatConfig(),
False,
),
LlmProviders.BEDROCK_MANTLE: (ProviderConfigManager._get_bedrock_mantle_config, True),
LlmProviders.A2A: (lambda: litellm.A2AConfig(), False),
LlmProviders.BYTEZ: (lambda: litellm.BytezChatConfig(), False),
LlmProviders.DATABRICKS: (lambda: litellm.DatabricksConfig(), False),
@ -8669,6 +8637,12 @@ class ProviderConfigManager:
return get_bedrock_chat_config(model=model)
@staticmethod
def _get_bedrock_mantle_config(model: str) -> BaseConfig:
from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantle_chat_config
return bedrock_mantle_chat_config(model)
@staticmethod
def _get_cohere_config(model: str) -> BaseConfig:
"""Get Cohere config based on route."""

View file

@ -16085,6 +16085,28 @@
"input_cost_per_image_token": 4.7e-07,
"source": "https://cohere.com/pricing"
},
"cohere/embed-v5.0-pro": {
"input_cost_per_token": 1.2e-07,
"litellm_provider": "cohere",
"max_input_tokens": 128000,
"max_tokens": 128000,
"mode": "embedding",
"output_cost_per_token": 0.0,
"output_vector_size": 2048,
"supports_embedding_image_input": true,
"source": "https://cohere.com/pricing"
},
"cohere/embed-v5.0-fast": {
"input_cost_per_token": 8e-08,
"litellm_provider": "cohere",
"max_input_tokens": 128000,
"max_tokens": 128000,
"mode": "embedding",
"output_cost_per_token": 0.0,
"output_vector_size": 2048,
"supports_embedding_image_input": true,
"source": "https://cohere.com/pricing"
},
"cohere/parse-v5.0": {
"litellm_provider": "cohere",
"mode": "ocr",
@ -61056,6 +61078,9 @@
"input_cost_per_audio_token": 2e-06,
"input_cost_per_token": 2e-06,
"litellm_provider": "gemini",
"max_input_tokens": 98304,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "audio_transcription",
"output_cost_per_token": 1.2e-05,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
@ -61078,6 +61103,9 @@
"input_cost_per_audio_token": 3.5e-06,
"input_cost_per_token": 3.5e-06,
"litellm_provider": "gemini",
"max_input_tokens": 131072,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "audio_transcription",
"output_cost_per_token": 2.1e-05,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
@ -64092,6 +64120,23 @@
"video"
]
},
"xai/grok-imagine-video-1.5-lite": {
"input_cost_per_image": 0.01,
"litellm_provider": "xai",
"mode": "video_generation",
"output_cost_per_second": 0.02,
"output_cost_per_second_1080p": 0.14,
"output_cost_per_second_480p": 0.02,
"output_cost_per_second_720p": 0.03,
"source": "https://docs.x.ai/developers/pricing",
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"video"
]
},
"xai/grok-imagine-video-1.5-2026-05-30": {
"input_cost_per_image": 0.01,
"litellm_provider": "xai",
@ -75511,6 +75556,42 @@
"supports_vision": true,
"supports_web_search": true
},
"openrouter/openai/gpt-image-2": {
"input_cost_per_image_token": 8e-06,
"input_cost_per_token": 5e-06,
"litellm_provider": "openrouter",
"mode": "image_generation",
"output_cost_per_image_token": 3e-05,
"source": "https://openrouter.ai/api/v1/images/models/openai/gpt-image-2/endpoints",
"supported_endpoints": [
"/v1/images/generations"
],
"supports_vision": true
},
"openrouter/openai/gpt-image-2.5-flare": {
"input_cost_per_image_token": 8e-06,
"input_cost_per_token": 5e-06,
"litellm_provider": "openrouter",
"mode": "image_generation",
"output_cost_per_image_token": 3e-05,
"source": "https://openrouter.ai/api/v1/images/models/openai/gpt-image-2.5-flare/endpoints",
"supported_endpoints": [
"/v1/images/generations"
],
"supports_vision": true
},
"openrouter/openai/gpt-image-2.5-sunburst": {
"input_cost_per_image_token": 8e-06,
"input_cost_per_token": 5e-06,
"litellm_provider": "openrouter",
"mode": "image_generation",
"output_cost_per_image_token": 3e-05,
"source": "https://openrouter.ai/api/v1/images/models/openai/gpt-image-2.5-sunburst/endpoints",
"supported_endpoints": [
"/v1/images/generations"
],
"supports_vision": true
},
"openrouter/openai/gpt-5.4-mini:batch": {
"cache_read_input_token_cost": 3.75e-08,
"input_cost_per_token": 3.75e-07,

View file

@ -254,6 +254,7 @@ proxy-dev = [
"a2a-sdk==1.1.0",
]
ci = [
"psutil==7.2.2",
# These are lazily imported at call sites; keep them out of core deps to
# avoid bloating the base SDK install (google-generativeai pulls grpcio +
# protobuf, Pillow is a compiled C extension).
@ -314,7 +315,8 @@ include = [
"litellm/router_strategy/complexity_router/artifacts/*.json",
"litellm/router_strategy/complexity_router/fuse_presets.json",
"litellm/proxy/model_insights_tasks.json",
"litellm/proxy/client/cli/commands/codex_base_instructions.md",
"litellm/proxy/common_utils/codex_base_instructions.md",
"litellm/proxy/common_utils/codex_bundled_models_0.159.3.json",
"litellm/proxy/lens/prompts/*.md",
]
exclude = [

643
scripts/quickstart.ps1 Normal file
View file

@ -0,0 +1,643 @@
# LiteLLM Gateway quickstart for Windows: the gateway, Postgres, and the admin UI in one command.
# powershell -c "irm https://raw.githubusercontent.com/BerriAI/litellm/main/scripts/quickstart.ps1 | iex"
# On macOS and Linux, scripts/quickstart.sh does the same.
#
# To read it before running it:
# irm https://raw.githubusercontent.com/BerriAI/litellm/main/scripts/quickstart.ps1 -OutFile quickstart.ps1
# notepad quickstart.ps1
# powershell -ExecutionPolicy Bypass -File quickstart.ps1
#
# Asks where to keep the files, then offers to copy the admin password and
# open the admin UI; every question has a default you accept by pressing Enter.
# It asks nothing when input is not a console, under CI or Claude Code, or
# with -Yes.
#
# -Yes, or LITELLM_YES=1 no questions: install to ~\litellm-gateway, don't open a browser
# LITELLM_DIR folder to install into (skips the folder question)
# LITELLM_PORT port for the gateway (default 4000, or the next free one)
# NO_COLOR plain output, which agents, CI, and log files always get
#
# New installs listen on this machine only (127.0.0.1). To reach the gateway
# from other machines, remove LITELLM_BIND from .env and put it behind TLS.
#
# Keys and the database password are random, written only to .env, which only
# your Windows account can read, and never printed. When you ask, the admin
# password goes straight to your clipboard. Needs Docker Desktop, Podman, or
# Rancher Desktop.
# Works in Windows PowerShell 5.1 and PowerShell 7. Everything runs inside
# Invoke-LiteLLMQuickstart, so a partial download runs nothing.
param([switch]$Yes)
function Invoke-LiteLLMQuickstart {
param([switch]$Yes)
$ErrorActionPreference = 'Stop'
# Windows PowerShell draws a progress bar for every web request, which makes them crawl.
$ProgressPreference = 'SilentlyContinue'
Set-StrictMode -Version 2
$composeUrl = 'https://raw.githubusercontent.com/BerriAI/litellm/main/docker/docker-compose.quickstart.yml'
if ($env:LITELLM_COMPOSE_URL) { $composeUrl = $env:LITELLM_COMPOSE_URL }
$onWindows = ($PSVersionTable.PSEdition -eq 'Desktop') -or ((Test-Path variable:IsWindows) -and $IsWindows)
# ------------------------------------------------------------ output
# A person at a console gets colors, step marks, and a spinner. Agents, CI,
# log files, and NO_COLOR get the same lines as plain text.
$style = (-not [Console]::IsOutputRedirected) -and (-not $env:NO_COLOR) -and (-not $env:CI) -and
(-not $env:CLAUDECODE) -and ($env:TERM -ne 'dumb')
# Escape sequences give the brand blue and bold; older consoles get the
# sixteen console colors instead.
$vt = $style -and $Host.UI.PSObject.Properties['SupportsVirtualTerminal'] -and $Host.UI.SupportsVirtualTerminal
# Symbols need a console font that has them: Windows Terminal, VS Code, or
# any macOS or Linux terminal. The classic console gets plain ASCII.
$unicode = $style -and ((-not $onWindows) -or $env:WT_SESSION -or ($env:TERM_PROGRAM -eq 'vscode'))
if ($unicode) {
$sym = @{ Ok = [char]0x2713; Warn = '!'; Err = [char]0x2717; Head = [char]0x25C6; Ask = '?'; Pointer = [char]0x276F
Frames = @([char]0x280B, [char]0x2819, [char]0x2839, [char]0x2838, [char]0x283C, [char]0x2834, [char]0x2826, [char]0x2827, [char]0x2807, [char]0x280F)
Hint = "$([char]0x2191)/$([char]0x2193) to move, Enter to choose"; Dot = [char]0x00B7
TL = [char]0x256D; TR = [char]0x256E; BL = [char]0x2570; BR = [char]0x256F; H = [char]0x2500; V = [char]0x2502 }
} else {
$sym = @{ Ok = '+'; Warn = '!'; Err = 'x'; Head = '*'; Ask = '?'; Pointer = '>'; Frames = @('|', '/', '-', '\')
Hint = 'Up/Down to move, Enter to choose'; Dot = '-'; TL = '+'; TR = '+'; BL = '+'; BR = '+'; H = '-'; V = '|' }
}
$e = [char]27
$ansi = @{ a = "$e[38;2;91;108;255m"; b = "$e[1m"; d = "$e[90m"; g = "$e[32m"; y = "$e[33m"; r = "$e[1;31m"; u = "$e[1;38;2;91;108;255m"; off = "$e[0m" }
$colors = @{ a = 'Blue'; b = $null; d = 'DarkGray'; g = 'Green'; y = 'Yellow'; r = 'Red'; u = 'Blue' }
# Say "text with {a:accent} {b:bold} {d:dim} {g:green} {y:yellow} {r:red} {u:link} spans"
function Say {
param([string]$Text, [switch]$Err, [switch]$NoNewline)
$parts = [regex]::Split($Text, '(\{[abdgyru]:[^}]*\})')
# Errors go out plain when stderr is redirected, so a log file gets no escape codes.
if (-not $style -or ($Err -and [Console]::IsErrorRedirected)) {
$plain = ($parts | ForEach-Object { if ($_ -match '^\{[abdgyru]:(.*)\}$') { $Matches[1] } else { $_ } }) -join ''
$stream = if ($Err) { [Console]::Error } else { [Console]::Out }
if ($NoNewline) { $stream.Write($plain) } else { $stream.WriteLine($plain) }
return
}
if ($vt) {
$line = ($parts | ForEach-Object { if ($_ -match '^\{([abdgyru]):(.*)\}$') { $ansi[$Matches[1]] + $Matches[2] + $ansi.off } else { $_ } }) -join ''
Write-Host $line -NoNewline:$NoNewline
return
}
foreach ($p in $parts) {
if ($p -match '^\{([abdgyru]):(.*)\}$') {
$c = $colors[$Matches[1]]
if ($c) { Write-Host $Matches[2] -NoNewline -ForegroundColor $c } else { Write-Host $Matches[2] -NoNewline }
} elseif ($p) { Write-Host $p -NoNewline }
}
if (-not $NoNewline) { Write-Host '' }
}
# Markup characters in a value (a path, a URL) must not open a span.
function Lit([string]$s) { return $s.Replace('{', '(').Replace('}', ')') }
function Step([string]$Text) { if ($style) { Say ("{g:$($sym.Ok)} " + $Text) } else { Say $Text } }
function Warn([string]$Text) { if ($style) { Say ("{y:$($sym.Warn)} " + $Text) } else { Say $Text } }
function Fail([string]$Head, [string]$Rest = '') {
$tail = if ($Rest) { " $Rest" } else { '' }
if ($style) { Say ("{r:$($sym.Err) $Head}" + $tail) -Err } else { Say ($Head + $tail) -Err }
}
function Tildify([string]$Path) {
if ($Path -eq $HOME) { return '~' }
if ($Path.StartsWith($HOME + [IO.Path]::DirectorySeparatorChar, [StringComparison]::OrdinalIgnoreCase)) { return '~' + $Path.Substring($HOME.Length) }
return $Path
}
function Elapsed([datetime]$Since) {
$s = [int]((Get-Date) - $Since).TotalSeconds
if ($s -lt 60) { return "${s}s" } else { return "$([math]::Floor($s / 60))m $($s % 60)s" }
}
function Clear-Line {
if ($vt) { Write-Host "`r$e[2K" -NoNewline } else { Write-Host ("`r" + (' ' * ([Console]::WindowWidth - 1)) + "`r") -NoNewline }
}
function Show-Cursor([bool]$Shown) { try { [Console]::CursorVisible = $Shown } catch { <# a host without a cursor #> } }
# Run a native program; returns its exit code and stdout lines. stderr is
# dropped so Windows PowerShell does not turn it into a terminating error.
function Invoke-Native {
param([string]$File, [string[]]$Arguments)
$prev = $ErrorActionPreference
$ErrorActionPreference = 'Continue'
try {
$out = & $File @Arguments 2>$null
$code = $LASTEXITCODE
} catch {
$out = @(); $code = 1
} finally {
$ErrorActionPreference = $prev
}
return [pscustomobject]@{ Code = $code; Out = @($out) }
}
# Write a text file as UTF-8 without a byte order mark, with LF line endings,
# which is what Docker Compose and git expect.
function Write-PlainText([string]$Path, [string]$Text, [switch]$Append) {
$enc = New-Object System.Text.UTF8Encoding $false
if ($Append) { [IO.File]::AppendAllText($Path, $Text, $enc) } else { [IO.File]::WriteAllText($Path, $Text, $enc) }
}
# The POSIX cksum of a string's UTF-8 bytes: CRC-32 over the bytes and then
# their length. quickstart.sh names projects with the same number, so both
# scripts find the same database for a folder. Int64 keeps the arithmetic
# unsigned; hex literals with the top bit set would be negative here.
function Get-Cksum([string]$Text) {
$bytes = New-Object System.Collections.Generic.List[byte]
$bytes.AddRange([Text.Encoding]::UTF8.GetBytes($Text))
for ($n = [int64]$bytes.Count; $n -gt 0; $n = $n -shr 8) { $bytes.Add([byte]($n -band 255)) }
[int64]$crc = 0
foreach ($b in $bytes) {
$crc = $crc -bxor ([int64]$b -shl 24)
for ($i = 0; $i -lt 8; $i++) {
$crc = if ($crc -band 2147483648) { (($crc -shl 1) -bxor 79764919) -band 4294967295 } else { ($crc -shl 1) -band 4294967295 }
}
}
return (-bnot $crc) -band 4294967295
}
# ------------------------------------------------------------ questions
$interactive = (-not $Yes) -and (-not $env:LITELLM_YES) -and (-not $env:CI) -and (-not $env:CLAUDECODE) -and
(-not [Console]::IsInputRedirected) -and [Environment]::UserInteractive
# Menu "Question" Default @(@('label', 'note'), ...) -> the 1-based pick.
# ReadKey holds on to Ctrl+C, so take it as a key and stop the way PowerShell
# does; finally blocks still run and put the terminal back.
function Read-Key {
$saved = [Console]::TreatControlCAsInput
try {
[Console]::TreatControlCAsInput = $true
$key = [Console]::ReadKey($true)
} finally {
[Console]::TreatControlCAsInput = $saved
}
if ($key.Key -eq 'C' -and ($key.Modifiers -band [ConsoleModifiers]::Control)) {
throw [System.Management.Automation.PipelineStoppedException]::new()
}
return $key
}
function Menu {
param([string]$Question, [int]$Default, [object[]]$Options)
if (-not $interactive) { return $Default }
$count = $Options.Count
$pad = ($Options | ForEach-Object { $_[0].Length } | Measure-Object -Maximum).Maximum
Write-Host ''
if ($style) { Say ("{a:$($sym.Ask)} {b:" + (Lit $Question) + '}') } else { Write-Host $Question }
$choice = $Default
try {
Show-Cursor $false
$first = $true
$top = 0
while ($true) {
# Escape-code consoles move up relative to the cursor; the classic
# Windows console jumps back to the row the menu started on.
if (-not $first) { if ($vt) { Write-Host "$e[$($count + 1)A" -NoNewline } else { [Console]::SetCursorPosition(0, $top) } }
$first = $false
for ($i = 1; $i -le $count; $i++) {
$label = (Lit $Options[$i - 1][0]).PadRight($pad)
$note = if ($Options[$i - 1].Count -gt 1) { Lit $Options[$i - 1][1] } else { '' }
Clear-Line
if ($i -eq $choice) { Say " {u:$($sym.Pointer) $label} {d:$note}" } else { Say " $label {d:$note}" }
}
Clear-Line
Say " {d:$($sym.Hint)}"
if (-not $vt) { $top = [Console]::CursorTop - ($count + 1) }
$key = Read-Key
if ($key.Key -eq 'UpArrow' -or $key.KeyChar -eq 'k') { if ($choice -gt 1) { $choice-- } }
elseif ($key.Key -eq 'DownArrow' -or $key.KeyChar -eq 'j') { if ($choice -lt $count) { $choice++ } }
elseif ($key.KeyChar -match '^[1-9]$' -and [int]"$($key.KeyChar)" -le $count) { $choice = [int]"$($key.KeyChar)" }
elseif ($key.Key -eq 'Enter') { break }
}
if ($vt) { Write-Host "$e[1A" -NoNewline } else { [Console]::SetCursorPosition(0, $top + $count) }
Clear-Line
} catch [System.Management.Automation.PipelineStoppedException] {
throw
} catch {
# No cursor control here: fall back to a numbered list.
for ($i = 1; $i -le $count; $i++) { Write-Host (" $i) " + $Options[$i - 1][0]) }
$answer = Read-Host "Choose [$Default]"
if ($answer -match '^[1-9]$' -and [int]$answer -le $count) { $choice = [int]$answer }
} finally {
Show-Cursor $true
}
return $choice
}
# ------------------------------------------------------------ steps
function Test-PortFree([int]$Port) {
# Binding answers right away and is accurate. Probing with a connect is
# not: the WSL2 localhost relay (Podman, Rancher Desktop) swallows the
# refusal on closed ports, so every connect waits out its timeout and
# closed ports look taken.
$listener = New-Object System.Net.Sockets.TcpListener([Net.IPAddress]::Loopback, $Port)
try {
$listener.Start()
return $true
} catch {
return $false
} finally {
$listener.Stop()
}
}
function Test-Ready([int]$Port) {
try {
$r = Invoke-WebRequest -UseBasicParsing -TimeoutSec 2 -Uri "http://127.0.0.1:$Port/health/readiness"
return $r.StatusCode -eq 200
} catch { return $false }
}
# Run a program with a spinner and the elapsed time, keeping its output to
# show if it fails. Without styling it runs in the open.
function Invoke-WithSpinner {
param([string]$Label, [string]$Detail, [string]$File, [string[]]$Arguments)
if (-not $style) {
$prev = $ErrorActionPreference
$ErrorActionPreference = 'Continue'
try { & $File @Arguments 2>&1 | ForEach-Object { [Console]::Out.WriteLine("$_") }; $code = $LASTEXITCODE }
finally { $ErrorActionPreference = $prev }
return $code
}
$out = [IO.Path]::GetTempFileName(); $err = [IO.Path]::GetTempFileName()
try {
$p = Start-Process -FilePath $File -ArgumentList $Arguments -NoNewWindow -PassThru -RedirectStandardOutput $out -RedirectStandardError $err
$null = $p.Handle # keeps ExitCode available after exit in Windows PowerShell
$start = Get-Date
$n = 0
Show-Cursor $false
try {
while (-not $p.HasExited) {
$s = [int]((Get-Date) - $start).TotalSeconds
$clock = '{0}:{1:00}' -f [math]::Floor($s / 60), ($s % 60)
$extra = if ($Detail) { "$Detail $($sym.Dot) " } else { '' }
Clear-Line
Say ("{a:$($sym.Frames[$n % $sym.Frames.Count])} $Label {d:$($sym.Dot) $extra$clock}") -NoNewline
$n++
Start-Sleep -Milliseconds 100
}
$p.WaitForExit()
Clear-Line
} finally { Show-Cursor $true }
$code = $p.ExitCode
if ($code -ne 0) {
Get-Content $out, $err -ErrorAction SilentlyContinue | ForEach-Object { [Console]::Error.WriteLine($_) }
}
return $code
} finally {
# Also on Ctrl+C, so a cancelled run leaves no temporary files.
Remove-Item -LiteralPath $out, $err -Force -ErrorAction SilentlyContinue
}
}
function Wait-Ready([int]$Port) {
# Up to 3 minutes, like the shell version. A spinner when styled.
$start = Get-Date
$n = 0
$nextCheck = Get-Date
if ($style) { Show-Cursor $false }
try {
while (((Get-Date) - $start).TotalSeconds -lt 180) {
if ((Get-Date) -ge $nextCheck) {
if (Test-Ready $Port) { if ($style) { Clear-Line }; return $true }
$nextCheck = (Get-Date).AddSeconds(2)
}
if ($style) {
$s = [int]((Get-Date) - $start).TotalSeconds
Clear-Line
Say ("{a:$($sym.Frames[$n % $sym.Frames.Count])} Waiting for the gateway to be ready {d:$($sym.Dot) $('{0}:{1:00}' -f [math]::Floor($s / 60), ($s % 60))}") -NoNewline
$n++
}
Start-Sleep -Milliseconds 100
}
if ($style) { Clear-Line }
return $false
} finally { if ($style) { Show-Cursor $true } }
}
# The closing summary. Plain output, and a console too narrow for the frame,
# get the same lines without it.
function Open-Url([string]$Url) {
try {
if ($onWindows) { Start-Process $Url } elseif (Get-Command open -ErrorAction SilentlyContinue) { & open $Url } else { & xdg-open $Url }
Step "Opened {a:$Url} in your browser"
} catch {
# No browser to open; print the address instead.
Say " {d:Open} {a:$Url} {d:when you are ready.}"
}
}
# The admin password is the master key. It goes straight to the clipboard,
# never to the screen.
function Copy-Password([string]$Folder) {
$pw = Get-Content -LiteralPath (Join-Path $Folder '.env') |
Where-Object { $_.StartsWith('LITELLM_MASTER_KEY=') } | Select-Object -First 1
if (-not $pw) { Warn ('No LITELLM_MASTER_KEY in ' + (Lit (Join-Path (Tildify $Folder) '.env')) + '.'); return }
try {
Set-Clipboard -Value $pw.Substring('LITELLM_MASTER_KEY='.Length) -ErrorAction Stop
Step 'Copied the admin password. Paste it into the sign-in page.'
} catch {
# Another app can hold the Windows clipboard open.
Warn ('Could not copy it. The password is LITELLM_MASTER_KEY in ' + (Lit (Join-Path (Tildify $Folder) '.env')) + '.')
} finally {
$pw = $null
}
}
function Show-Box([string]$Title, [object[]]$Rows) {
$width = $Title.Length
foreach ($r in $Rows) { if (11 + $r[1].Length -gt $width) { $width = 11 + $r[1].Length } }
$cols = 0
try { $cols = [Console]::WindowWidth } catch { <# no console: print without the frame #> }
if (-not $style -or $cols -lt $width + 4) {
if ($style) { Say "{u:$Title}" } else { Say "$Title." }
foreach ($r in $Rows) { Say (' ' + ($r[0] + ':').PadRight(12) + (Lit $r[1])) }
return
}
$rule = "$($sym.H)" * ($width + 2)
Say "{a:$($sym.TL)$rule$($sym.TR)}"
Say ("{a:$($sym.V)} {u:" + $Title.PadRight($width) + "} {a:$($sym.V)}")
foreach ($r in $Rows) {
$value = (Lit $r[1]).PadRight($width - 11)
if ($r[0] -eq 'Admin UI') { $value = "{u:$value}" }
Say ("{a:$($sym.V)} {d:" + $r[0].PadRight(10) + "} $value {a:$($sym.V)}")
}
Say "{a:$($sym.BL)$rule$($sym.BR)}"
}
# ------------------------------------------------------------ main
$savedEncoding = $null
try {
if ($unicode -and $onWindows) {
# Windows PowerShell writes the console in the OEM code page, which has no check marks.
$savedEncoding = [Console]::OutputEncoding
[Console]::OutputEncoding = New-Object System.Text.UTF8Encoding $false
}
if ($PSVersionTable.PSEdition -eq 'Desktop') {
# Windows PowerShell can default to TLS 1.0, which GitHub refuses.
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
}
if ($style) {
Write-Host ''
Say "{u:$($sym.Head) LiteLLM quickstart}"
Say '{d: The gateway, Postgres, and the admin UI in one command}'
Write-Host ''
} else {
Say 'LiteLLM quickstart'
}
# Podman and Rancher Desktop (nerdctl in containerd mode; its dockerd mode
# already provides a docker CLI) work the same way through Compose v2.
$engineCmd = foreach ($cand in 'docker', 'podman', 'nerdctl') {
$c = Get-Command $cand -CommandType Application -ErrorAction SilentlyContinue | Select-Object -First 1
if ($c) { $c; break }
}
if (-not $engineCmd) {
Fail 'No container engine found (docker, podman, or nerdctl).' 'The LiteLLM Gateway runs in a container alongside a Postgres database.'
$install = if ($onWindows) { 'https://docs.docker.com/desktop/setup/install/windows-install/' } else { 'https://docs.docker.com/get-docker/' }
Say '' -Err
Say " Install Docker Desktop ($install)," -Err
Say ' Podman (https://podman.io), or Rancher Desktop (https://rancherdesktop.io), then run this again.' -Err
Say ' Or deploy in one click (Railway or Render): https://docs.litellm.ai/docs/proxy/docker_quick_start' -Err
Say ' Only need to call models from Python? pip install litellm' -Err
return 1
}
$docker = $engineCmd.Source
$engine = $engineCmd.Name -replace '\.exe$', ''
$engineName = switch ($engine) { 'podman' { 'Podman' } 'nerdctl' { 'nerdctl' } default { 'Docker' } }
$compose = Invoke-Native $docker @('compose', 'version', '--short')
if ($compose.Code -ne 0) { Fail "Compose v2 ('$engine compose') is required."; return 1 }
if ((Invoke-Native $docker @('info')).Code -ne 0) {
$startHint = switch ($engine) {
'podman' { 'Start it (podman machine start) and run this again.' }
'nerdctl' { 'Start Rancher Desktop and run this again.' }
default { 'Start Docker Desktop and run this again.' }
}
Fail "$engineName is installed but not running." $startHint
return 1
}
$server = Invoke-Native $docker @('version', '--format', '{{.Server.Version}}')
$engineVersion = if ($server.Code -eq 0 -and $server.Out.Count) { "$($server.Out[0]) " } else { '' }
$composeVersion = "$($compose.Out[0])".TrimStart('v')
Step "$engineName ${engineVersion}with Compose $composeVersion is running"
$homeDir = Join-Path $HOME 'litellm-gateway'
$hereDir = Join-Path (Get-Location).ProviderPath 'litellm-gateway'
$found = $false
if ($env:LITELLM_DIR) {
$dir = $env:LITELLM_DIR
} elseif (Test-Path -LiteralPath (Join-Path $hereDir '.env')) {
$dir = $hereDir; $found = $true # installed in this folder before
} elseif (Test-Path -LiteralPath (Join-Path $homeDir '.env')) {
$dir = $homeDir; $found = $true # installed in the home folder before
} elseif ($hereDir -eq $homeDir) {
$dir = $homeDir
} else {
$pick = Menu 'Where should LiteLLM keep its files (.env with your keys, and the compose file)?' 1 @(
, @((Tildify $homeDir), 'recommended, reruns always find it')
, @((Tildify $hereDir), 'this folder'))
$dir = if ($pick -eq 2) { $hereDir } else { $homeDir }
}
$created = -not (Test-Path -LiteralPath $dir)
$null = New-Item -ItemType Directory -Force -Path $dir
$dir = (Resolve-Path -LiteralPath $dir).ProviderPath
Set-Location -LiteralPath $dir
if ($found) { Step ('Found your install in {b:' + (Lit (Tildify $dir)) + '}') } else { Step ('Files go in {b:' + (Lit (Tildify $dir)) + '}') }
$git = Get-Command git -CommandType Application -ErrorAction SilentlyContinue | Select-Object -First 1
$inRepo = $git -and (Invoke-Native $git.Source @('rev-parse', '--is-inside-work-tree')).Code -eq 0
if ($inRepo -and -not (Test-Path -LiteralPath (Join-Path $dir '.env')) -and
(Invoke-Native $git.Source @('ls-files', '--error-unmatch', '.env')).Code -eq 0) {
# An ignore rule does not cover a tracked file, so new keys written here
# would show up as a change to commit.
Fail 'Git tracks a .env file in this folder,' 'so your keys could be committed.'
Say '' -Err
Say ' Install into another folder (set LITELLM_DIR), or stop tracking the file' -Err
Say ' first with: git rm --cached .env' -Err
return 1
}
if ($created) {
# A folder this script made holds only its own files, so keep all of it out of git.
Write-PlainText (Join-Path $dir '.gitignore') "*`n"
} elseif ($inRepo -and (Invoke-Native $git.Source @('check-ignore', '-q', '.env')).Code -ne 0) {
# In a folder that already existed, such as a repository root, leave the
# tracked .gitignore alone and add only .env to this clone's local exclude
# list, so the generated keys cannot be committed.
$exclude = "$((Invoke-Native $git.Source @('rev-parse', '--git-path', 'info/exclude')).Out[0])"
$exclude = [IO.Path]::GetFullPath([IO.Path]::Combine($dir, $exclude))
$null = New-Item -ItemType Directory -Force -Path (Split-Path $exclude)
$prefix = "$((Invoke-Native $git.Source @('rev-parse', '--show-prefix')).Out | Select-Object -First 1)"
Write-PlainText $exclude "/$prefix.env`n" -Append
Step ("Kept .env out of git {d:(added it to this repository's local exclude list, " + (Lit (Tildify $exclude)) + ')}')
} elseif (-not $inRepo) {
# An existing folder outside git: ignore only .env, so it stays out of
# commits if the folder becomes a repository later.
$ignore = Join-Path $dir '.gitignore'
$current = if (Test-Path -LiteralPath $ignore) { [IO.File]::ReadAllText($ignore) } else { '' }
if (($current -split "`r?`n") -notcontains '.env') {
# Start on a new line if the file does not end with one.
$lead = if ($current.Length -gt 0 -and -not $current.EndsWith("`n")) { "`n" } else { '' }
Write-PlainText $ignore "$lead.env`n" -Append
}
}
# The engine names containers and the database volume after the project, so
# an install outside the home folder gets its own name and never shares a
# database with another litellm-gateway folder. Windows paths ignore case,
# so there the name does too.
$envFile = Join-Path $dir '.env'
$project = 'litellm-gateway'
if ($dir -ne $homeDir) {
$key = if ($onWindows) { $dir.ToLowerInvariant() } else { $dir }
$project = "litellm-gateway-$(Get-Cksum $key)"
}
if (-not (Test-Path -LiteralPath $envFile)) {
# Postgres keeps the password it was created with, so a new password over an
# old database volume would lock the gateway out. Stop and explain instead.
if ((Invoke-Native $docker @('volume', 'inspect', "${project}_postgres_data")).Code -eq 0) {
Fail 'Found a database from an earlier install' "($engineName volume ${project}_postgres_data)"
Say ('but no ' + (Lit (Join-Path (Tildify $dir) '.env')) + ' with its password.') -Err
Say '' -Err
Say ' Restore that .env file and run this again to keep your models and keys, or' -Err
Say ' delete the old database and start fresh (this removes its models and keys):' -Err
Say " $engine volume rm ${project}_postgres_data" -Err
return 1
}
}
Invoke-WebRequest -UseBasicParsing -Uri $composeUrl -OutFile (Join-Path $dir 'docker-compose.quickstart.yml')
Step 'Downloaded docker-compose.quickstart.yml'
$saved = ''
if (Test-Path -LiteralPath $envFile) {
$m = Get-Content -LiteralPath $envFile | Where-Object { $_ -match '^LITELLM_PORT=(.*)$' } | Select-Object -Last 1
if ($m) { $saved = ($m -split '=', 2)[1] }
}
if ($env:LITELLM_PORT) {
$port = [int]$env:LITELLM_PORT
} elseif ($saved) {
$port = [int]$saved
} elseif (Test-Path -LiteralPath $envFile) {
$port = 4000 # an existing install without a saved port runs on the compose default
} else {
$port = 4000
while (-not (Test-PortFree $port)) {
$port++
if ($port -gt 4099) {
Fail 'Ports 4000 to 4099 are all in use.' 'Set LITELLM_PORT to a free port and run this again.'
return 1
}
}
if ($port -eq 4000) { Step 'Port {b:4000} is free' } else { Warn "Port 4000 is in use, so LiteLLM will use {b:$port}" }
}
if (Test-Path -LiteralPath $envFile) {
Step 'Reusing .env, so existing keys and data keep working'
} else {
$rng = [Security.Cryptography.RandomNumberGenerator]::Create()
$hex = {
param([int]$n)
$bytes = New-Object byte[] $n
$rng.GetBytes($bytes)
-join ($bytes | ForEach-Object { $_.ToString('x2') })
}
# Lock a temporary file down before anything secret goes into it, then
# rename it, so a failed write never leaves a partial .env that a rerun
# would mistake for a finished install.
$tmp = "$envFile.tmp"
try {
Write-PlainText $tmp ''
if ($onWindows) {
$acl = New-Object System.Security.AccessControl.FileSecurity
$acl.SetAccessRuleProtection($true, $false)
$me = [Security.Principal.WindowsIdentity]::GetCurrent().User
$acl.AddAccessRule((New-Object System.Security.AccessControl.FileSystemAccessRule($me, 'FullControl', 'Allow')))
Set-Acl -LiteralPath $tmp -AclObject $acl
} else {
& chmod 600 $tmp
if ($LASTEXITCODE -ne 0) { throw "Could not restrict the permissions of $tmp." }
}
Write-PlainText $tmp (("LITELLM_MASTER_KEY=sk-{0}`nLITELLM_SALT_KEY=sk-{1}`nPOSTGRES_PASSWORD={2}`n" +
"LITELLM_PORT={3}`nLITELLM_BIND=127.0.0.1:`nCOMPOSE_PROJECT_NAME={4}`n") -f (& $hex 32), (& $hex 32), (& $hex 24), $port, $project)
[IO.File]::Move($tmp, $envFile)
} finally {
Remove-Item -LiteralPath $tmp -Force -ErrorAction SilentlyContinue
}
Step 'Generated .env with your master key, salt key, and database password {d:(keep this file; only you can read it)}'
}
# Compose prefers values already set in the environment over .env, so drop
# inherited ones: .env stays the only source for keys and the project name.
foreach ($name in 'LITELLM_MASTER_KEY', 'LITELLM_SALT_KEY', 'POSTGRES_PASSWORD', 'COMPOSE_PROJECT_NAME') {
Remove-Item "Env:$name" -ErrorAction SilentlyContinue
}
# The bind address follows .env when .env sets it. For an older .env without
# it, a value already in the environment is kept.
if (Get-Content -LiteralPath $envFile | Where-Object { $_ -match '^LITELLM_BIND=' }) { Remove-Item Env:LITELLM_BIND -ErrorAction SilentlyContinue }
$env:LITELLM_PORT = "$port"
$started = Get-Date
$detail = ''
foreach ($image in (Invoke-Native $docker @('compose', '-f', 'docker-compose.quickstart.yml', 'config', '--images')).Out) {
if ($image -and (Invoke-Native $docker @('image', 'inspect', "$image")).Code -ne 0) { $detail = 'downloading images, first run only' }
}
if (-not $style) { Say 'Starting LiteLLM and Postgres (the first run downloads the images)...' }
$code = Invoke-WithSpinner 'Starting LiteLLM and Postgres' $detail $docker @('compose', '-f', 'docker-compose.quickstart.yml', 'up', '-d')
if ($code -ne 0) {
Fail "$engineName could not start LiteLLM and Postgres." 'Its output is above.'
return 1
}
if (-not (Wait-Ready $port)) {
Fail 'The gateway did not become ready in 3 minutes.' 'See what it logged:'
Say (' cd ' + (Lit (Tildify $dir)) + "; $engine compose -f docker-compose.quickstart.yml logs litellm") -Err
return 1
}
Step ("LiteLLM and Postgres are up {d:$($sym.Dot) $(Elapsed $started)}")
$url = "http://localhost:$port/ui"
$where = Tildify $dir
if ($where -match ' ') { $where = "`"$where`"" }
Write-Host ''
# Over SSH the clipboard would be the server's, not yours. On Linux,
# Set-Clipboard without xclip quietly keeps the text inside PowerShell, so
# only Windows and macOS, which always have a clipboard, get the option.
$canCopy = $interactive -and (-not $env:SSH_CONNECTION) -and
($onWindows -or ((Test-Path variable:IsMacOS) -and $IsMacOS))
$password = "the LITELLM_MASTER_KEY value in $(Join-Path (Tildify $dir) '.env')"
if ($canCopy) { $password = "copy it below, or LITELLM_MASTER_KEY in $(Join-Path (Tildify $dir) '.env')" }
Show-Box 'LiteLLM is running' @(
, @('Admin UI', $url)
, @('Username', 'admin')
, @('Password', $password)
, @('Next', 'in the UI, open Models + Endpoints > Add Model and paste a provider API key')
, @('Stop it', "cd $where; $engine compose -f docker-compose.quickstart.yml down"))
if ($canCopy) {
# Come back to the menu after each choice, with the next step as the default.
$pick = 1
while ($true) {
$pick = Menu 'What next?' $pick @(
, @('Copy the admin password', 'to the clipboard, ready to paste')
, @('Open the admin UI in your browser')
, @('Done'))
if ($pick -eq 1) { Copy-Password $dir; $pick = 2 }
elseif ($pick -eq 2) { Open-Url $url; $pick = 3 }
else { break }
}
} elseif ($interactive) {
if ((Menu 'Open the admin UI in your browser?' 1 @(, @('Yes'), @('No'))) -eq 1) { Open-Url $url }
}
return 0
} finally {
if ($savedEncoding) { [Console]::OutputEncoding = $savedEncoding }
}
}
$code = Invoke-LiteLLMQuickstart -Yes:$Yes | Select-Object -Last 1
$global:LASTEXITCODE = $code
# Run from a file, or as `powershell -c "irm ... | iex"`, the exit code should
# reach the caller. Pasted into an open window, `exit` would close that window,
# so there it only sets $LASTEXITCODE.
$hostArgs = [Environment]::GetCommandLineArgs()
$oneShot = ($hostArgs -match '^-(c|command)$') -and -not ($hostArgs -match '^-noexit$')
if ($code -ne 0 -and ($PSCommandPath -or $oneShot)) { exit $code }

View file

@ -1,36 +1,183 @@
#!/bin/sh
# LiteLLM Gateway quickstart: the gateway, Postgres, and the admin UI in one command.
# curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/main/scripts/quickstart.sh | sh
# On Windows, scripts/quickstart.ps1 does the same in PowerShell.
#
# To read it before running it:
# curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/main/scripts/quickstart.sh -o quickstart.sh
# less quickstart.sh
# sh quickstart.sh
#
# Asks at most two questions (where to keep the files, and whether to open the
# admin UI), each with a default you accept by pressing Enter. It asks nothing
# when there is no terminal, under CI or Claude Code, or when run with --yes.
# Asks where to keep the files, then offers to copy the admin password and
# open the admin UI; every question has a default you accept by pressing Enter.
# It asks nothing when there is no terminal, under CI or Claude Code, or when
# run with --yes.
#
# --yes, -y no questions: install to ~/litellm-gateway, don't open a browser
# LITELLM_DIR folder to install into (skips the folder question)
# LITELLM_PORT port for the gateway (default 4000, or the next free one)
# NO_COLOR plain output, which agents, CI, and log files always get
#
# New installs listen on this machine only (127.0.0.1). To reach the gateway
# from other machines, remove LITELLM_BIND from .env and put it behind TLS.
#
# Keys and the database password are random (openssl rand), written only to
# .env with permissions 600, and never printed. Needs Docker with Compose v2.
# .env with permissions 600, and never printed. When you ask, the admin
# password goes straight to your clipboard. Needs Docker, Podman, or Rancher
# Desktop, each with Compose v2.
# Everything runs inside main(), so a partial download runs nothing.
set -eu
COMPOSE_URL="${LITELLM_COMPOSE_URL:-https://raw.githubusercontent.com/BerriAI/litellm/main/docker/docker-compose.quickstart.yml}"
# ---------------------------------------------------------------- output
# A person watching a terminal gets colors, step marks, and a spinner. Agents,
# CI, log files, and NO_COLOR get the same lines as plain text.
STYLE=0
ERR_STYLE=0 # stderr is styled only when it is a terminal too
C_ACC='' C_OK='' C_WARN='' C_ERR='' C_DIM='' C_BOLD='' C_OFF=''
POINTER='>' S_OK='+' S_WARN='!' S_ERR='x' S_HEAD='*' S_ASK='?'
# shellcheck disable=SC1003 # the last frame is a backslash
SPIN_FRAMES='| / - \'
HINT='Up/Down to move, Enter to choose'
BOX_TL='+' BOX_TR='+' BOX_BL='+' BOX_BR='+' BOX_H='-' BOX_V='|'
SPIN_PID='' SPIN_LOG='' ENV_TMP=''
setup_output() {
case "${LC_ALL:-${LC_CTYPE:-${LANG:-}}}" in
*UTF-8* | *utf-8* | *UTF8* | *utf8*)
POINTER='❯' S_OK='✓' S_WARN='!' S_ERR='✗' S_HEAD='◆' S_ASK='?'
SPIN_FRAMES='⠋ ⠙ ⠹ ⠸ ⠼ ⠴ ⠦ ⠧ ⠇ ⠏'
HINT='↑/↓ to move, Enter to choose'
BOX_TL='╭' BOX_TR='╮' BOX_BL='╰' BOX_BR='╯' BOX_H='─' BOX_V='│'
;;
esac
if [ -t 1 ] && [ -z "${NO_COLOR:-}" ] && [ "${TERM:-dumb}" != "dumb" ] &&
[ -z "${CI:-}" ] && [ -z "${CLAUDECODE:-}" ]; then
STYLE=1
e="$(printf '\033')"
# The accent is the LiteLLM blue, lightened so it reads on dark and light themes.
case "${COLORTERM:-}" in
truecolor | 24bit) C_ACC="${e}[38;2;91;108;255m" ;;
*) C_ACC="${e}[94m" ;;
esac
C_OK="${e}[32m" C_WARN="${e}[33m" C_ERR="${e}[31m" C_DIM="${e}[90m" C_BOLD="${e}[1m" C_OFF="${e}[0m"
if [ -t 2 ]; then ERR_STYLE=1; fi
fi
}
step() {
if [ "$STYLE" = 1 ]; then
printf '%s%s%s %s%s\n' "${C_OK}" "$S_OK" "${C_OFF}" "$1" "${C_OFF}"
else
printf '%s\n' "$1"
fi
}
warn() {
if [ "$STYLE" = 1 ]; then
printf '%s%s%s %s%s\n' "${C_WARN}" "$S_WARN" "${C_OFF}" "$1" "${C_OFF}"
else
printf '%s\n' "$1"
fi
}
fail() {
if [ "$ERR_STYLE" = 1 ]; then
printf '%s%s %s%s%s\n' "${C_ERR}${C_BOLD}" "$S_ERR" "$1" "${C_OFF}" "${2:+ $2}" >&2
else
printf '%s%s\n' "$1" "${2:+ $2}" >&2
fi
}
tildify() {
case "$1" in
"$HOME") printf '~' ;;
"$HOME"/*) printf '~%s' "${1#"$HOME"}" ;;
*) printf '%s' "$1" ;;
esac
}
elapsed_since() {
s=$(($(date +%s) - $1))
if [ "$s" -lt 60 ]; then printf '%ss' "$s"; else printf '%sm %ss' "$((s / 60))" "$((s % 60))"; fi
}
# spin "label" "detail" command...: run the command behind a spinner with the
# elapsed time, keeping its output to show if it fails. Without styling the
# command runs in the open, as before.
spin() {
label="$1" detail="$2"
shift 2
if [ "$STYLE" != 1 ]; then
"$@"
return
fi
SPIN_LOG="$(mktemp)"
"$@" >"$SPIN_LOG" 2>&1 &
SPIN_PID=$!
start="$(date +%s)" s=0 n=0
frames="$SPIN_FRAMES "
printf '\033[?25l'
while kill -0 "$SPIN_PID" 2>/dev/null; do
# Rotate the frames in the shell and read the clock once a second, so a
# redraw starts no process besides sleep.
frame="${frames%% *}"
frames="${frames#* }$frame "
if [ $((n % 10)) = 0 ]; then s=$(($(date +%s) - start)); fi
printf '\r\033[2K%s%s%s %s %s· %s%d:%02d%s' "${C_ACC}" "$frame" "${C_OFF}" "$label" "${C_DIM}" \
"${detail:+$detail · }" "$((s / 60))" "$((s % 60))" "${C_OFF}"
n=$((n + 1))
sleep 0.1
done
rc=0
wait "$SPIN_PID" || rc=$?
SPIN_PID=''
printf '\r\033[2K\033[?25h'
if [ "$rc" != 0 ]; then cat "$SPIN_LOG" >&2; fi
rm -f "$SPIN_LOG"
SPIN_LOG=''
return "$rc"
}
# box "Title" "Label|value"...: the closing summary. Plain output, and a
# terminal too narrow for the frame, get the same lines without it.
box() {
title="$1"
shift
cols=''
if [ "$STYLE" = 1 ]; then cols="$( (stty size </dev/tty) 2>/dev/null | cut -d ' ' -f 2)" || cols=''; fi
width=${#title}
for row in "$@"; do
value="${row#*|}"
if [ $((11 + ${#value})) -gt "$width" ]; then width=$((11 + ${#value})); fi
done
if [ "$STYLE" != 1 ] || [ -z "$cols" ] || [ $((width + 4)) -gt "$cols" ]; then
if [ "$STYLE" = 1 ]; then printf '%s%s%s\n' "${C_ACC}${C_BOLD}" "$title" "${C_OFF}"; else printf '%s.\n' "$title"; fi
for row in "$@"; do
printf ' %-12s%s\n' "${row%%|*}:" "${row#*|}"
done
return 0
fi
rule="$(printf "%$((width + 2))s" '' | sed "s/ /$BOX_H/g")"
printf '%s%s%s%s%s\n' "${C_ACC}" "$BOX_TL" "$rule" "$BOX_TR" "${C_OFF}"
printf '%s%s%s %s%-*s%s %s%s%s\n' "${C_ACC}" "$BOX_V" "${C_OFF}" "${C_ACC}${C_BOLD}" "$width" "$title" "${C_OFF}" \
"${C_ACC}" "$BOX_V" "${C_OFF}"
for row in "$@"; do
label="${row%%|*}" value="${row#*|}" shade=''
if [ "$label" = "Admin UI" ]; then shade="${C_ACC}${C_BOLD}"; fi
printf '%s%s%s %s%-10s%s %s%-*s%s %s%s%s\n' "${C_ACC}" "$BOX_V" "${C_OFF}" "${C_DIM}" "$label" "${C_OFF}" \
"$shade" "$((width - 11))" "$value" "${C_OFF}" "${C_ACC}" "$BOX_V" "${C_OFF}"
done
printf '%s%s%s%s%s\n' "${C_ACC}" "$BOX_BL" "$rule" "$BOX_BR" "${C_OFF}"
}
# ---------------------------------------------------------------- terminal
INTERACTIVE=0 # a person is at a terminal we can ask
ARROWS=0 # that terminal supports the arrow-key menu
STTY_SAVED=""
POINTER='>'
detect_terminal() {
# Piped from curl, stdin is the script itself, so questions go to /dev/tty.
@ -40,19 +187,22 @@ detect_terminal() {
ARROWS=1
fi
fi
case "${LC_ALL:-${LC_CTYPE:-${LANG:-}}}" in
*UTF-8* | *utf-8* | *UTF8* | *utf8*) POINTER='❯' ;;
esac
}
restore_terminal() {
if [ -n "$STTY_SAVED" ]; then
stty "$STTY_SAVED" </dev/tty 2>/dev/null || true
fi
if [ "$STYLE" = 1 ] || [ -n "$STTY_SAVED" ]; then
printf '\033[?25h' >/dev/tty 2>/dev/null || true
fi
}
on_interrupt() {
if [ -n "$SPIN_PID" ]; then kill "$SPIN_PID" 2>/dev/null || true; fi
if [ -n "$SPIN_LOG" ]; then rm -f "$SPIN_LOG"; fi
if [ -n "$ENV_TMP" ]; then rm -f "$ENV_TMP"; fi
if [ "$STYLE" = 1 ]; then printf '\r\033[2K'; fi
restore_terminal
printf '\nCancelled.\n' >&2
exit 130
@ -74,6 +224,7 @@ read_key() {
}
# menu "Question" DEFAULT OPTION... -> sets CHOICE to the 1-based pick.
# An option may carry a note after a tab, shown dimmed and lined up.
menu() {
question="$1"
CHOICE="$2"
@ -81,24 +232,38 @@ menu() {
count=$#
if [ "$INTERACTIVE" != 1 ]; then return 0; fi
printf '\n%s\n' "$question" >/dev/tty
tab="$(printf '\t')"
pad=0
for opt in "$@"; do
label="${opt%%"$tab"*}"
if [ ${#label} -gt "$pad" ]; then pad=${#label}; fi
done
if [ "$STYLE" = 1 ]; then
printf '\n%s%s%s %s%s%s\n' "${C_ACC}" "$S_ASK" "${C_OFF}" "${C_BOLD}" "$question" "${C_OFF}" >/dev/tty
else
printf '\n%s\n' "$question" >/dev/tty
fi
if [ "$ARROWS" = 1 ]; then
trap on_interrupt INT TERM
stty -icanon -echo min 1 time 0 </dev/tty
printf '\033[?25l' >/dev/tty
first=1
while :; do
[ "$first" = 1 ] || printf '\033[%sA' "$count" >/dev/tty
[ "$first" = 1 ] || printf '\033[%sA' "$((count + 1))" >/dev/tty
first=0
i=1
for opt in "$@"; do
label="${opt%%"$tab"*}" note=''
[ "$label" = "$opt" ] || note="${opt#*"$tab"}"
if [ "$i" = "$CHOICE" ]; then
printf '\033[2K \033[1;36m%s %s\033[0m\n' "$POINTER" "$opt" >/dev/tty
printf '\033[2K %s%s %-*s%s %s%s%s\n' "${C_ACC}${C_BOLD}" "$POINTER" "$pad" "$label" "${C_OFF}" \
"${C_DIM}" "$note" "${C_OFF}" >/dev/tty
else
printf '\033[2K %s\n' "$opt" >/dev/tty
printf '\033[2K %-*s %s%s%s\n' "$pad" "$label" "${C_DIM}" "$note" "${C_OFF}" >/dev/tty
fi
i=$((i + 1))
done
printf '\033[2K %s%s%s\n' "${C_DIM}" "$HINT" "${C_OFF}" >/dev/tty
key="$(read_key)"
case "$key" in
up | k) [ "$CHOICE" -gt 1 ] && CHOICE=$((CHOICE - 1)) ;;
@ -107,12 +272,15 @@ menu() {
'' | "$(printf '\r')") break ;;
esac
done
restore_terminal
trap - INT TERM
printf '\033[1A\033[2K' >/dev/tty
stty "$STTY_SAVED" </dev/tty 2>/dev/null || true
printf '\033[?25h' >/dev/tty
else
i=1
for opt in "$@"; do
printf ' %s) %s\n' "$i" "$opt" >/dev/tty
label="${opt%%"$tab"*}" note=''
[ "$label" = "$opt" ] || note=" ${opt#*"$tab"}"
printf ' %s) %s%s\n' "$i" "$label" "$note" >/dev/tty
i=$((i + 1))
done
printf 'Choose [%s]: ' "$CHOICE" >/dev/tty
@ -138,18 +306,20 @@ port_free() {
pick_folder() {
home_dir="$HOME/litellm-gateway"
here_dir="$(pwd)/litellm-gateway"
found=0
if [ -n "${LITELLM_DIR:-}" ]; then
DIR="$LITELLM_DIR"
elif [ -f "$here_dir/.env" ]; then
DIR="$here_dir" # installed in this folder before
DIR="$here_dir" found=1 # installed in this folder before
elif [ -f "$home_dir/.env" ]; then
DIR="$home_dir" # installed in the home folder before
DIR="$home_dir" found=1 # installed in the home folder before
elif [ "$here_dir" = "$home_dir" ]; then
DIR="$home_dir"
else
tab="$(printf '\t')"
menu "Where should LiteLLM keep its files (.env with your keys, and the compose file)?" 1 \
"$home_dir recommended, reruns always find it" \
"$here_dir this folder"
"$(tildify "$home_dir")${tab}recommended, reruns always find it" \
"$(tildify "$here_dir")${tab}this folder"
if [ "$CHOICE" = 2 ]; then DIR="$here_dir"; else DIR="$home_dir"; fi
fi
created=0
@ -157,6 +327,23 @@ pick_folder() {
mkdir -p "$DIR"
cd "$DIR"
DIR="$(pwd)"
if [ "$found" = 1 ]; then
step "Found your install in ${C_BOLD}$(tildify "$DIR")${C_OFF}"
else
step "Files go in ${C_BOLD}$(tildify "$DIR")${C_OFF}"
fi
if [ ! -f .env ] && command -v git >/dev/null 2>&1 &&
git ls-files --error-unmatch .env >/dev/null 2>&1; then
# An ignore rule does not cover a tracked file, so new keys written here
# would show up as a change to commit.
fail "Git tracks a .env file in this folder," "so your keys could be committed."
cat >&2 <<'EOF'
Install into another folder (set LITELLM_DIR), or stop tracking the file
first with: git rm --cached .env
EOF
exit 1
fi
if [ "$created" = 1 ]; then
# A folder this script made holds only its own files, so keep all of it out of git.
printf '*\n' >.gitignore
@ -169,7 +356,7 @@ pick_folder() {
mkdir -p "$(dirname "$exclude")"
exclude="$(cd "$(dirname "$exclude")" && pwd)/exclude"
printf '/%s.env\n' "$(git rev-parse --show-prefix)" >>"$exclude"
echo "Added .env to this repository's local git exclude list ($exclude), so your keys stay out of commits."
step "Kept .env out of git ${C_DIM}(added it to this repository's local exclude list, $(tildify "$exclude"))"
elif ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
# An existing folder outside git: ignore only .env, so it stays out of
# commits if the folder becomes a repository later.
@ -196,11 +383,15 @@ pick_port() {
while ! port_free "$PORT"; do
PORT=$((PORT + 1))
if [ "$PORT" -gt 4099 ]; then
echo "Ports 4000 to 4099 are all in use. Set LITELLM_PORT to a free port and run this again." >&2
fail "Ports 4000 to 4099 are all in use." "Set LITELLM_PORT to a free port and run this again."
exit 1
fi
done
[ "$PORT" = 4000 ] || echo "Port 4000 is in use, so LiteLLM will use $PORT."
if [ "$PORT" = 4000 ]; then
step "Port ${C_BOLD}4000${C_OFF} is free"
else
warn "Port 4000 is in use, so LiteLLM will use ${C_BOLD}$PORT${C_OFF}"
fi
fi
export LITELLM_PORT="$PORT"
}
@ -213,30 +404,96 @@ check_new_install() {
[ "$DIR" = "$HOME/litellm-gateway" ] || project="litellm-gateway-$(printf '%s' "$DIR" | cksum | cut -d ' ' -f 1)"
# Postgres keeps the password it was created with, so a new password over an
# old database volume would lock the gateway out. Stop and explain instead.
if docker volume inspect "${project}_postgres_data" >/dev/null 2>&1; then
if "$ENGINE" volume inspect "${project}_postgres_data" >/dev/null 2>&1; then
fail "Found a database from an earlier install" "($ENGINE_NAME volume ${project}_postgres_data)"
cat >&2 <<EOF
Found a database from an earlier install (Docker volume ${project}_postgres_data)
but no $DIR/.env with its password.
but no $(tildify "$DIR")/.env with its password.
Restore that .env file and run this again to keep your models and keys, or
delete the old database and start fresh (this removes its models and keys):
docker volume rm ${project}_postgres_data
$ENGINE volume rm ${project}_postgres_data
EOF
exit 1
fi
}
open_browser() {
url="$1"
menu "Open the admin UI in your browser?" 1 "Yes" "No"
[ "$INTERACTIVE" = 1 ] && [ "$CHOICE" = 1 ] || return 0
if command -v open >/dev/null 2>&1; then
open "$url" >/dev/null 2>&1 || true
elif command -v xdg-open >/dev/null 2>&1; then
xdg-open "$url" >/dev/null 2>&1 || true
start_stack() {
"$ENGINE" compose -f docker-compose.quickstart.yml up -d
}
wait_ready() {
i=0
until curl -fsS "http://127.0.0.1:$PORT/health/readiness" >/dev/null 2>&1; do
i=$((i + 1))
if [ "$i" -gt 90 ]; then return 1; fi
sleep 2
done
}
open_url() {
for opener in open xdg-open; do
if command -v "$opener" >/dev/null 2>&1 && "$opener" "$1" >/dev/null 2>&1; then
step "Opened ${C_ACC}$1${C_OFF} in your browser"
return 0
fi
done
printf ' %sOpen%s %s%s%s %swhen you are ready.%s\n' "${C_DIM}" "${C_OFF}" "${C_ACC}" "$1" "${C_OFF}" "${C_DIM}" "${C_OFF}"
}
# A clipboard to copy the admin password to, or nothing. Over SSH the
# clipboard would be the server's, not yours.
find_clipboard() {
[ -z "${SSH_CONNECTION:-}${SSH_TTY:-}" ] || return 0
if command -v pbcopy >/dev/null 2>&1; then CLIP=pbcopy
elif [ -n "${WAYLAND_DISPLAY:-}" ] && command -v wl-copy >/dev/null 2>&1; then CLIP=wl-copy
elif [ -n "${DISPLAY:-}" ] && command -v xclip >/dev/null 2>&1; then CLIP="xclip -selection clipboard"
elif [ -n "${DISPLAY:-}" ] && command -v xsel >/dev/null 2>&1; then CLIP="xsel --clipboard --input"
elif command -v clip.exe >/dev/null 2>&1; then CLIP=clip.exe
fi
}
# The admin password is the master key. It goes through a pipe, so it is
# never on screen or in the process list.
copy_password() {
pw="$(sed -n 's/^LITELLM_MASTER_KEY=//p' .env | head -n 1)"
if [ -z "$pw" ]; then
warn "No LITELLM_MASTER_KEY in $(tildify "$DIR")/.env."
return 0
fi
# shellcheck disable=SC2086 # CLIP is a command and its flags
if printf '%s' "$pw" | $CLIP >/dev/null 2>&1; then
step "Copied the admin password. Paste it into the sign-in page."
else
warn "Could not copy it. The password is LITELLM_MASTER_KEY in $(tildify "$DIR")/.env."
fi
pw=''
}
# After the summary: copy the password, open the admin UI, or finish. The
# menu comes back after each choice with the next step as its default.
next_steps() {
url="$1"
[ "$INTERACTIVE" = 1 ] || return 0
if [ -z "$CLIP" ]; then
menu "Open the admin UI in your browser?" 1 "Yes" "No"
if [ "$CHOICE" = 1 ]; then open_url "$url"; fi
return 0
fi
tab="$(printf '\t')"
pick=1
while :; do
menu "What next?" "$pick" \
"Copy the admin password${tab}to the clipboard, ready to paste" \
"Open the admin UI in your browser" \
"Done"
case "$CHOICE" in
1) copy_password; pick=2 ;;
2) open_url "$url"; pick=3 ;;
*) return 0 ;;
esac
done
}
main() {
NO_QUESTIONS=0
for arg in "$@"; do
@ -246,37 +503,84 @@ main() {
esac
done
setup_output
detect_terminal
# Agents and CI get the defaults even inside a terminal, so nothing waits on a keypress.
if [ "$NO_QUESTIONS" = 1 ] || [ -n "${CI:-}" ] || [ -n "${CLAUDECODE:-}" ]; then INTERACTIVE=0; fi
trap restore_terminal EXIT
trap on_interrupt INT TERM
if ! command -v docker >/dev/null 2>&1; then
if [ "$STYLE" = 1 ]; then
printf '\n%s%s LiteLLM quickstart%s\n' "${C_ACC}${C_BOLD}" "$S_HEAD" "${C_OFF}"
printf '%s The gateway, Postgres, and the admin UI in one command%s\n\n' "${C_DIM}" "${C_OFF}"
else
echo "LiteLLM quickstart"
fi
# Podman and Rancher Desktop (nerdctl in containerd mode; its dockerd mode
# already provides a docker CLI) work the same way through Compose v2.
ENGINE=''
for cand in docker podman nerdctl; do
if command -v "$cand" >/dev/null 2>&1; then
ENGINE="$cand"
break
fi
done
if [ -z "$ENGINE" ]; then
fail "No container engine found (docker, podman, or nerdctl)." "The LiteLLM Gateway runs in a container alongside a Postgres database."
cat >&2 <<'EOF'
Docker is not installed. The LiteLLM Gateway runs in Docker alongside a Postgres database.
Install Docker, then run this again: https://docs.docker.com/get-docker/
Install Docker (https://docs.docker.com/get-docker/), Podman (https://podman.io),
or Rancher Desktop (https://rancherdesktop.io), then run this again.
Or deploy in one click (Railway or Render): https://docs.litellm.ai/docs/proxy/docker_quick_start
Only need to call models from Python? pip install litellm
EOF
exit 1
fi
docker compose version >/dev/null 2>&1 || { echo "Docker Compose v2 ('docker compose') is required." >&2; exit 1; }
docker info >/dev/null 2>&1 || { echo "Docker is installed but not running. Start it and run this again." >&2; exit 1; }
command -v openssl >/dev/null 2>&1 || { echo "openssl is required to generate keys." >&2; exit 1; }
case "$ENGINE" in
podman) ENGINE_NAME=Podman ;;
nerdctl) ENGINE_NAME=nerdctl ;;
*) ENGINE_NAME=Docker ;;
esac
compose_version="$("$ENGINE" compose version --short 2>/dev/null)" ||
{ fail "Compose v2 ('$ENGINE compose') is required."; exit 1; }
if ! "$ENGINE" info >/dev/null 2>&1; then
if [ "$ENGINE" = podman ]; then
fail "Podman is installed but not running." "Start it (podman machine start) and run this again."
else
fail "$ENGINE_NAME is installed but not running." "Start it and run this again."
fi
exit 1
fi
command -v openssl >/dev/null 2>&1 || { fail "openssl is required to generate keys."; exit 1; }
engine_version="$("$ENGINE" version --format '{{.Server.Version}}' 2>/dev/null)" || engine_version=''
step "$ENGINE_NAME ${engine_version:+$engine_version }with Compose ${compose_version#v} is running"
echo "LiteLLM quickstart"
pick_folder
[ -f .env ] || check_new_install
curl -fsSL -o docker-compose.quickstart.yml "$COMPOSE_URL"
step "Downloaded docker-compose.quickstart.yml"
pick_port
if [ -f .env ]; then
echo "Reusing $DIR/.env, so existing keys and data keep working."
step "Reusing .env, so existing keys and data keep working"
else
(umask 077 && printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\nPOSTGRES_PASSWORD=%s\nLITELLM_PORT=%s\nLITELLM_BIND=127.0.0.1:\nCOMPOSE_PROJECT_NAME=%s\n' \
"$(openssl rand -hex 32)" "$(openssl rand -hex 32)" "$(openssl rand -hex 24)" "$PORT" "$project" >.env)
echo "Generated $DIR/.env with your master key, salt key, and database password. Keep this file."
master="$(openssl rand -hex 32)"
salt="$(openssl rand -hex 32)"
db_password="$(openssl rand -hex 24)"
# Write a fresh mktemp file (mode 600, never a reused one) and rename it, so a
# failed write never leaves a partial .env that a rerun would mistake for a
# finished install.
if ! ENV_TMP="$(mktemp .env.XXXXXX)" ||
! printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\nPOSTGRES_PASSWORD=%s\nLITELLM_PORT=%s\nLITELLM_BIND=127.0.0.1:\nCOMPOSE_PROJECT_NAME=%s\n' \
"$master" "$salt" "$db_password" "$PORT" "$project" >"$ENV_TMP" || ! mv -f "$ENV_TMP" .env; then
if [ -n "$ENV_TMP" ]; then rm -f "$ENV_TMP"; fi
fail "Could not write $(tildify "$DIR")/.env."
exit 1
fi
ENV_TMP=''
unset master salt db_password
step "Generated .env with your master key, salt key, and database password ${C_DIM}(keep this file; only you can read it)"
fi
# Compose prefers values already set in the shell over .env, so drop any
@ -287,28 +591,39 @@ EOF
# is kept, so an intentional LITELLM_BIND=127.0.0.1: is not dropped.
if grep -q '^LITELLM_BIND=' .env; then unset LITELLM_BIND; fi
echo "Starting LiteLLM and Postgres (the first run downloads the images)..."
docker compose -f docker-compose.quickstart.yml up -d
i=0
until curl -fsS "http://127.0.0.1:$PORT/health/readiness" >/dev/null 2>&1; do
i=$((i + 1))
if [ "$i" -gt 90 ]; then
echo "The gateway did not become ready in 3 minutes. Check: cd $DIR && docker compose -f docker-compose.quickstart.yml logs litellm" >&2
exit 1
fi
sleep 2
started="$(date +%s)"
detail=''
for image in $("$ENGINE" compose -f docker-compose.quickstart.yml config --images 2>/dev/null); do
"$ENGINE" image inspect "$image" >/dev/null 2>&1 || detail="downloading images, first run only"
done
[ "$STYLE" = 1 ] || echo "Starting LiteLLM and Postgres (the first run downloads the images)..."
if ! spin "Starting LiteLLM and Postgres" "$detail" start_stack; then
fail "$ENGINE_NAME could not start LiteLLM and Postgres." "Its output is above."
exit 1
fi
if ! spin "Waiting for the gateway to be ready" "" wait_ready; then
fail "The gateway did not become ready in 3 minutes." "See what it logged:"
echo " cd $(tildify "$DIR") && $ENGINE compose -f docker-compose.quickstart.yml logs litellm" >&2
exit 1
fi
step "LiteLLM and Postgres are up ${C_DIM}· $(elapsed_since "$started")"
url="http://localhost:$PORT/ui"
password="the LITELLM_MASTER_KEY value in $(tildify "$DIR")/.env"
CLIP=''
if [ "$INTERACTIVE" = 1 ]; then find_clipboard; fi
if [ -n "$CLIP" ]; then
password="copy it below, or LITELLM_MASTER_KEY in $(tildify "$DIR")/.env"
fi
echo
echo "LiteLLM is running."
echo " Admin UI: http://localhost:$PORT/ui"
echo " Username: admin"
echo " Password: the LITELLM_MASTER_KEY value in $DIR/.env"
echo " Next: in the UI, open Models + Endpoints > Add Model and paste a provider API key"
echo " Stop it: cd $DIR && docker compose -f docker-compose.quickstart.yml down"
box "LiteLLM is running" \
"Admin UI|$url" \
"Username|admin" \
"Password|$password" \
"Next|in the UI, open Models + Endpoints > Add Model and paste a provider API key" \
"Stop it|cd $(tildify "$DIR") && $ENGINE compose -f docker-compose.quickstart.yml down"
open_browser "http://localhost:$PORT/ui"
next_steps "$url"
}
main "$@"

View file

@ -54,13 +54,9 @@ def get_all_functions_called_in_tests(base_dir):
if file == "test_router_validate_fallbacks.py":
print(f"tree: {tree}")
for node in ast.walk(tree):
if isinstance(node, ast.Call) and isinstance(
node.func, ast.Name
):
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
called_functions.add(node.func.id)
elif isinstance(node, ast.Call) and isinstance(
node.func, ast.Attribute
):
elif isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
called_functions.add(node.func.attr)
return called_functions
@ -91,6 +87,8 @@ ignored_function_names = [
"_get_claude_code_session_router_binding", # Tested through the two-worker session routing test in test_router.py
"_apply_updated_routing_strategy_args", # Tested via update_settings in test_lowest_latency.py (file lacks "router" in name)
"arm_routing_read_prefetch", # Tested in tests/unit/caching/test_request_redis_batch_pre_call.py (file lacks "router" in name)
"_configured_model_info", # Tested through get_configured_service_tiers in test_router.py
"_routable_deployments", # Tested through get_configured_service_tiers and get_routable_upstream_model in test_router.py
"_async_get_available_deployment", # Body of the `route {model}` phase wrapper, exercised through async_get_available_deployment in test_router.py
"_async_get_available_deployment_for_pass_through", # Same, through async_get_available_deployment_for_pass_through in test_router.py
"_embedding",
@ -109,9 +107,7 @@ def main():
# "../../litellm/router_utils/pattern_match_deployments.py",
# "../../litellm/router_utils/batch_utils.py",
# ] ## LOCAL TESTING
tests_dir = (
"./tests/" # Update this path if your tests directory is located elsewhere
)
tests_dir = "./tests/" # Update this path if your tests directory is located elsewhere
# tests_dir = "../../tests/" # LOCAL TESTING
router_functions = []
@ -119,9 +115,7 @@ def main():
router_functions.extend(get_functions_from_router(file))
print("router_functions: ", router_functions)
called_functions_in_tests = get_all_functions_called_in_tests(tests_dir)
untested_functions = [
fn for fn in router_functions if fn not in called_functions_in_tests
]
untested_functions = [fn for fn in router_functions if fn not in called_functions_in_tests]
if untested_functions:
all_untested_functions = []

View file

@ -4,7 +4,7 @@ These tests exercise a running gateway, PostgreSQL and Redis with an owned local
The `cost` group is driven by `cost_tracking_cases.json`, which contains the cost map, literal requests, literal provider responses and expected accounting values. Each case has a name, contract ID, cost-map model, optional deployment overrides, request body, tagged response and exact or recount expectations. Request bodies use `$MODEL` for the registered proxy model, while responses use `$REQUEST_ID` for the per-run scenario ID. To add a case, add a cost-map entry when the model is new, add the request body and exact provider response data, and add hand-computed expected values. The upstream serves each stored response for any path under `/<scenario_id>`, while the test-owned cost map is served over loopback through `LITELLM_MODEL_COST_MAP_URL`
Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions`, `mcp`, `sdk` or `cost` to run a selected group. The group to directory mapping is the `GROUPS` literal at the top of `run.py`; a new directory needs a `GROUPS` entry and an `OWNED_DIRECTORIES` entry in `_support/manifest.py`. Set `INTEGRATION_WORKERS` above 1 to run a group under pytest-xdist; the `mcp` job does this in CI, so MCP tests must own their resources per scenario. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate
Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions`, `mcp`, `sdk` or `cost` to run a selected group. The group to directory mapping is the `GROUPS` literal at the top of `run.py`; a new directory needs a `GROUPS` entry and an `OWNED_DIRECTORIES` entry in `_support/manifest.py`. Set `INTEGRATION_WORKERS` above 1 to run a group under pytest-xdist; the `mcp` job does this in CI, so MCP tests must own their resources per scenario. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. When that deployment runs more than one proxy worker, set `INTEGRATION_PROXY_WORKERS` to the count so a test that writes a model and then calls it waits out the config reload interval, the only cross-worker convergence bound the wire exposes. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate
Management also requires `INTEGRATION_PEER_URL`, `REDIS_HOST` and `REDIS_PORT`. CircleCI starts two directly addressed proxy processes sharing only that job's stores. The test-only CLI wrapper supplies enterprise route entitlement, following the existing behavior suite's convention. It does not qualify license validation; run it with one worker and no reload

View file

@ -0,0 +1,384 @@
from __future__ import annotations
import json
import uuid
from typing import Final, Literal
import anthropic
import httpx
import openai
import pytest
import websockets
from integration._support.client import Gateway, eventually
from integration._support.database import read_rows
from pydantic import JsonValue
def _marker() -> str:
return uuid.uuid4().hex
def _register_scenario(gateway: Gateway, scenario_id: str, response: dict[str, JsonValue]) -> None:
with httpx.Client(timeout=5, trust_env=False) as client:
result: Final = client.post(
f"{gateway.upstream_url}/__scenarios",
json={"scenario_id": scenario_id, "response": response},
)
assert result.status_code == 200, result.text
_ANTHROPIC_MESSAGE: Final = {
"id": "msg_$UNIQUE_ID",
"type": "message",
"role": "assistant",
"model": "claude-3-5-sonnet-20241022",
"content": [{"type": "text", "text": "scripted reply"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 9, "output_tokens": 5},
}
_ANTHROPIC_EVENTS: Final = (
'event: message_start\ndata: {"type":"message_start","message":{"id":"msg_$UNIQUE_ID","type":"message","role":"assistant","model":"claude-3-5-sonnet-20241022","content":[],"usage":{"input_tokens":9,"output_tokens":0}}}',
'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}',
'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"scripted reply"}}',
'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}',
'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":5}}',
'event: message_stop\ndata: {"type":"message_stop"}',
)
_RESPONSES_BODY: Final = {
"id": "resp_$UNIQUE_ID",
"object": "response",
"created_at": 1700000000,
"status": "completed",
"model": "openai/gpt-4o-mini",
"output": [{"type": "message", "content": [{"type": "output_text", "text": "done"}]}],
"usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7},
}
_RESPONSES_EVENTS: Final = (
'event: response.created\ndata: {"type":"response.created","response":{"id":"$UNIQUE_ID","object":"response","status":"in_progress"}}',
'event: response.completed\ndata: {"type":"response.completed","response":{"id":"$UNIQUE_ID","object":"response","status":"completed","output":[{"type":"message","content":[{"type":"output_text","text":"done"}]}],"usage":{"input_tokens":3,"output_tokens":4,"total_tokens":7}}}',
)
def _anthropic_model(gateway: Gateway, *, stream: bool) -> str:
scenario_id: Final = "audit-anthropic-stream" if stream else "audit-anthropic"
if stream:
_register_scenario(gateway, scenario_id, {"content_type": "text/event-stream", "frames": _ANTHROPIC_EVENTS})
else:
_register_scenario(
gateway,
scenario_id,
{
"content_type": "application/x-routed",
"routes": {"POST /v1/messages": {"content_type": "application/json", "body": _ANTHROPIC_MESSAGE}},
},
)
name: Final = f"integration-{uuid.uuid4().hex}"
created: Final = gateway.post(
"/model/new",
{
"model_name": name,
"litellm_params": {
"model": "anthropic/claude-3-5-sonnet-20241022",
"api_key": "integration-provider-key",
"api_base": f"{gateway.upstream_url}/{scenario_id}",
},
},
)
assert isinstance(created["model_info"], dict) and created["model_info"]["id"], created
return name
def _responses_model(gateway: Gateway, *, stream: bool) -> str:
scenario_id: Final = "audit-resp-stream" if stream else "audit-resp"
if stream:
_register_scenario(gateway, scenario_id, {"content_type": "text/event-stream", "frames": _RESPONSES_EVENTS})
else:
_register_scenario(
gateway,
scenario_id,
{
"content_type": "application/x-routed",
"routes": {"POST /responses": {"content_type": "application/json", "body": _RESPONSES_BODY}},
},
)
name: Final = f"integration-{uuid.uuid4().hex}"
created: Final = gateway.post(
"/model/new",
{
"model_name": name,
"litellm_params": {
"model": "openai/gpt-4o-mini",
"api_key": "integration-provider-key",
"api_base": f"{gateway.upstream_url}/{scenario_id}",
},
},
)
assert isinstance(created["model_info"], dict) and created["model_info"]["id"], created
return name
def _openai_client(gateway: Gateway, key: str) -> openai.OpenAI:
return openai.OpenAI(
base_url=f"{gateway.client.base_url}/v1",
api_key=key,
http_client=httpx.Client(trust_env=False),
)
def _openai_async_client(gateway: Gateway, key: str) -> openai.AsyncOpenAI:
return openai.AsyncOpenAI(
base_url=f"{gateway.client.base_url}/v1",
api_key=key,
http_client=httpx.AsyncClient(trust_env=False),
)
def _anthropic_client(gateway: Gateway, key: str) -> anthropic.Anthropic:
return anthropic.Anthropic(
base_url=str(gateway.client.base_url).rstrip("/"),
api_key=key,
http_client=httpx.Client(trust_env=False),
)
def _anthropic_async_client(gateway: Gateway, key: str) -> anthropic.AsyncAnthropic:
return anthropic.AsyncAnthropic(
base_url=str(gateway.client.base_url).rstrip("/"),
api_key=key,
http_client=httpx.AsyncClient(trust_env=False),
)
def _sse_chunk_models(text: str) -> tuple[str, ...]:
models: list[str] = []
for line in text.splitlines():
if not line.startswith("data: ") or line == "data: [DONE]":
continue
value = json.loads(line.removeprefix("data: "))
if isinstance(value, dict) and isinstance(value.get("model"), str):
models.append(value["model"])
return tuple(models)
@pytest.mark.parametrize("client_kind", ("httpx", "openai", "openai_async"))
@pytest.mark.parametrize("stream", (False, True))
def test_allowed_key_chat_completions_restamps_alias(
gateway: Gateway, client_kind: Literal["httpx", "openai", "openai_async"], stream: bool
) -> None:
with gateway.scenario() as scenario:
model: Final = scenario.model()
key: Final = scenario.key(models=[model])
body: Final = {
"model": model,
"messages": [{"role": "user", "content": _marker()}],
"stream": stream,
**({"stream_options": {"include_usage": True}} if stream else {}),
}
if client_kind == "httpx":
response: Final = gateway.request("POST", "/v1/chat/completions", body, key=key)
assert response.status_code == 200, response.text
if stream:
models: Final = _sse_chunk_models(response.text)
assert models and all(chunk_model == model for chunk_model in models), response.text
else:
assert response.json()["model"] == model, response.text
elif client_kind == "openai":
client: Final = _openai_client(gateway, key)
completion: Final = client.chat.completions.create(**body)
if stream:
chunk_models: list[str] = []
for chunk in completion:
if chunk.model:
chunk_models.append(chunk.model)
assert chunk_models and all(chunk_model == model for chunk_model in chunk_models)
else:
assert completion.model == model
else:
async def _run() -> None:
client_async: Final = _openai_async_client(gateway, key)
completion_async: Final = await client_async.chat.completions.create(**body)
if stream:
seen: list[str] = []
async for chunk in completion_async:
if chunk.model:
seen.append(chunk.model)
assert seen and all(chunk_model == model for chunk_model in seen)
else:
assert completion_async.model == model
import asyncio
asyncio.run(_run())
@pytest.mark.parametrize("client_kind", ("httpx", "anthropic", "anthropic_async"))
@pytest.mark.parametrize("stream", (False, True))
def test_allowed_key_messages(gateway: Gateway, client_kind: str, stream: bool) -> None:
model: Final = _anthropic_model(gateway, stream=stream)
with gateway.scenario() as scenario:
key: Final = scenario.key(models=[model])
body: Final = {
"model": model,
"max_tokens": 64,
"messages": [{"role": "user", "content": _marker()}],
"stream": stream,
}
if client_kind == "httpx":
response: Final = gateway.request("POST", "/v1/messages", body, key=key)
assert response.status_code == 200, response.text
if stream:
assert "message_start" in response.text and "message_stop" in response.text, response.text
else:
assert response.json()["type"] == "message", response.text
elif client_kind == "anthropic":
client: Final = _anthropic_client(gateway, key)
if stream:
with client.messages.stream(**{k: v for k, v in body.items() if k != "stream"}) as event_stream:
message: Final = event_stream.get_final_message()
assert message.type == "message" and message.content, message
else:
message = client.messages.create(**{k: v for k, v in body.items() if k != "stream"})
assert message.type == "message" and message.content, message
else:
async def _run() -> None:
client_async: Final = _anthropic_async_client(gateway, key)
params: Final = {k: v for k, v in body.items() if k != "stream"}
if stream:
async with client_async.messages.stream(**params) as event_stream:
message_async: Final = await event_stream.get_final_message()
else:
message_async = await client_async.messages.create(**params)
assert message_async.type == "message" and message_async.content, message_async
import asyncio
asyncio.run(_run())
@pytest.mark.parametrize("client_kind", ("httpx", "openai", "openai_async"))
@pytest.mark.parametrize("stream", (False, True))
def test_allowed_key_responses(gateway: Gateway, client_kind: str, stream: bool) -> None:
model: Final = _responses_model(gateway, stream=stream)
with gateway.scenario() as scenario:
key: Final = scenario.key(models=[model])
body: Final = {
"model": model,
"input": [{"role": "user", "content": [{"type": "input_text", "text": _marker()}]}],
"stream": stream,
}
if client_kind == "httpx":
response: Final = gateway.request("POST", "/v1/responses", body, key=key)
assert response.status_code == 200, response.text
if stream:
assert "response.completed" in response.text, response.text
else:
assert response.json()["object"] == "response", response.text
elif client_kind == "openai":
client: Final = _openai_client(gateway, key)
result: Final = client.responses.create(**body)
if stream:
events: list[str] = []
for event in result:
events.append(event.type)
assert "response.completed" in events, events
else:
assert result.object == "response", result
else:
async def _run() -> None:
client_async: Final = _openai_async_client(gateway, key)
result_async: Final = await client_async.responses.create(**body)
if stream:
events_async: list[str] = []
async for event in result_async:
events_async.append(event.type)
assert "response.completed" in events_async, events_async
else:
assert result_async.object == "response", result_async
import asyncio
asyncio.run(_run())
def test_denied_key_rejected(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
model: Final = scenario.model()
other_model: Final = scenario.model()
key: Final = scenario.key(models=[other_model])
for path, body in (
("/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": "x"}]}),
("/v1/messages", {"model": model, "max_tokens": 4, "messages": [{"role": "user", "content": "x"}]}),
("/v1/responses", {"model": model, "input": "x"}),
):
response: Final = gateway.request("POST", path, body, key=key)
assert response.status_code in (401, 403), f"{path}: {response.status_code} {response.text}"
async def test_realtime_websocket_allowed_and_denied(gateway: Gateway) -> None:
rt_scenario: Final = f"auditrt{uuid.uuid4().hex[:8]}"
_register_scenario(
gateway,
rt_scenario,
{
"content_type": "application/x-realtime",
"session_model": "gpt-4o-realtime-preview",
"events": (
{"type": "response.done", "response": {"id": "resp_$UNIQUE_ID", "model": "gpt-4o-realtime-preview"}},
),
},
)
name: Final = f"integration-{uuid.uuid4().hex}"
created: Final = gateway.post(
"/model/new",
{
"model_name": name,
"litellm_params": {
"model": "openai/gpt-4o-realtime-preview",
"api_key": rt_scenario,
"api_base": gateway.upstream_url,
},
},
)
assert isinstance(created["model_info"], dict) and created["model_info"]["id"], created
with gateway.scenario() as scenario:
key: Final = scenario.key(models=[name])
other_model: Final = scenario.model()
denied_key: Final = scenario.key(models=[other_model])
url: Final = f"{str(gateway.client.base_url).replace('http://', 'ws://')}/v1/realtime?model={name}"
async with websockets.connect(url, additional_headers={"Authorization": f"Bearer {key}"}) as websocket:
session: Final = json.loads(await websocket.recv())
assert session["type"] == "session.created", session
assert session["session"]["model"] == "gpt-4o-realtime-preview", session
await websocket.send(json.dumps({"type": "response.create"}))
event: Final = json.loads(await websocket.recv())
assert event["type"] == "response.done", event
with pytest.raises(websockets.exceptions.InvalidHandshake):
async with websockets.connect(
url, additional_headers={"Authorization": f"Bearer {denied_key}"}
) as denied_socket:
await denied_socket.recv()
def test_max_budget_allows_then_denies_and_records_spend(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
model: Final = scenario.model()
key: Final = scenario.key(models=[model], max_budget=0.0000001)
body: Final = {"model": model, "messages": [{"role": "user", "content": _marker()}]}
first: Final = gateway.request("POST", "/v1/chat/completions", body, key=key)
assert first.status_code == 200, first.text
request_id: Final = first.json()["id"]
rows: Final = eventually(
lambda: read_rows('SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id = %s', (request_id,)),
lambda values: len(values) == 1,
seconds=70,
)
assert rows, first.text
second: Final = gateway.request("POST", "/v1/chat/completions", body, key=key)
assert second.status_code == 422, second.text
assert "budget_exceeded" in second.text, second.text

View file

@ -0,0 +1,826 @@
import asyncio
import json
import os
import time
import uuid
from collections.abc import Callable, Mapping, Sequence
from concurrent.futures import ThreadPoolExecutor
from hashlib import sha256
from itertools import chain
from pathlib import Path
from typing import Final, TypeVar
import anthropic
import httpx
import openai
from pydantic import JsonValue
from litellm.constants import PROXY_CONFIG_RELOAD_INTERVAL_SECONDS
from tests.integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, object_value, string_value
from tests.integration._support.database import read_rows
from tests.integration._support.wire import Reply, Request, Wire, wire_server
T = TypeVar("T")
CLIENT_VERSION: Final = "0.159.3"
CONSISTENT_READS: Final = 6
CONVERGENCE_SECONDS: Final = 60
RELOAD_MARGIN_SECONDS: Final = 5.0
PROXY_WORKERS: Final = int(os.environ.get("INTEGRATION_PROXY_WORKERS", "1"))
WORKER_SYNC_SECONDS: Final = 0.0 if PROXY_WORKERS == 1 else PROXY_CONFIG_RELOAD_INTERVAL_SECONDS + RELOAD_MARGIN_SECONDS
ROOT: Final = Path(os.environ.get("INTEGRATION_PROXY_ROOT") or Path(__file__).resolve().parents[3])
BUNDLED_STOCK_PATH: Final = ROOT / "litellm" / "proxy" / "common_utils" / "codex_bundled_models_0.159.3.json"
BASE_INSTRUCTIONS_PATH: Final = ROOT / "litellm" / "proxy" / "common_utils" / "codex_base_instructions.md"
FRESH_CONNECTION: Final = {"Connection": "close"}
ANTHROPIC_HEADERS: Final = {"anthropic-version": "2023-06-01"}
USAGE: Final = {"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15}
def _generic_tier(identity: str) -> dict[str, JsonValue]:
return {"id": identity, "name": identity.capitalize(), "description": f"Sends service_tier={identity} upstream"}
def _bundled_stock() -> dict[str, dict[str, JsonValue]]:
catalog: Final = JSON_OBJECT.validate_json(BUNDLED_STOCK_PATH.read_bytes())
models: Final = catalog["models"]
assert isinstance(models, list), catalog
return {string_value(object_value(model)["slug"]): object_value(model) for model in models}
def _catalog_response(
gateway: Gateway,
*,
key: str | None = None,
params: Mapping[str, str] | None = None,
headers: Mapping[str, str] | None = None,
path: str = "/v1/models",
) -> httpx.Response:
return gateway.request(
"GET",
path,
key=key,
params={"client_version": CLIENT_VERSION, **(params or {})},
headers={**FRESH_CONNECTION, **(headers or {})},
)
def _entries(response: httpx.Response) -> list[dict[str, JsonValue]]:
assert response.status_code == 200, response.text
body: Final = JSON_OBJECT.validate_json(response.content)
assert set(body) == {"models"}, response.text
models: Final = body["models"]
assert isinstance(models, list), response.text
return [object_value(entry) for entry in models]
def _catalog(
gateway: Gateway, *, key: str | None = None, params: Mapping[str, str] | None = None
) -> list[dict[str, JsonValue]]:
return _entries(_catalog_response(gateway, key=key, params=params))
def _by_slug(entries: Sequence[Mapping[str, JsonValue]]) -> dict[str, dict[str, JsonValue]]:
return {string_value(entry["slug"]): dict(entry) for entry in entries}
def _tier_ids(entry: Mapping[str, JsonValue]) -> tuple[str, ...]:
tiers: Final = entry["service_tiers"]
assert isinstance(tiers, list), entry
return tuple(string_value(object_value(tier)["id"]) for tier in tiers)
def _tiers_in(entries: Sequence[Mapping[str, JsonValue]]) -> dict[str, tuple[str, ...]]:
return {slug: _tier_ids(entry) for slug, entry in _by_slug(entries).items()}
def _tiers_by_slug(
gateway: Gateway, *, key: str | None = None, params: Mapping[str, str] | None = None
) -> dict[str, tuple[str, ...]]:
return _tiers_in(_catalog(gateway, key=key, params=params))
def _converged_catalog(
gateway: Gateway,
satisfied: Callable[[dict[str, dict[str, JsonValue]]], bool],
*,
key: str | None = None,
params: Mapping[str, str] | None = None,
) -> dict[str, dict[str, JsonValue]]:
return eventually(
lambda: _by_slug(_catalog(gateway, key=key, params=params)), satisfied, seconds=CONVERGENCE_SECONDS
)
def _settled(read: Callable[[], T], satisfied: Callable[[T], bool], *, seconds: float = CONVERGENCE_SECONDS) -> T:
reads: Final = eventually(
lambda: tuple(read() for _ in range(CONSISTENT_READS)),
lambda values: all(satisfied(value) for value in values),
seconds=seconds,
)
return reads[-1]
def _settled_on_every_worker(read: Callable[[], T], satisfied: Callable[[T], bool], *, written_at: float) -> T:
eventually(
lambda: (time.monotonic() - written_at, read()),
lambda stamped: stamped[0] >= WORKER_SYNC_SECONDS and satisfied(stamped[1]),
seconds=WORKER_SYNC_SECONDS + CONVERGENCE_SECONDS,
)
return _settled(read, satisfied)
def _plain_ids(gateway: Gateway, *, key: str | None = None, params: Mapping[str, str] | None = None) -> list[str]:
response: Final = gateway.request("GET", "/v1/models", key=key, params=params, headers=FRESH_CONNECTION)
assert response.status_code == 200, response.text
body: Final = JSON_OBJECT.validate_json(response.content)
assert set(body) == {"data", "object"} and body["object"] == "list", response.text
data: Final = body["data"]
assert isinstance(data, list), response.text
return [string_value(object_value(entry)["id"]) for entry in data]
def _model_info_rows(gateway: Gateway, model_name: str) -> list[dict[str, JsonValue]]:
entries: Final = gateway.get("/model/info")["data"]
assert isinstance(entries, list), entries
return [object_value(entry) for entry in entries if object_value(entry)["model_name"] == model_name]
def _delete_model_if_present(gateway: Gateway, identity: str) -> None:
if read_rows('SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s', (identity,)):
gateway.post("/model/delete", {"id": identity})
assert read_rows('SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s', (identity,)) == []
def _new_model(
scenario: Scenario,
name: str,
*,
model: str = "openai/gpt-4o-mini",
model_info: Mapping[str, JsonValue] | None = None,
api_base: str | None = None,
) -> str:
created: Final = scenario.gateway.post(
"/model/new",
{
"model_name": name,
"litellm_params": {
"model": model,
"api_key": "integration-provider-key",
"api_base": api_base or f"{scenario.gateway.upstream_url}/v1",
},
"model_info": dict(model_info) if model_info is not None else {},
},
)
identity: Final = string_value(object_value(created["model_info"])["id"])
scenario.cleanups.callback(_delete_model_if_present, scenario.gateway, identity)
return identity
def _model_name(prefix: str) -> str:
return f"{prefix}-{uuid.uuid4().hex}"
def test_client_version_answers_codex_catalog_shape(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
model: Final = scenario.model(model_info={"max_input_tokens": 2048})
catalog: Final = _converged_catalog(
gateway, lambda value: model in value and value[model]["context_window"] == 2048
)
slugs: Final = list(catalog)
entry: Final = catalog[model]
assert entry["display_name"] == model, entry
assert entry["priority"] == slugs.index(model), entry
assert entry["visibility"] == "list" and entry["supported_in_api"] is True, entry
assert entry["service_tiers"] == [], entry
assert entry["context_window"] == 2048, entry
assert entry["base_instructions"] == BASE_INSTRUCTIONS_PATH.read_text(encoding="utf-8"), entry["slug"]
def test_models_route_matches_v1_models(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
model: Final = scenario.model()
key: Final = scenario.key(models=[model])
routes: Final = eventually(
lambda: (_entries(_catalog_response(gateway, key=key, path="/models")), _catalog(gateway, key=key)),
lambda pair: pair[0] == pair[1],
seconds=CONVERGENCE_SECONDS,
)
assert [entry["slug"] for entry in routes[0]] == [model], routes
def test_plain_listing_keeps_openai_shape(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
model: Final = scenario.model(model_info={"service_tiers": ["priority"]})
key: Final = scenario.key(models=[model])
ids: Final = eventually(
lambda: _plain_ids(gateway, key=key), lambda value: value == [model], seconds=CONVERGENCE_SECONDS
)
assert ids == [model], ids
response: Final = gateway.request("GET", "/v1/models", key=key, headers=FRESH_CONNECTION)
assert "service_tiers" not in response.text and "base_instructions" not in response.text, response.text
def test_client_version_wins_over_anthropic_headers(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
model: Final = scenario.model()
key: Final = scenario.key(models=[model])
codex: Final = eventually(
lambda: _catalog_response(gateway, key=key, headers={**ANTHROPIC_HEADERS, "x-api-key": key}),
lambda response: [entry["slug"] for entry in _entries(response)] == [model],
seconds=CONVERGENCE_SECONDS,
)
assert [entry["slug"] for entry in _entries(codex)] == [model], codex.text
anthropic_listing: Final = gateway.request(
"GET", "/v1/models", key=key, headers={**ANTHROPIC_HEADERS, "x-api-key": key, **FRESH_CONNECTION}
)
assert anthropic_listing.status_code == 200, anthropic_listing.text
body: Final = JSON_OBJECT.validate_json(anthropic_listing.content)
assert set(body) == {"data", "first_id", "has_more", "last_id"}, anthropic_listing.text
def test_cursor_models_route_ignores_client_version(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
model: Final = scenario.model()
key: Final = scenario.key(models=[model])
response: Final = eventually(
lambda: _catalog_response(gateway, key=key, path="/cursor/v1/models"),
lambda value: value.status_code == 200 and model in value.text,
seconds=CONVERGENCE_SECONDS,
)
assert response.status_code == 200, response.text
body: Final = JSON_OBJECT.validate_json(response.content)
assert set(body) == {"data", "object"}, response.text
data: Final = body["data"]
assert isinstance(data, list) and [object_value(entry)["id"] for entry in data] == [model], response.text
def test_client_version_value_shapes(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
model: Final = scenario.model(model_info={"service_tiers": ["priority"]})
key: Final = scenario.key(models=[model])
for value in ("", "1", "a" * 5000):
response: Final = gateway.request(
"GET", "/v1/models", key=key, params={"client_version": value}, headers=FRESH_CONNECTION
)
assert [entry["slug"] for entry in _entries(response)] == [model], (value[:20], response.text)
repeated: Final = gateway.client.get(
"/v1/models?client_version=1&client_version=2",
headers={"Authorization": f"Bearer {key}", **FRESH_CONNECTION},
)
assert [entry["slug"] for entry in _entries(repeated)] == [model], repeated.text
eventually(
lambda: (_catalog(gateway, key=key), _catalog(gateway, key=key)),
lambda pair: pair[0] == pair[1] and _tiers_in(pair[0]) == {model: ("priority",)},
seconds=CONVERGENCE_SECONDS,
)
unauthenticated: Final = gateway.client.get(
"/v1/models", params={"client_version": CLIENT_VERSION}, headers=FRESH_CONNECTION
)
assert unauthenticated.status_code == 401, unauthenticated.text
def test_configured_tiers_on_unknown_model(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
ultrafast: Final = {"id": "ultrafast", "name": "Ultra", "description": "fast lane"}
model: Final = scenario.model(
model_info={
"service_tiers": ["priority", ultrafast],
"display_name": "Probe Unknown",
"max_input_tokens": 1234,
}
)
entry: Final = _converged_catalog(
gateway, lambda value: model in value and value[model]["context_window"] == 1234
)[model]
assert entry["service_tiers"] == [_generic_tier("priority"), ultrafast], entry
assert entry["display_name"] == "Probe Unknown", entry
assert entry["context_window"] == 1234, entry
assert entry["upgrade"] is None, entry
def test_stock_model_keeps_codex_entry(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
name: Final = _model_name("codex-stock")
_new_model(scenario, name, model="openai/gpt-5.5")
catalog: Final = _converged_catalog(gateway, lambda value: name in value)
slugs: Final = list(catalog)
stock: Final = _bundled_stock()["gpt-5.5"]
assert stock["upgrade"] is not None and stock["service_tiers"] != [], stock
expected: Final = {
**stock,
"slug": name,
"display_name": name,
"priority": slugs.index(name),
"visibility": "list",
"supported_in_api": True,
"upgrade": None,
}
assert catalog[name] == expected, catalog[name]
def test_stock_model_configured_tiers_override(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
stock: Final = _bundled_stock()["gpt-6-sol"]
stock_tiers: Final = stock["service_tiers"]
assert isinstance(stock_tiers, list) and len(stock_tiers) == 1, stock
stock_tier: Final = object_value(stock_tiers[0])
assert stock["default_service_tier"] == stock_tier["id"], stock
unknown_tier: Final = _model_name("codex-unknown-tier")
known_tier: Final = _model_name("codex-known-tier")
_new_model(scenario, unknown_tier, model="openai/gpt-6-sol", model_info={"service_tiers": ["ultrafast"]})
_new_model(scenario, known_tier, model="openai/gpt-6-sol", model_info={"service_tiers": [stock_tier["id"]]})
catalog: Final = _converged_catalog(gateway, lambda value: unknown_tier in value and known_tier in value)
assert catalog[unknown_tier]["service_tiers"] == [_generic_tier("ultrafast")], catalog[unknown_tier]
assert catalog[unknown_tier]["default_service_tier"] is None, catalog[unknown_tier]
assert catalog[known_tier]["service_tiers"] == [stock_tier], catalog[known_tier]
assert catalog[known_tier]["default_service_tier"] == stock_tier["id"], catalog[known_tier]
assert catalog[known_tier]["display_name"] == known_tier, catalog[known_tier]
def test_stock_model_empty_tiers_disable(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
name: Final = _model_name("codex-no-tiers")
_new_model(scenario, name, model="openai/gpt-6-sol", model_info={"service_tiers": []})
entry: Final = _converged_catalog(gateway, lambda value: name in value)[name]
assert entry["service_tiers"] == [] and entry["default_service_tier"] is None, entry
def test_two_deployments_offer_only_shared_tiers(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
shared: Final = _model_name("codex-shared")
half_set: Final = _model_name("codex-half-set")
_new_model(scenario, shared, model_info={"service_tiers": ["a", "b"]})
_new_model(scenario, shared, model_info={"service_tiers": ["b", "c"]})
_new_model(scenario, half_set, model_info={"service_tiers": ["a"]})
_new_model(scenario, half_set)
tiers: Final = _settled(
lambda: _tiers_by_slug(gateway),
lambda value: value.get(shared) == ("b",) and value.get(half_set) == (),
)
assert tiers[shared] == ("b",) and tiers[half_set] == (), tiers
def test_invalid_tiers_offer_nothing_and_keep_listing(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
as_string: Final = scenario.model(model_info={"service_tiers": "priority"})
stray_key: Final = scenario.model(
model_info={"service_tiers": [{"id": "x", "name": "X", "description": "d", "stray": 1}]}
)
empty_id: Final = scenario.model(model_info={"service_tiers": [{"id": "", "name": "", "description": ""}]})
control: Final = scenario.model(model_info={"service_tiers": ["priority"]})
written: Final = (as_string, stray_key, empty_id, control)
catalog: Final = _converged_catalog(gateway, lambda value: all(slug in value for slug in written))
assert catalog[as_string]["service_tiers"] == [], catalog[as_string]
assert catalog[stray_key]["service_tiers"] == [], catalog[stray_key]
assert catalog[empty_id]["service_tiers"] == [], catalog[empty_id]
assert catalog[control]["service_tiers"] == [_generic_tier("priority")], catalog[control]
def test_blocked_deployment_tiers_are_ignored(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
name: Final = _model_name("codex-blockable")
_new_model(scenario, name, model_info={"service_tiers": ["a", "b"]})
narrowing: Final = _new_model(scenario, name, model_info={"service_tiers": ["b", "c"]})
_settled(lambda: _tiers_by_slug(gateway).get(name), lambda value: value == ("b",))
blocked: Final = gateway.request("PATCH", f"/model/{narrowing}/update", {"blocked": True})
assert blocked.status_code == 200, blocked.text
_settled(lambda: _tiers_by_slug(gateway).get(name), lambda value: value == ("a", "b"))
unblocked: Final = gateway.request("PATCH", f"/model/{narrowing}/update", {"blocked": False})
assert unblocked.status_code == 200, unblocked.text
_settled(lambda: _tiers_by_slug(gateway).get(name), lambda value: value == ("b",))
def test_restricted_key_lists_only_its_models(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
tiered: Final = scenario.model(model_info={"service_tiers": ["priority"]})
other: Final = scenario.model(model_info={"service_tiers": ["flex"]})
key: Final = scenario.key(models=[tiered])
tiers: Final = eventually(
lambda: _tiers_by_slug(gateway, key=key),
lambda value: value == {tiered: ("priority",)},
seconds=CONVERGENCE_SECONDS,
)
assert tiers == {tiered: ("priority",)}, tiers
admin_tiers: Final = eventually(
lambda: _tiers_by_slug(gateway), lambda value: value.get(other) == ("flex",), seconds=CONVERGENCE_SECONDS
)
assert admin_tiers[other] == ("flex",), admin_tiers
def test_team_scoped_model_tiers(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
owning_team: Final = scenario.team()
other_team: Final = scenario.team()
public_name: Final = _model_name("codex-team-model")
_new_model(scenario, public_name, model_info={"team_id": owning_team, "service_tiers": ["priority"]})
owning_key: Final = scenario.key(team_id=owning_team)
other_key: Final = scenario.key(team_id=other_team)
owning_tiers: Final = _settled(
lambda: _tiers_by_slug(gateway, key=owning_key), lambda value: value.get(public_name) == ("priority",)
)
assert owning_tiers[public_name] == ("priority",), owning_tiers
admin_tiers: Final = _settled(
lambda: _tiers_by_slug(gateway, params={"team_id": owning_team}),
lambda value: value.get(public_name) == ("priority",),
)
assert admin_tiers[public_name] == ("priority",), admin_tiers
assert public_name not in _tiers_by_slug(gateway, key=other_key), other_key[:8]
assert public_name not in _tiers_by_slug(gateway)
def test_wildcard_deployment_never_listed(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
model: Final = scenario.model()
_new_model(scenario, "openai/*", model="openai/*", model_info={"service_tiers": ["priority"]})
flags: Final = {"return_wildcard_routes": "true"}
plain: Final = eventually(
lambda: _plain_ids(gateway, params=flags), lambda ids: "openai/*" in ids and model in ids
)
assert "openai/*" in plain, plain
slugs: Final = eventually(
lambda: [string_value(entry["slug"]) for entry in _catalog(gateway, params=flags)],
lambda values: model in values,
)
assert all("*" not in slug for slug in slugs), slugs
def test_a_tiered_model_listed_last_survives_the_byte_cut(gateway: Gateway) -> None:
"""Codex 0.159.3 rejects a `model_catalog_url` body over `MAX_MODEL_CATALOG_BYTES` (1 MiB,
`codex-rs/model-provider/src/models_endpoint.rs`, read on 2026-10-03), so the proxy keeps the
entries offering a tier first and then listing order; fallback entries carry Codex's base
prompt, so some fifty of them overrun the limit."""
with gateway.scenario() as scenario:
plain: Final = tuple(scenario.model() for _ in range(52))
tiered: Final = scenario.model(model_info={"service_tiers": ["ultrafast"]})
key: Final = scenario.key(models=[*plain, tiered])
listing: Final = eventually(
lambda: _plain_ids(gateway, key=key), lambda ids: set(ids) == {*plain, tiered}, seconds=CONVERGENCE_SECONDS
)
assert listing.index(tiered) == len(plain), listing
response: Final = eventually(
lambda: _catalog_response(gateway, key=key),
lambda value: value.status_code == 200 and tiered in {entry["slug"] for entry in _entries(value)},
seconds=CONVERGENCE_SECONDS,
)
entries: Final = _entries(response)
slugs: Final = [string_value(entry["slug"]) for entry in entries]
assert len(response.content) <= 1024 * 1024, len(response.content)
assert 0 < len(slugs) < len(listing), (len(slugs), len(listing))
assert slugs == [slug for slug in listing if slug in slugs], slugs
assert slugs[-1] == tiered and _tier_ids(entries[-1]) == ("ultrafast",), entries[-1]
assert slugs[:-1] == list(plain[: len(slugs) - 1]), slugs
assert [entry["priority"] for entry in entries] == [listing.index(slug) for slug in slugs], slugs
def test_an_oversized_tiered_model_is_left_out_alone(gateway: Gateway) -> None:
"""A tier description longer than Codex's whole byte limit makes an entry that can never fit;
it is taken first as a tiered entry, so the cut passes over it and still keeps the models after it."""
with gateway.scenario() as scenario:
plain: Final = scenario.model()
oversized_tier: Final = {"id": "huge", "name": "Huge", "description": "x" * (1024 * 1024)}
oversized: Final = scenario.model(model_info={"service_tiers": [oversized_tier]})
tiered: Final = scenario.model(model_info={"service_tiers": ["ultrafast"]})
key: Final = scenario.key(models=[plain, oversized, tiered])
listing: Final = eventually(
lambda: _plain_ids(gateway, key=key),
lambda ids: set(ids) == {plain, oversized, tiered},
seconds=CONVERGENCE_SECONDS,
)
assert listing == [plain, oversized, tiered], listing
def kept() -> tuple[int, tuple[tuple[JsonValue, JsonValue, tuple[str, ...]], ...]]:
response: Final = _catalog_response(gateway, key=key)
return len(response.content), tuple(
(entry["slug"], entry["priority"], _tier_ids(entry)) for entry in _entries(response)
)
expected: Final = ((plain, 0, ()), (tiered, 2, ("ultrafast",)))
size, entries = _settled(kept, lambda value: value[1] == expected)
assert entries == expected, entries
assert size <= 1024 * 1024, size
def test_model_info_echoes_configured_tiers(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
configured: Final = ["priority", {"id": "ultrafast", "name": "Ultra", "description": "fast lane"}]
model: Final = scenario.model(model_info={"service_tiers": configured})
matching: Final = eventually(
lambda: _model_info_rows(gateway, model), lambda rows: len(rows) == 1, seconds=CONVERGENCE_SECONDS
)
assert len(matching) == 1, model
assert object_value(matching[0]["model_info"])["service_tiers"] == configured, matching[0]
def test_tier_update_propagates(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
name: Final = _model_name("codex-updatable")
identity: Final = _new_model(scenario, name, model_info={"service_tiers": ["priority"]})
_settled(lambda: _tiers_by_slug(gateway).get(name), lambda value: value == ("priority",))
updated: Final = gateway.request(
"PATCH", f"/model/{identity}/update", {"model_info": {"service_tiers": ["flex", "ultrafast"]}}
)
assert updated.status_code == 200, updated.text
_settled(lambda: _tiers_by_slug(gateway).get(name), lambda value: value == ("flex", "ultrafast"))
def test_model_delete_drops_the_slug(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
name: Final = _model_name("codex-deletable")
identity: Final = _new_model(scenario, name, model_info={"service_tiers": ["priority"]})
_settled(lambda: _tiers_by_slug(gateway).get(name), lambda value: value == ("priority",))
gateway.post("/model/delete", {"id": identity})
_settled(lambda: _tiers_by_slug(gateway).get(name), lambda value: value is None)
def _sse(events: Sequence[object]) -> tuple[bytes, ...]:
return tuple(b"data: " + json.dumps(event).encode() + b"\n\n" for event in events) + (b"data: [DONE]\n\n",)
def _chat_reply(stream: bool) -> Reply:
identity: Final = "chatcmpl-" + uuid.uuid4().hex
if not stream:
return Reply(
body=json.dumps(
{
"id": identity,
"object": "chat.completion",
"created": 1,
"model": "gpt-4o-mini",
"choices": [
{"index": 0, "message": {"role": "assistant", "content": "tier probe"}, "finish_reason": "stop"}
],
"usage": USAGE,
}
).encode()
)
head: Final = {"id": identity, "object": "chat.completion.chunk", "created": 1, "model": "gpt-4o-mini"}
return Reply(
content_type="text/event-stream",
chunks=_sse(
(
{**head, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "tier "}}]},
{**head, "choices": [{"index": 0, "delta": {"content": "probe"}}]},
{**head, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},
{**head, "choices": [], "usage": USAGE},
)
),
)
def _responses_reply() -> Reply:
identity: Final = uuid.uuid4().hex
return Reply(
body=json.dumps(
{
"id": "resp_" + identity,
"object": "response",
"created_at": 1,
"status": "completed",
"model": "gpt-4o-mini",
"output": [
{
"type": "message",
"id": "msg_" + identity,
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "tier probe", "annotations": []}],
}
],
"parallel_tool_calls": False,
"tool_choice": "auto",
"tools": [],
"usage": {"input_tokens": 11, "output_tokens": 4, "total_tokens": 15},
}
).encode()
)
def _upstream(request: Request) -> Reply:
if request.method == "GET" and request.target.endswith("/models"):
return Reply(body=json.dumps({"object": "list", "data": []}).encode())
if request.target.endswith("/responses"):
return _responses_reply()
return _chat_reply(json.loads(request.body).get("stream") is True)
def _v1(gateway: Gateway) -> str:
return str(gateway.client.base_url).rstrip("/") + "/v1"
def _llm_requests(wire: Wire) -> tuple[Request, ...]:
return tuple(request for request in wire.drain() if request.method == "POST")
def _request_bodies(requests: Sequence[Request]) -> list[dict[str, JsonValue]]:
return [JSON_OBJECT.validate_json(request.body) for request in requests]
def _texts(value: JsonValue) -> tuple[str, ...]:
if isinstance(value, str):
return (value,)
if isinstance(value, list):
return tuple(chain.from_iterable(_texts(item) for item in value))
if isinstance(value, dict):
return tuple(chain.from_iterable(_texts(value.get(key)) for key in ("content", "text", "input", "messages")))
return ()
def _prompt_of(body: Mapping[str, JsonValue]) -> str:
return " ".join(_texts(body.get("input", body.get("messages"))))
def _sent_prompt(body: Mapping[str, JsonValue], sent: Sequence[str]) -> str:
text: Final = _prompt_of(body)
matches: Final = [prompt for prompt in sent if prompt in text]
assert len(matches) == 1, (matches, text)
return matches[0]
def _spend_ids_for_key(key: str) -> list[str]:
rows: Final = read_rows(
'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE api_key=%s', (sha256(key.encode()).hexdigest(),)
)
return [string_value(row["request_id"]) for row in rows]
def test_advertised_tier_reaches_the_upstream(gateway: Gateway) -> None:
with wire_server(_upstream) as wire, gateway.scenario() as scenario:
written_at: Final = time.monotonic()
model: Final = scenario.model(api_base=f"{wire.url}/v1", model_info={"service_tiers": ["priority"]})
key: Final = scenario.key(models=[model])
marker: Final = uuid.uuid4().hex
advertised: Final = _settled_on_every_worker(
lambda: _tiers_by_slug(gateway, key=key),
lambda tiers: tiers == {model: ("priority",)},
written_at=written_at,
)
assert advertised == {model: ("priority",)}, advertised
sync_client: Final = openai.OpenAI(
api_key=key, base_url=_v1(gateway), max_retries=0, http_client=httpx.Client(timeout=15, trust_env=False)
)
responses_id: Final = sync_client.responses.create(
model=model, input=f"{marker} responses probe", service_tier="priority"
).id
chat_id: Final = sync_client.chat.completions.create(
model=model, messages=[{"role": "user", "content": f"{marker} chat probe"}], service_tier="priority"
).id
async def streamed() -> str:
client: Final = openai.AsyncOpenAI(
api_key=key,
base_url=_v1(gateway),
max_retries=0,
http_client=httpx.AsyncClient(timeout=15, trust_env=False),
)
stream: Final = await client.chat.completions.create(
model=model,
messages=[{"role": "user", "content": f"{marker} stream probe"}],
service_tier="priority",
stream=True,
)
chunks: Final = [chunk async for chunk in stream]
assert chunks, "no chunks streamed"
return chunks[0].id
stream_id: Final = asyncio.run(streamed())
messages_id: Final = (
anthropic.Anthropic(
api_key=key,
base_url=str(gateway.client.base_url).rstrip("/"),
max_retries=0,
http_client=httpx.Client(timeout=15, trust_env=False),
)
.messages.create(
model=model, max_tokens=64, messages=[{"role": "user", "content": f"{marker} messages probe"}]
)
.id
)
received: Final = _llm_requests(wire)
assert len(received) == 4, [request.target for request in received]
targets: Final = sorted(request.target for request in received)
assert targets == ["/v1/chat/completions"] * 2 + ["/v1/responses"] * 2, targets
openai_shaped: Final = [
body for body in _request_bodies(received) if f"{marker} messages probe" not in _prompt_of(body)
]
assert len(openai_shaped) == 3, _request_bodies(received)
assert all(body["service_tier"] == "priority" for body in openai_shaped), openai_shaped
ids: Final = sorted((responses_id, chat_id, stream_id, messages_id))
assert len(set(ids)) == 4, ids
landed: Final = eventually(lambda: _spend_ids_for_key(key), lambda values: len(values) == 4, seconds=70)
assert sorted(landed) == ids, (landed, ids)
BURST_CATALOG: Final = 12
BURST_CHAT: Final = 8
BURST_STREAM: Final = 8
BURST_RESPONSES: Final = 4
BURST_MESSAGES: Final = 4
BURST_SIZES: Final = (
("chat", BURST_CHAT),
("stream", BURST_STREAM),
("responses", BURST_RESPONSES),
("messages", BURST_MESSAGES),
)
def _llm_tasks() -> tuple[tuple[str, int], ...]:
return tuple(
(kind, index) for kind, size in BURST_SIZES for index in range(size)
) # comprehension-ok: a flat task list of two small axes
def _stream_id(response: httpx.Response) -> str:
assert response.status_code == 200, response.text
events: Final = [
object_value(json.loads(line.removeprefix("data:")))
for line in response.iter_lines()
if line.startswith("data:") and line.removeprefix("data:").strip() != "[DONE]"
]
assert events, response.text
return string_value(events[0]["id"])
def _json_id(response: httpx.Response) -> str:
assert response.status_code == 200, response.text
return string_value(JSON_OBJECT.validate_json(response.content)["id"])
def test_catalog_burst_with_model_churn(gateway: Gateway) -> None:
with wire_server(_upstream) as wire, gateway.scenario() as scenario:
written_at: Final = time.monotonic()
model: Final = scenario.model(api_base=f"{wire.url}/v1", model_info={"service_tiers": ["priority"]})
key: Final = scenario.key(models=[model])
marker: Final = uuid.uuid4().hex
_settled_on_every_worker(
lambda: _tiers_by_slug(gateway, key=key),
lambda tiers: tiers == {model: ("priority",)},
written_at=written_at,
)
def prompt(kind: str, index: int) -> str:
return f"{marker} {kind} {index}"
def catalog(index: int) -> tuple[str, str]:
tiers: Final = _tiers_by_slug(gateway, key=key)
assert tiers == {model: ("priority",)}, (index, tiers)
return ("catalog", "")
def chat(index: int) -> tuple[str, str]:
body: Final = {"model": model, "messages": [{"role": "user", "content": prompt("chat", index)}]}
return ("chat", _json_id(gateway.request("POST", "/v1/chat/completions", body, key=key)))
def stream(index: int) -> tuple[str, str]:
body: Final = {
"model": model,
"messages": [{"role": "user", "content": prompt("stream", index)}],
"stream": True,
}
return ("stream", _stream_id(gateway.request("POST", "/v1/chat/completions", body, key=key)))
def responses(index: int) -> tuple[str, str]:
body: Final = {"model": model, "input": prompt("responses", index), "service_tier": "priority"}
return ("responses", _json_id(gateway.request("POST", "/v1/responses", body, key=key)))
def messages(index: int) -> tuple[str, str]:
body: Final = {
"model": model,
"max_tokens": 64,
"messages": [{"role": "user", "content": prompt("messages", index)}],
}
return ("messages", _json_id(gateway.request("POST", "/v1/messages", body, key=key)))
def churn(index: int) -> tuple[str, str]:
churned: Final = _new_model(
scenario, _model_name(f"codex-churn-{index}"), model_info={"service_tiers": ["flex"]}
)
gateway.post("/model/delete", {"id": churned})
return ("churn", churned)
tasks: Final[tuple[tuple[Callable[[int], tuple[str, str]], int], ...]] = (
*((catalog, index) for index in range(BURST_CATALOG)),
*((chat, index) for index in range(BURST_CHAT)),
*((stream, index) for index in range(BURST_STREAM)),
*((responses, index) for index in range(BURST_RESPONSES)),
*((messages, index) for index in range(BURST_MESSAGES)),
(churn, 0),
(churn, 1),
)
with ThreadPoolExecutor(max_workers=len(tasks)) as pool:
outcomes: Final = tuple(pool.map(lambda task: task[0](task[1]), tasks))
llm_kinds: Final = frozenset({"chat", "stream", "responses", "messages"})
llm_ids: Final = sorted(identity for kind, identity in outcomes if kind in llm_kinds)
assert len(llm_ids) == BURST_CHAT + BURST_STREAM + BURST_RESPONSES + BURST_MESSAGES, outcomes
assert len(set(llm_ids)) == len(llm_ids), llm_ids
received: Final = _request_bodies(_llm_requests(wire))
assert len(received) == len(llm_ids), len(received)
prompts_sent: Final = sorted(prompt(kind, index) for kind, index in _llm_tasks())
prompts_seen: Final = sorted(_sent_prompt(body, prompts_sent) for body in received)
assert prompts_seen == prompts_sent, (prompts_seen, prompts_sent)
landed: Final = eventually(
lambda: _spend_ids_for_key(key), lambda values: len(values) == len(llm_ids), seconds=70
)
assert sorted(landed) == llm_ids, (landed, llm_ids)
final_tiers: Final = _settled(
lambda: _tiers_by_slug(gateway, key=key), lambda value: value == {model: ("priority",)}
)
assert final_tiers == {model: ("priority",)}, final_tiers

View file

@ -0,0 +1,430 @@
import json
import os
import signal
import stat
import subprocess
import sys
import uuid
from collections.abc import Mapping, Sequence
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import Final
import httpx
import psutil
import pytest
from pydantic import JsonValue
from tests.integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, object_value, string_value
from tests.integration._support.process import group_members, owned_proxy, owned_proxy_process
CLIENT_VERSION: Final = "0.159.3"
FRESH_CONNECTION: Final = {"Connection": "close"}
CONSISTENT_READS: Final = 8
CONVERGENCE_SECONDS: Final = 60
BURST: Final = 24
CATALOG_FILENAME: Final = "litellm-models.json"
SHIM_STOCK_ENV: Final = "CODEX_SHIM_STOCK"
SHIM_LAUNCH_ENV: Final = "CODEX_SHIM_LAUNCH"
SHIM_SCRIPT: Final = """\
import json
import os
import sys
argv = sys.argv[1:]
if argv[-2:] == ["debug", "models"]:
override = next((arg for arg in argv if arg.startswith("model_catalog_json=")), None)
if override is None:
sys.stdout.write(open(os.environ[%(stock)r], encoding="utf-8").read())
else:
catalog = json.load(open(json.loads(override.partition("=")[2]), encoding="utf-8"))
if not catalog.get("models"):
sys.stderr.write("empty model catalog\\n")
sys.exit(2)
sys.stdout.write(json.dumps(catalog))
sys.exit(0)
recorded = {name: os.environ.get(name) for name in ("OPENAI_BASE_URL", "OPENAI_API_KEY", "LITELLM_PROXY_API_KEY")}
with open(os.environ[%(launch)r], "w", encoding="utf-8") as handle:
json.dump({"argv": argv, "env": recorded}, handle)
""" % {"stock": SHIM_STOCK_ENV, "launch": SHIM_LAUNCH_ENV}
def _generic_tier(identity: str) -> dict[str, JsonValue]:
return {"id": identity, "name": identity.capitalize(), "description": f"Sends service_tier={identity} upstream"}
def _catalog_response(proxy: Gateway, *, key: str | None = None) -> httpx.Response:
return proxy.request(
"GET", "/v1/models", key=key, params={"client_version": CLIENT_VERSION}, headers=FRESH_CONNECTION
)
def _entries(response: httpx.Response) -> list[dict[str, JsonValue]]:
assert response.status_code == 200, response.text
body: Final = JSON_OBJECT.validate_json(response.content)
assert set(body) == {"models"}, response.text
models: Final = body["models"]
assert isinstance(models, list), response.text
return [object_value(entry) for entry in models]
def _by_slug(entries: Sequence[Mapping[str, JsonValue]]) -> dict[str, dict[str, JsonValue]]:
return {string_value(entry["slug"]): dict(entry) for entry in entries}
def _write_config(directory: Path, upstream_url: str, tiered: str, plain: str, alias: str) -> Path:
config: Final = directory / f"codex_catalog_{uuid.uuid4().hex}.yaml"
params: Final = {
"model": "openai/gpt-4o-mini",
"api_base": f"{upstream_url}/v1",
"api_key": "integration-provider-key",
}
config.write_text(
json.dumps(
{
"model_list": [
{
"model_name": tiered,
"litellm_params": params,
"model_info": {
"service_tiers": ["priority", {"id": "flex", "name": "Flex", "description": "cheap lane"}],
"display_name": "YAML Tiered",
"max_input_tokens": 4321,
},
},
{"model_name": plain, "litellm_params": params},
],
"router_settings": {"model_group_alias": {alias: tiered}},
"general_settings": {
"master_key": "os.environ/LITELLM_MASTER_KEY",
"database_url": "os.environ/DATABASE_URL",
},
}
)
)
return config
@pytest.mark.timeout(240)
def test_yaml_tiers_and_alias_in_owned_proxy(gateway: Gateway, tmp_path: Path) -> None:
tiered: Final = f"codex-yaml-tiered-{uuid.uuid4().hex}"
plain: Final = f"codex-yaml-plain-{uuid.uuid4().hex}"
alias: Final = f"codex-yaml-alias-{uuid.uuid4().hex}"
config: Final = _write_config(tmp_path, gateway.upstream_url, tiered, plain, alias)
with owned_proxy(gateway, tmp_path, {"STORE_MODEL_IN_DB": "False"}, config=config, workers=2) as candidate:
readings: Final = tuple(_entries(_catalog_response(candidate)) for _ in range(CONSISTENT_READS))
assert all(reading == readings[0] for reading in readings), readings
catalog: Final = _by_slug(readings[0])
assert sorted(catalog) == sorted((tiered, plain, alias)), sorted(catalog)
assert catalog[tiered]["service_tiers"] == [
_generic_tier("priority"),
{"id": "flex", "name": "Flex", "description": "cheap lane"},
], catalog[tiered]
assert catalog[tiered]["display_name"] == "YAML Tiered", catalog[tiered]
assert catalog[tiered]["context_window"] == 4321, catalog[tiered]
assert catalog[plain]["service_tiers"] == [] and catalog[plain]["display_name"] == plain, catalog[plain]
assert catalog[alias]["service_tiers"] == catalog[tiered]["service_tiers"], catalog[alias]
assert catalog[alias]["display_name"] == alias, catalog[alias]
assert catalog[alias]["context_window"] == 4321, catalog[alias]
assert string_value(catalog[alias]["base_instructions"]), alias
ROOT: Final = Path(os.environ.get("INTEGRATION_PROXY_ROOT") or Path(__file__).resolve().parents[3])
BUNDLED_STOCK_PATH: Final = ROOT / "litellm" / "proxy" / "common_utils" / "codex_bundled_models_0.159.3.json"
STOCK_UPSTREAM: Final = "gpt-5.5"
def _bundled_stock_entry(slug: str) -> dict[str, JsonValue]:
models: Final = JSON_OBJECT.validate_json(BUNDLED_STOCK_PATH.read_bytes())["models"]
assert isinstance(models, list), slug
(entry,) = (object_value(model) for model in models if object_value(model)["slug"] == slug)
return entry
def _write_team_owned_config(
directory: Path, upstream_url: str, stock_team: str, plain_team: str, owned: str, mixed: str, alias: str
) -> Path:
config: Final = directory / f"codex_catalog_teams_{uuid.uuid4().hex}.yaml"
plain_params: Final = {
"model": "openai/gpt-4o-mini",
"api_base": f"{upstream_url}/v1",
"api_key": "integration-provider-key",
}
stock_params: Final = {**plain_params, "model": f"openai/{STOCK_UPSTREAM}"}
config.write_text(
json.dumps(
{
"model_list": [
{"model_name": owned, "litellm_params": stock_params, "model_info": {"team_id": stock_team}},
{
"model_name": owned,
"litellm_params": plain_params,
"model_info": {"team_id": plain_team, "service_tiers": ["flex"]},
},
{"model_name": mixed, "litellm_params": stock_params, "model_info": {"team_id": stock_team}},
{"model_name": mixed, "litellm_params": plain_params, "model_info": {"service_tiers": ["flex"]}},
],
"router_settings": {"model_group_alias": {alias: owned}},
"general_settings": {
"master_key": "os.environ/LITELLM_MASTER_KEY",
"database_url": "os.environ/DATABASE_URL",
},
}
)
)
return config
@pytest.mark.timeout(240)
def test_team_owned_deployments_of_one_name_serve_each_team_its_own_upstream(gateway: Gateway, tmp_path: Path) -> None:
"""Two teams own a deployment of one model name (and of a `model_group_alias` of it): Codex's stock
entry for the first deployment's upstream goes only to the team whose requests reach it, and a
caller outside the owning team reads the deployment no team owns."""
owned: Final = f"codex-team-owned-{uuid.uuid4().hex}"
mixed: Final = f"codex-team-mixed-{uuid.uuid4().hex}"
alias: Final = f"codex-team-alias-{uuid.uuid4().hex}"
stock: Final = _bundled_stock_entry(STOCK_UPSTREAM)
assert stock["supported_reasoning_levels"] != [] and stock["service_tiers"] != [], stock
with gateway.scenario() as scenario:
stock_team: Final = scenario.team()
plain_team: Final = scenario.team()
stock_key: Final = scenario.key(team_id=stock_team, models=[owned, alias, mixed])
plain_key: Final = scenario.key(team_id=plain_team, models=[owned, alias, mixed])
teamless_key: Final = scenario.key(models=[mixed])
config: Final = _write_team_owned_config(
tmp_path, gateway.upstream_url, stock_team, plain_team, owned, mixed, alias
)
with owned_proxy(gateway, tmp_path, {"STORE_MODEL_IN_DB": "False"}, config=config, workers=2) as candidate:
def stable_catalog(key: str) -> dict[str, dict[str, JsonValue]]:
readings: Final = tuple(
_entries(_catalog_response(candidate, key=key)) for _ in range(CONSISTENT_READS)
)
assert all(reading == readings[0] for reading in readings), readings
return _by_slug(readings[0])
for_stock_team: Final = stable_catalog(stock_key)
for_plain_team: Final = stable_catalog(plain_key)
for_teamless: Final = stable_catalog(teamless_key)
assert sorted(for_stock_team) == sorted(for_plain_team) == sorted((owned, alias, mixed)), sorted(for_stock_team)
assert sorted(for_teamless) == [mixed], sorted(for_teamless)
for slug in (owned, alias):
assert for_stock_team[slug]["supported_reasoning_levels"] == stock["supported_reasoning_levels"], slug
assert for_stock_team[slug]["model_messages"] == stock["model_messages"], slug
assert for_stock_team[slug]["service_tiers"] == stock["service_tiers"], slug
assert for_plain_team[slug]["supported_reasoning_levels"] == [], for_plain_team[slug]
assert "model_messages" not in for_plain_team[slug], slug
assert string_value(for_plain_team[slug]["base_instructions"]), slug
assert for_plain_team[slug]["service_tiers"] == [_generic_tier("flex")], for_plain_team[slug]
assert for_stock_team[mixed]["supported_reasoning_levels"] == stock["supported_reasoning_levels"], mixed
assert for_stock_team[mixed]["service_tiers"] == [], for_stock_team[mixed]
for outsider in (for_plain_team, for_teamless):
assert outsider[mixed]["supported_reasoning_levels"] == [], outsider[mixed]
assert "model_messages" not in outsider[mixed], mixed
assert outsider[mixed]["service_tiers"] == [_generic_tier("flex")], outsider[mixed]
def _worker_pids(root_pid: int) -> frozenset[int]:
def is_worker(process: psutil.Process) -> bool:
try:
return "spawn_main" in " ".join(process.cmdline())
except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
return False
return frozenset(process.pid for process in group_members(root_pid) if is_worker(process))
def _catalog_outcome(proxy: Gateway, slugs: Sequence[JsonValue]) -> str:
try:
response: Final = _catalog_response(proxy)
except httpx.TransportError as error:
return f"transport:{type(error).__name__}"
assert response.status_code == 200, response.text
assert sorted(entry["slug"] for entry in _entries(response)) == list(slugs), response.text
return "ok"
@pytest.mark.timeout(300)
def test_catalog_survives_a_worker_kill(gateway: Gateway, tmp_path: Path) -> None:
tiered: Final = f"codex-yaml-tiered-{uuid.uuid4().hex}"
plain: Final = f"codex-yaml-plain-{uuid.uuid4().hex}"
alias: Final = f"codex-yaml-alias-{uuid.uuid4().hex}"
config: Final = _write_config(tmp_path, gateway.upstream_url, tiered, plain, alias)
with owned_proxy_process(gateway, tmp_path, {"STORE_MODEL_IN_DB": "False"}, config=config, workers=2) as owned:
candidate: Final = owned.gateway
workers: Final = eventually(lambda: _worker_pids(owned.process.pid), lambda pids: len(pids) == 2, seconds=30)
victim: Final = min(workers)
slugs: Final = sorted(entry["slug"] for entry in _entries(_catalog_response(candidate)))
assert slugs == sorted((tiered, plain, alias)), slugs
def attempt(index: int) -> str:
if index == 2:
os.kill(victim, signal.SIGKILL)
return _catalog_outcome(candidate, slugs)
with ThreadPoolExecutor(max_workers=BURST) as pool:
outcomes: Final = tuple(pool.map(attempt, range(BURST)))
assert outcomes.count("ok") >= 1, outcomes
assert all(outcome == "ok" or outcome.startswith("transport:") for outcome in outcomes), outcomes
respawned: Final = eventually(
lambda: _worker_pids(owned.process.pid),
lambda pids: len(pids) == 2 and victim not in pids,
seconds=60,
)
assert victim not in respawned, respawned
def settled_burst() -> tuple[str, ...]:
with ThreadPoolExecutor(max_workers=BURST) as pool:
return tuple(pool.map(lambda _: _catalog_outcome(candidate, slugs), range(BURST)))
final: Final = eventually(settled_burst, lambda values: all(value == "ok" for value in values), seconds=40)
assert final == ("ok",) * BURST, final
def _install_shim(directory: Path, stock: Mapping[str, JsonValue]) -> tuple[Path, dict[str, str]]:
shim_dir: Final = directory / "bin"
shim_dir.mkdir()
script: Final = directory / "codex_shim.py"
script.write_text(SHIM_SCRIPT, encoding="utf-8")
shim: Final = shim_dir / "codex"
shim.write_text(f'#!/bin/sh\nexec "{sys.executable}" "{script}" "$@"\n', encoding="utf-8")
shim.chmod(shim.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
stock_path: Final = directory / "stock.json"
stock_path.write_text(json.dumps(stock), encoding="utf-8")
launch_path: Final = directory / "launch.json"
home: Final = directory / "home"
home.mkdir()
codex_home: Final = directory / "codex-home"
environment: Final = {
**os.environ,
"HOME": str(home),
"CODEX_HOME": str(codex_home),
"PATH": f"{shim_dir}{os.pathsep}{os.environ['PATH']}",
SHIM_STOCK_ENV: str(stock_path),
SHIM_LAUNCH_ENV: str(launch_path),
}
return launch_path, environment
def _run_lite_codex(proxy: Gateway, key: str, environment: Mapping[str, str]) -> subprocess.CompletedProcess[str]:
return subprocess.run(
[sys.executable, "-I", "-c", "from litellm.proxy.client.cli import cli; cli()", "codex", "--shim-marker"],
env={**environment, "LITELLM_PROXY_URL": str(proxy.client.base_url).rstrip("/"), "LITELLM_PROXY_API_KEY": key},
stdin=subprocess.DEVNULL,
capture_output=True,
text=True,
timeout=90,
check=False,
)
def _new_model(scenario: Scenario, name: str, model_info: Mapping[str, JsonValue]) -> None:
created: Final = scenario.gateway.post(
"/model/new",
{
"model_name": name,
"litellm_params": {
"model": "openai/gpt-4o-mini",
"api_key": "integration-provider-key",
"api_base": f"{scenario.gateway.upstream_url}/v1",
},
"model_info": dict(model_info),
},
)
scenario.cleanups.callback(scenario.delete_model, string_value(object_value(created["model_info"])["id"]))
def _launch_record(path: Path) -> dict[str, JsonValue]:
assert path.exists(), "the shim was never launched"
return JSON_OBJECT.validate_json(path.read_bytes())
def _catalog_override(argv: Sequence[JsonValue]) -> str | None:
overrides: Final = [
string_value(arg) for arg in argv if isinstance(arg, str) and arg.startswith("model_catalog_json=")
]
assert len(overrides) <= 1, argv
return json.loads(overrides[0].partition("=")[2]) if overrides else None
def test_lite_codex_writes_the_shared_catalog(gateway: Gateway, tmp_path: Path) -> None:
"""`lite codex` writes the installed Codex's own entry for a proxy model whose name is a stock slug and
the fallback entry for any other; the proxy's configured tiers never reach the file (Decision 17)."""
stock_slug: Final = f"codex-stock-{uuid.uuid4().hex}"
stock_tiers: Final = [{"id": "priority", "name": "Fast", "description": "1.5x speed"}]
stock: Final = {
"models": [
{
"slug": stock_slug,
"display_name": "Stock Model",
"priority": 7,
"visibility": "hide",
"supported_in_api": False,
"service_tiers": stock_tiers,
"default_service_tier": None,
"supported_reasoning_levels": [{"effort": "high", "description": "thinks longer"}],
}
]
}
launch_path, environment = _install_shim(tmp_path, stock)
with gateway.scenario() as scenario:
_new_model(scenario, stock_slug, {"service_tiers": ["ultrafast"]})
fallback: Final = scenario.model(model_info={"service_tiers": ["priority"]})
key: Final = scenario.key(models=[stock_slug, fallback])
completed: Final = _run_lite_codex(gateway, key, environment)
assert completed.returncode == 0, (completed.stdout, completed.stderr)
assert "not syncing" not in completed.stderr, completed.stderr
record: Final = _launch_record(launch_path)
argv: Final = record["argv"]
assert isinstance(argv, list) and argv[-1] == "--shim-marker", argv
catalog_path: Final = Path(environment["CODEX_HOME"]) / CATALOG_FILENAME
assert _catalog_override(argv) == str(catalog_path), argv
env: Final = object_value(record["env"])
assert env["OPENAI_BASE_URL"] == str(gateway.client.base_url).rstrip("/") + "/v1", env
assert env["OPENAI_API_KEY"] == key, env.keys()
written: Final = JSON_OBJECT.validate_json(catalog_path.read_bytes())
assert set(written) == {"models"}, written.keys()
models: Final = written["models"]
assert isinstance(models, list), written
file_entries: Final = _by_slug([object_value(entry) for entry in models])
served: Final = eventually(
lambda: _by_slug(_entries(_catalog_response(gateway, key=key))),
lambda value: (
stock_slug in value
and fallback in value
and value[stock_slug]["service_tiers"] == [_generic_tier("ultrafast")]
and value[fallback]["service_tiers"] == [_generic_tier("priority")]
),
seconds=CONVERGENCE_SECONDS,
)
assert list(file_entries) == list(served), (list(file_entries), list(served))
assert file_entries[stock_slug]["service_tiers"] == stock_tiers, file_entries[stock_slug]
assert file_entries[stock_slug]["visibility"] == "list", file_entries[stock_slug]
assert file_entries[stock_slug]["supported_in_api"] is True, file_entries[stock_slug]
assert file_entries[stock_slug]["priority"] == list(file_entries).index(stock_slug), file_entries[stock_slug]
assert file_entries[stock_slug]["supported_reasoning_levels"] == [
{"effort": "high", "description": "thinks longer"}
]
assert file_entries[fallback]["service_tiers"] == [], file_entries[fallback]
assert file_entries[fallback]["display_name"] == fallback, file_entries[fallback]
assert string_value(file_entries[fallback]["base_instructions"]), fallback
assert served[stock_slug]["service_tiers"] == [_generic_tier("ultrafast")], served[stock_slug]
assert served[fallback]["service_tiers"] == [_generic_tier("priority")], served[fallback]
def test_lite_codex_skips_a_stock_dump_without_display_name(gateway: Gateway, tmp_path: Path) -> None:
"""A `codex debug models` dump that predates `display_name` is reported, not written, and Codex still
launches without a catalog override."""
stock: Final = {"models": [{"slug": "gpt-5.5", "priority": 0, "visibility": "list"}]}
launch_path, environment = _install_shim(tmp_path, stock)
with gateway.scenario() as scenario:
model: Final = scenario.model(model_info={"service_tiers": ["priority"]})
key: Final = scenario.key(models=[model])
completed: Final = _run_lite_codex(gateway, key, environment)
assert completed.returncode == 0, (completed.stdout, completed.stderr)
assert "printed no model catalog" in completed.stderr, completed.stderr
record: Final = _launch_record(launch_path)
argv: Final = record["argv"]
assert isinstance(argv, list) and argv[-1] == "--shim-marker", argv
assert _catalog_override(argv) is None, argv
assert not (Path(environment["CODEX_HOME"]) / CATALOG_FILENAME).exists(), environment["CODEX_HOME"]

View file

@ -0,0 +1,128 @@
from __future__ import annotations
import uuid
from typing import Final
import httpx
import pytest
from integration._support.client import Gateway
from pydantic import JsonValue
from litellm.harness.context import GatewayTarget
from litellm.harness.endpoint import ModelEndpoint
from litellm.harness.types import Harness
def _register_scenario(gateway: Gateway, scenario_id: str, response: dict[str, JsonValue]) -> None:
with httpx.Client(timeout=5, trust_env=False) as client:
result: Final = client.post(
f"{gateway.upstream_url}/__scenarios",
json={"scenario_id": scenario_id, "response": response},
)
assert result.status_code == 200, result.text
def _client(endpoint: ModelEndpoint) -> httpx.AsyncClient:
return httpx.AsyncClient(
base_url=endpoint.url,
headers={"Authorization": f"Bearer {endpoint.token}"},
trust_env=False,
timeout=15,
)
_CHAT_BODY: Final = {
"id": "chatcmpl-$UNIQUE_ID",
"object": "chat.completion",
"created": 1,
"model": "scripted-model",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 11, "completion_tokens": 7, "total_tokens": 18},
}
_RESPONSES_BODY: Final = {
"id": "resp_$UNIQUE_ID",
"object": "response",
"status": "completed",
"model": "scripted-model",
"output": [{"type": "message", "content": [{"type": "output_text", "text": "hi"}]}],
"usage": {"input_tokens": 5, "output_tokens": 3, "total_tokens": 8},
}
async def test_endpoint_forwards_json_object_bodies(gateway: Gateway) -> None:
scenario_id: Final = f"audithend{uuid.uuid4().hex[:8]}"
_register_scenario(
gateway,
scenario_id,
{
"content_type": "application/x-routed",
"routes": {
"POST /v1/chat/completions": {"content_type": "application/json", "body": _CHAT_BODY},
"POST /v1/responses": {"content_type": "application/json", "body": _RESPONSES_BODY},
},
},
)
target: Final = GatewayTarget(api_base=f"{gateway.upstream_url}/{scenario_id}", api_key="scripted")
async with ModelEndpoint(Harness.CODEX, "audit-model", target, api_key="x") as endpoint:
assert isinstance(endpoint.port, int) and endpoint.port > 0
async with _client(endpoint) as client:
chat: Final = await client.post("/v1/chat/completions", json={"messages": []})
assert chat.status_code == 200, chat.text
assert chat.json()["object"] == "chat.completion", chat.text
responses: Final = await client.post("/v1/responses", json={"input": "x"})
assert responses.status_code == 200, responses.text
assert responses.json()["object"] == "response", responses.text
usage: Final = endpoint.usage.snapshot()
assert usage.input_tokens == 11 + 5 and usage.output_tokens == 7 + 3, usage
@pytest.mark.parametrize("raw", (b"[1, 2]", b'"a string"', b"42", b"null"))
async def test_endpoint_rejects_valid_json_non_object(gateway: Gateway, raw: bytes) -> None:
async with ModelEndpoint(Harness.CODEX, "audit-model", None, api_key="x") as endpoint:
async with _client(endpoint) as client:
response: Final = await client.post("/v1/chat/completions", content=raw)
assert response.status_code == 400, response.text
assert "must be a JSON object" in response.text, response.text
async def test_endpoint_rejects_unparseable_body(gateway: Gateway) -> None:
async with ModelEndpoint(Harness.CODEX, "audit-model", None, api_key="x") as endpoint:
async with _client(endpoint) as client:
response: Final = await client.post("/v1/chat/completions", content=b"this is not json{")
assert response.status_code == 400, response.text
async def test_endpoint_records_usage_from_sse_stream(gateway: Gateway) -> None:
scenario_id: Final = f"audithend{uuid.uuid4().hex[:8]}"
_register_scenario(
gateway,
scenario_id,
{
"content_type": "text/event-stream",
"frames": (
'data: {"id":"chatcmpl-$UNIQUE_ID","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":"hi"},"finish_reason":null}]}',
"data: 42",
'data: {"id":"chatcmpl-$UNIQUE_ID","object":"chat.completion.chunk","choices":[],"usage":{"prompt_tokens":13,"completion_tokens":4}}',
"data: [DONE]",
),
},
)
target: Final = GatewayTarget(api_base=f"{gateway.upstream_url}/{scenario_id}", api_key="scripted")
async with ModelEndpoint(Harness.CODEX, "audit-model", target, api_key="x") as endpoint:
async with _client(endpoint) as client:
response: Final = await client.post("/v1/chat/completions", json={"messages": []})
assert response.status_code == 200, response.text
assert "[DONE]" in response.text, response.text
usage: Final = endpoint.usage.snapshot()
assert usage.input_tokens == 13 and usage.output_tokens == 4, usage
async def test_endpoint_binds_free_port() -> None:
async with ModelEndpoint(Harness.CODEX, "audit-model", None) as endpoint:
port: Final = endpoint.port
assert isinstance(port, int) and port > 0
async with _client(endpoint) as client:
response: Final = await client.get("/v1/models")
assert response.status_code == 200, response.text
assert response.json()["object"] == "list", response.text

View file

@ -0,0 +1,346 @@
from __future__ import annotations
import json
import uuid
from collections.abc import Iterator
from contextlib import ExitStack
from dataclasses import dataclass
from pathlib import Path
from typing import Final
import httpx
import openai
import pytest
import yaml
from integration._support.client import Gateway, gateway_from_environment, object_value
from integration._support.process import owned_proxy
from integration.authorization._guardrail_opt_out import upstream_observations
from pydantic import JsonValue
CONFIG_STORE_ID: Final = "vs_integration_config_store"
FAIL_STORE_ID: Final = "vs_audit_fail_store"
PROXY_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml"
def _marker() -> str:
return uuid.uuid4().hex
def _chat_body(model: str, marker: str, store_id: str, *, stream: bool = False) -> dict[str, JsonValue]:
return {
"model": model,
"messages": [{"role": "user", "content": marker}],
"vector_store_ids": [store_id],
"stream": stream,
**({"stream_options": {"include_usage": True}} if stream else {}),
}
def _sdk_chat_body(body: dict[str, JsonValue]) -> dict[str, JsonValue]:
return {
**{key: value for key, value in body.items() if key != "vector_store_ids"},
"extra_body": {"vector_store_ids": body["vector_store_ids"]},
}
def _search_observations(gateway: Gateway, marker: str, store_id: str) -> tuple[dict[str, JsonValue], ...]:
path: Final = f"/vector_stores/{store_id}/search"
return tuple(
observation
for observation in upstream_observations(gateway)
if observation["path"] == path and marker in json.dumps(observation["body"])
)
def _chat_observations(gateway: Gateway, marker: str) -> tuple[dict[str, JsonValue], ...]:
return tuple(
observation
for observation in upstream_observations(gateway)
if observation["path"] == "/v1/chat/completions" and marker in json.dumps(observation["body"])
)
def _message_provider_fields(payload: dict[str, JsonValue]) -> dict[str, JsonValue]:
choices: Final = payload["choices"]
assert isinstance(choices, list) and choices, payload
message: Final = object_value(object_value(choices[0])["message"])
fields: Final = message.get("provider_specific_fields")
assert isinstance(fields, dict), payload
return object_value(fields)
def _sse_data_lines(text: str) -> tuple[dict[str, JsonValue], ...]:
parsed: list[dict[str, JsonValue]] = []
for line in text.splitlines():
if not line.startswith("data: ") or line == "data: [DONE]":
continue
value = json.loads(line.removeprefix("data: "))
if isinstance(value, dict):
parsed.append(value)
return tuple(parsed)
def _chunk_annotated(chunk: dict[str, JsonValue]) -> bool:
choices: Final = chunk.get("choices")
if not isinstance(choices, list) or not choices:
return False
delta: Final = object_value(choices[0]).get("delta")
if not isinstance(delta, dict):
return False
fields: Final = object_value(delta).get("provider_specific_fields")
return "search_results" in json.dumps(fields)
def _openai_client(gateway: Gateway, key: str) -> openai.OpenAI:
return openai.OpenAI(
base_url=f"{gateway.client.base_url}/v1",
api_key=key,
http_client=httpx.Client(trust_env=False),
)
def _openai_async_client(gateway: Gateway, key: str) -> openai.AsyncOpenAI:
return openai.AsyncOpenAI(
base_url=f"{gateway.client.base_url}/v1",
api_key=key,
http_client=httpx.AsyncClient(trust_env=False),
)
def _register_scenario(gateway: Gateway, scenario_id: str, response: dict[str, JsonValue]) -> None:
with httpx.Client(timeout=5, trust_env=False) as client:
result: Final = client.post(
f"{gateway.upstream_url}/__scenarios",
json={"scenario_id": scenario_id, "response": response},
)
assert result.status_code == 200, result.text
def test_vector_store_hook_chat_completions_raw_httpx(gateway: Gateway) -> None:
marker: Final = _marker()
with gateway.scenario() as scenario:
model: Final = scenario.model()
key: Final = scenario.key(models=[model])
response: Final = gateway.request(
"POST", "/v1/chat/completions", _chat_body(model, marker, CONFIG_STORE_ID), key=key
)
assert response.status_code == 200, response.text
assert _message_provider_fields(response.json())["search_results"]
assert len(_search_observations(gateway, marker, CONFIG_STORE_ID)) == 1
def test_vector_store_hook_chat_completions_raw_httpx_stream(gateway: Gateway) -> None:
marker: Final = _marker()
with gateway.scenario() as scenario:
model: Final = scenario.model()
key: Final = scenario.key(models=[model])
response: Final = gateway.request(
"POST", "/v1/chat/completions", _chat_body(model, marker, CONFIG_STORE_ID, stream=True), key=key
)
assert response.status_code == 200, response.text
chunks: Final = _sse_data_lines(response.text)
assert any(_chunk_annotated(chunk) for chunk in chunks), response.text
assert len(_search_observations(gateway, marker, CONFIG_STORE_ID)) == 1
@pytest.mark.parametrize("stream", (False, True))
def test_vector_store_hook_chat_completions_openai_sdk(gateway: Gateway, stream: bool) -> None:
marker: Final = _marker()
with gateway.scenario() as scenario:
model: Final = scenario.model()
key: Final = scenario.key(models=[model])
client: Final = _openai_client(gateway, key)
body: Final = _chat_body(model, marker, CONFIG_STORE_ID, stream=stream)
if stream:
with client.chat.completions.stream(
**{k: v for k, v in _sdk_chat_body(body).items() if k != "stream"}
) as event_stream:
for _event in event_stream:
pass
snapshot: Final = event_stream.get_final_completion().model_dump(mode="json")
assert _message_provider_fields(snapshot)["search_results"], snapshot
else:
completion: Final = client.chat.completions.create(**_sdk_chat_body(body))
assert _message_provider_fields(completion.model_dump(mode="json"))["search_results"]
assert len(_search_observations(gateway, marker, CONFIG_STORE_ID)) == 1
@pytest.mark.parametrize("stream", (False, True))
async def test_vector_store_hook_chat_completions_openai_async_sdk(gateway: Gateway, stream: bool) -> None:
marker: Final = _marker()
with gateway.scenario() as scenario:
model: Final = scenario.model()
key: Final = scenario.key(models=[model])
client: Final = _openai_async_client(gateway, key)
body: Final = _chat_body(model, marker, CONFIG_STORE_ID, stream=stream)
if stream:
stream_response: Final = await client.chat.completions.create(**_sdk_chat_body(body))
chunks: list[dict[str, JsonValue]] = []
async for chunk in stream_response:
chunks.append(chunk.model_dump(mode="json"))
assert any(_chunk_annotated(chunk) for chunk in chunks), chunks
else:
completion: Final = await client.chat.completions.create(**_sdk_chat_body(body))
assert _message_provider_fields(completion.model_dump(mode="json"))["search_results"]
assert len(_search_observations(gateway, marker, CONFIG_STORE_ID)) == 1
@pytest.mark.parametrize("stream", (False, True))
def test_vector_store_hook_responses_api(gateway: Gateway, stream: bool) -> None:
marker: Final = _marker()
scenario_id: Final = "auditresp" if not stream else "auditrespstream"
if stream:
_register_scenario(
gateway,
scenario_id,
{
"content_type": "text/event-stream",
"frames": (
'event: response.created\ndata: {"type":"response.created","response":{"id":"$UNIQUE_ID","object":"response","status":"in_progress"}}',
'event: response.completed\ndata: {"type":"response.completed","response":{"id":"$UNIQUE_ID","object":"response","status":"completed","output":[{"type":"message","content":[{"type":"output_text","text":"done"}]}],"usage":{"input_tokens":3,"output_tokens":4,"total_tokens":7}}}',
),
},
)
else:
_register_scenario(
gateway,
scenario_id,
{
"content_type": "application/x-routed",
"routes": {
"POST /responses": {
"content_type": "application/json",
"body": {
"id": "resp_$UNIQUE_ID",
"object": "response",
"created_at": 1700000000,
"status": "completed",
"model": "openai/gpt-4o-mini",
"output": [{"type": "message", "content": [{"type": "output_text", "text": "done"}]}],
"usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7},
},
}
},
},
)
with gateway.scenario() as scenario:
model: Final = scenario.model(api_base=f"{gateway.upstream_url}/{scenario_id}")
key: Final = scenario.key(models=[model])
response: Final = gateway.request(
"POST",
"/v1/responses",
{
"model": model,
"input": marker,
"vector_store_ids": [CONFIG_STORE_ID],
"stream": stream,
},
key=key,
)
assert response.status_code == 200, response.text
if stream:
assert "response.completed" in response.text
else:
payload: Final = response.json()
assert payload["object"] == "response" and payload["id"], payload
assert len(_search_observations(gateway, marker, CONFIG_STORE_ID)) == 1
def test_vector_store_annotation_survives_cache_hit(gateway: Gateway) -> None:
marker: Final = _marker()
with gateway.scenario() as scenario:
model: Final = scenario.model()
key: Final = scenario.key(models=[model])
body: Final = _chat_body(model, marker, CONFIG_STORE_ID)
first: Final = gateway.request("POST", "/v1/chat/completions", body, key=key)
assert first.status_code == 200, first.text
second: Final = gateway.request("POST", "/v1/chat/completions", body, key=key)
assert second.status_code == 200, second.text
assert second.json()["id"] == first.json()["id"]
assert _message_provider_fields(second.json())["search_results"]
assert len(_chat_observations(gateway, marker)) == 1
@dataclass(frozen=True, slots=True)
class FailureModeGateways:
annotate: Gateway
error: Gateway
upstream: Gateway
def _failure_mode_config(directory: Path, mode: str, upstream_url: str) -> Path:
config: Final = yaml.safe_load(PROXY_CONFIG.read_text())
config["litellm_settings"]["vector_store_search_failure_mode"] = mode
config["vector_store_registry"] = [
{
"vector_store_name": "audit-fail-store",
"litellm_params": {
"vector_store_id": FAIL_STORE_ID,
"custom_llm_provider": "openai",
"api_base": f"{upstream_url}/auditnostore",
"api_key": "integration-provider-key",
},
}
]
path: Final = directory / f"proxy_vector_store_{mode}.yaml"
path.write_text(yaml.safe_dump(config))
return path
@pytest.fixture(scope="module")
def failure_gateways(tmp_path_factory: pytest.TempPathFactory) -> Iterator[FailureModeGateways]:
with gateway_from_environment() as upstream_gateway, ExitStack() as stack:
gateways: dict[str, Gateway] = {}
for mode in ("annotate", "error"):
directory: Final = tmp_path_factory.mktemp(f"audit_vs_{mode}")
config: Final = _failure_mode_config(directory, mode, upstream_gateway.upstream_url)
owned: Final = stack.enter_context(owned_proxy(upstream_gateway, directory, {}, config=config, workers=2))
gateways[mode] = owned
yield FailureModeGateways(gateways["annotate"], gateways["error"], upstream_gateway)
def _register_failing_store(gateway: Gateway) -> None:
_register_scenario(
gateway,
"auditnostore",
{
"content_type": "application/x-routed",
"routes": {
f"POST /vector_stores/{FAIL_STORE_ID}/search": {
"content_type": "application/json",
"body": {"error": {"message": "scripted store failure"}},
"status": 500,
}
},
},
)
def test_vector_store_search_failure_annotate_mode(failure_gateways: FailureModeGateways) -> None:
_register_failing_store(failure_gateways.upstream)
marker: Final = _marker()
with failure_gateways.annotate.scenario() as scenario:
model: Final = scenario.model()
key: Final = scenario.key(models=[model])
response: Final = failure_gateways.annotate.request(
"POST", "/v1/chat/completions", _chat_body(model, marker, FAIL_STORE_ID), key=key
)
assert response.status_code == 200, response.text
fields: Final = _message_provider_fields(response.json())
failures: Final = fields.get("vector_store_search_failures")
assert isinstance(failures, list) and failures, response.text
assert FAIL_STORE_ID in json.dumps(failures) and "scripted store failure" in json.dumps(failures), failures
def test_vector_store_search_failure_error_mode(failure_gateways: FailureModeGateways) -> None:
_register_failing_store(failure_gateways.upstream)
marker: Final = _marker()
with failure_gateways.error.scenario() as scenario:
model: Final = scenario.model()
key: Final = scenario.key(models=[model])
response: Final = failure_gateways.error.request(
"POST", "/v1/chat/completions", _chat_body(model, marker, FAIL_STORE_ID), key=key
)
assert response.status_code != 200, response.text
assert "error" in response.json(), response.text
assert "vector store" in response.text.lower() or FAIL_STORE_ID in response.text, response.text

View file

@ -0,0 +1,340 @@
import asyncio
import json
import re
import signal
import threading
import uuid
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
from queue import SimpleQueue
from types import MappingProxyType
from typing import Final, Literal
from urllib.parse import urlsplit
import httpx
import psutil
import pytest
import yaml
from integration._support.client import Gateway, eventually, string_value
from integration._support.database import read_rows
from integration._support.process import owned_proxy_process
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue, TypeAdapter
_BACKEND: Final = "anthropic.claude-haiku-4-5"
_API_KEY: Final = "synthetic-mantle-bearer"
_CONFIG_MODEL: Final = "bedrock-mantle-claude-chat-chaos"
_MESSAGES_PATH: Final = "/anthropic/v1/messages"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_MARKER: Final = re.compile(r"marker-([0-9a-f]{32})")
_STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]")
_ROWS_BY_CALL: Final = (
"SELECT litellm_call_id, status FROM \"LiteLLM_SpendLogs\" WHERE litellm_call_id = ANY(string_to_array(%s, ','))"
)
Endpoint = Literal["chat", "messages", "responses"]
@dataclass(frozen=True, slots=True)
class _Call:
endpoint: Endpoint
stream: bool
marker: str
@dataclass(frozen=True, slots=True)
class _Served:
call: _Call
status: int
text: str
call_id: str
def _answer(marker: str) -> str:
return f"answer marker-{marker}"
def _path(endpoint: Endpoint) -> str:
match endpoint:
case "chat":
return "/v1/chat/completions"
case "messages":
return "/v1/messages"
case "responses":
return "/v1/responses"
def _body(model: str, call: _Call) -> dict[str, JsonValue]:
question: Final = f"Question marker-{call.marker}"
common: Final[dict[str, JsonValue]] = {"model": model, "stream": call.stream}
match call.endpoint:
case "chat":
return {**common, "messages": [{"role": "user", "content": question}]}
case "messages":
return {**common, "max_tokens": 64, "messages": [{"role": "user", "content": question}]}
case "responses":
return {**common, "input": question}
def _mantle_reply(marker: str, stream: bool, *, abort: bool = False, pause: float = 0) -> Reply:
message: Final = {
"id": f"msg_bdrk_{marker}",
"type": "message",
"role": "assistant",
"model": _BACKEND,
"stop_sequence": None,
}
if not stream:
payload: Final = json.dumps(
{
**message,
"content": [{"type": "text", "text": _answer(marker)}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 23, "output_tokens": 7},
}
).encode()
return Reply(chunks=(payload,), abort_after=0) if abort else Reply(body=payload)
opening: Final = {**message, "content": [], "stop_reason": None, "usage": {"input_tokens": 23, "output_tokens": 1}}
events: Final = (
("message_start", {"message": opening}),
("content_block_start", {"index": 0, "content_block": {"type": "text", "text": ""}}),
("content_block_delta", {"index": 0, "delta": {"type": "text_delta", "text": "answer "}}),
("content_block_delta", {"index": 0, "delta": {"type": "text_delta", "text": f"marker-{marker}"}}),
("content_block_stop", {"index": 0}),
("message_delta", {"delta": {"stop_reason": "end_turn", "stop_sequence": None}, "usage": {"output_tokens": 7}}),
("message_stop", {}),
)
return Reply(
content_type="text/event-stream",
chunks=tuple(
f"event: {kind}\ndata: {json.dumps({'type': kind, **payload})}\n\n".encode() for kind, payload in events
),
abort_after=0 if abort else None,
pause_between_chunks=pause,
)
def _marker_of(request: Request) -> str:
found: Final = _MARKER.search(request.body.decode())
assert found is not None, request.body
return found.group(1)
def _native_peer(aborted: frozenset[str] = frozenset(), pause: float = 0) -> Callable[[Request], Reply]:
def respond(request: Request) -> Reply:
assert (request.method, request.target) == ("POST", _MESSAGES_PATH), request.target
assert request.headers["authorization"] == f"Bearer {_API_KEY}", sorted(request.headers)
marker: Final = _marker_of(request)
stream: Final = _JSON_OBJECT.validate_json(request.body).get("stream") is True
return _mantle_reply(marker, stream, abort=marker in aborted, pause=pause)
return respond
def _assert_native_requests_for(received: tuple[Request, ...], calls: tuple[_Call, ...]) -> None:
assert {(request.method, request.target) for request in received} == {("POST", _MESSAGES_PATH)}
assert sorted(_marker_of(request) for request in received) == sorted(call.marker for call in calls)
def _spend_status_by_call(served: tuple[_Served, ...]) -> dict[str, JsonValue]:
wanted: Final = sorted(item.call_id for item in served)
rows: Final = eventually(
lambda: read_rows(_ROWS_BY_CALL, (",".join(wanted),)), lambda found: len(found) >= len(wanted), seconds=70
)
assert sorted(string_value(row["litellm_call_id"]) for row in rows) == wanted, rows
return {string_value(row["litellm_call_id"]): row["status"] for row in rows}
async def _send(client: httpx.AsyncClient, key: str, model: str, call: _Call) -> _Served:
async with client.stream(
"POST",
_path(call.endpoint),
json=_body(model, call),
headers={"Authorization": f"Bearer {key}", "anthropic-version": "2023-06-01"},
) as response:
raw: Final = await response.aread()
return _Served(
call=call,
status=response.status_code,
text=raw.decode(),
call_id=response.headers.get("x-litellm-call-id", ""),
)
async def _burst(
base_url: str, key: str, model: str, calls: tuple[_Call, ...], *, tolerate_transport_errors: bool = False
) -> tuple[_Served, ...]:
async with httpx.AsyncClient(base_url=base_url, timeout=60, trust_env=False) as client:
results: Final = await asyncio.gather(
*(_send(client, key, model, call) for call in calls), return_exceptions=tolerate_transport_errors
)
for result in results:
assert not isinstance(result, BaseException) or isinstance(result, httpx.TransportError), repr(result)
return tuple(result for result in results if isinstance(result, _Served))
def _calls(count: int, endpoints: tuple[Endpoint, ...], stream: Callable[[int], bool]) -> tuple[_Call, ...]:
return tuple(
_Call(endpoint=endpoints[index % len(endpoints)], stream=stream(index), marker=uuid.uuid4().hex)
for index in range(count)
)
def _assert_answered_with_its_own_marker(served: _Served) -> None:
assert served.status == 200, served.text
assert set(_MARKER.findall(served.text)) == {served.call.marker}, served.text
async def test_concurrent_burst_across_endpoints_is_answered_from_the_native_route_and_logged_once_per_call(
gateway: Gateway,
) -> None:
calls: Final = _calls(30, ("chat", "messages", "responses"), lambda index: index % 2 == 0)
with wire_server(_native_peer()) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"bedrock_mantle/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
served: Final = await _burst(str(gateway.client.base_url), gateway.key, model, calls)
assert len(served) == 30
for item in served:
_assert_answered_with_its_own_marker(item)
_assert_native_requests_for(wire.drain(), calls)
assert _spend_status_by_call(served) == {item.call_id: "success" for item in served}
def _unhealthy_count(gateway: Gateway, model: str) -> JsonValue:
response: Final = gateway.request("GET", "/health", params={"model": model})
return _JSON_OBJECT.validate_json(response.content).get("unhealthy_count")
async def test_upstream_aborts_then_an_outage_fail_each_call_once_and_the_native_route_recovers(
gateway: Gateway,
) -> None:
calls: Final = _calls(21, ("chat", "messages", "responses"), lambda index: index % 2 == 0)
aborted: Final = frozenset(call.marker for call in calls if call.endpoint == "chat")
during_outage: Final = (
_Call(endpoint="chat", stream=True, marker=uuid.uuid4().hex),
_Call(endpoint="chat", stream=False, marker=uuid.uuid4().hex),
_Call(endpoint="messages", stream=False, marker=uuid.uuid4().hex),
_Call(endpoint="responses", stream=False, marker=uuid.uuid4().hex),
)
after_restart: Final = _calls(3, ("chat", "messages", "responses"), lambda index: index == 0)
proxy: Final = str(gateway.client.base_url)
with gateway.scenario() as scenario:
with wire_server(_native_peer(aborted)) as wire:
upstream: Final = wire.url
model: Final = scenario.model(model=f"bedrock_mantle/{_BACKEND}", api_base=upstream, api_key=_API_KEY)
burst: Final = await _burst(proxy, gateway.key, model, calls)
assert len(burst) == 21
for item in burst:
if item.call.marker in aborted:
assert item.status == (500 if item.call.stream else 503), item.text
assert "Response payload is not completed" in item.text and "answer marker-" not in item.text
else:
_assert_answered_with_its_own_marker(item)
_assert_native_requests_for(wire.drain(), calls)
refused: Final = await _burst(proxy, gateway.key, model, during_outage)
assert len(refused) == 4
for item in refused:
assert item.status == 503, item.text
assert "Cannot connect to host" in item.text, item.text
assert await asyncio.to_thread(
eventually, lambda: _unhealthy_count(gateway, model), lambda count: count == 1, 30
)
with wire_server(_native_peer(), port=urlsplit(upstream).port or 0) as restarted:
recovered: Final = await _burst(proxy, gateway.key, model, after_restart)
assert len(recovered) == 3
for item in recovered:
_assert_answered_with_its_own_marker(item)
_assert_native_requests_for(restarted.drain(), after_restart)
failed: Final = frozenset(item.call_id for item in (*burst, *refused) if item.status != 200)
assert len(failed) == len(aborted) + 4
assert _spend_status_by_call((*burst, *refused, *recovered)) == {
item.call_id: "failure" if item.call_id in failed else "success" for item in (*burst, *refused, *recovered)
}
async def test_slow_native_streams_reach_every_caller_whole_and_are_logged_once(gateway: Gateway) -> None:
calls: Final = _calls(10, ("chat", "responses"), lambda _: True)
with wire_server(_native_peer(pause=0.3)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"bedrock_mantle/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
served: Final = await _burst(str(gateway.client.base_url), gateway.key, model, calls)
assert len(served) == 10
for item in served:
_assert_answered_with_its_own_marker(item)
assert item.text.rstrip().endswith("data: [DONE]"), item.text
_assert_native_requests_for(wire.drain(), calls)
assert _spend_status_by_call(served) == {item.call_id: "success" for item in served}
def _chaos_config(wire: Wire, tmp_path: Path) -> Path:
config: Final = {
**yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()),
"model_list": [
{
"model_name": _CONFIG_MODEL,
"litellm_params": {"model": f"bedrock_mantle/{_BACKEND}", "api_base": wire.url, "api_key": _API_KEY},
}
],
}
target: Final = tmp_path / "bedrock-mantle-claude-chat-chaos.yaml"
target.write_text(yaml.safe_dump(config))
return target
def _live_workers(log: Path) -> tuple[int, ...]:
started: Final = (int(pid) for pid in _STARTED_WORKER.findall(log.read_text()))
return tuple(pid for pid in started if psutil.pid_exists(pid))
def _open_upstream_connections(pid: int, upstream: str) -> int:
port: Final = urlsplit(upstream).port
return sum(
1
for connection in psutil.Process(pid).net_connections(kind="tcp")
if connection.status == psutil.CONN_ESTABLISHED and connection.raddr and connection.raddr.port == port
)
@pytest.mark.timeout(300)
async def test_worker_sigkill_mid_burst_leaves_the_sibling_serving_the_native_route(
gateway: Gateway, tmp_path: Path
) -> None:
calls: Final = _calls(20, ("chat",), lambda _: False)
release: Final = threading.Event()
held_markers: Final[SimpleQueue[str]] = SimpleQueue()
answer: Final = _native_peer()
def held(request: Request) -> Reply:
held_markers.put(_marker_of(request))
assert release.wait(timeout=60), "The burst was never released"
return answer(request)
with wire_server(held) as wire:
config: Final = _chaos_config(wire, tmp_path)
with owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2) as owned:
candidate: Final = owned.gateway
workers: Final = eventually(lambda: _live_workers(owned.log), lambda pids: len(pids) == 2, seconds=30)
burst: Final = asyncio.create_task(
_burst(
str(candidate.client.base_url), candidate.key, _CONFIG_MODEL, calls, tolerate_transport_errors=True
)
)
await asyncio.to_thread(eventually, held_markers.qsize, lambda size: size == 20, 60)
held_by: Final = MappingProxyType({pid: _open_upstream_connections(pid, wire.url) for pid in workers})
assert sum(held_by.values()) == 20, held_by
victim_pid, survivor_pid = sorted(workers, key=held_by.__getitem__)
victim: Final = psutil.Process(victim_pid)
victim.suspend()
victim.send_signal(signal.SIGKILL)
release.set()
served: Final = await burst
assert held_by[survivor_pid] >= 10, held_by
assert len(served) == held_by[survivor_pid], (held_by, len(served))
for item in served:
_assert_answered_with_its_own_marker(item)
follow_up: Final = _Call(endpoint="chat", stream=False, marker=uuid.uuid4().hex)
(answered,) = await _burst(str(candidate.client.base_url), candidate.key, _CONFIG_MODEL, (follow_up,))
_assert_answered_with_its_own_marker(answered)
_assert_native_requests_for(wire.drain(), (*calls, follow_up))
assert _spend_status_by_call((*served, answered)) == {
item.call_id: "success" for item in (*served, answered)
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,203 @@
from __future__ import annotations
import json
import re
import signal
from collections.abc import Mapping
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from types import MappingProxyType
from typing import Final
import httpx
import psutil
from integration._support.client import Gateway, eventually, gateway_from_environment
from integration._support.process import owned_proxy_process
_RESPONSES_JSON_SCENARIO: Final = "audit-chaos-responses"
_RESPONSES_STREAM_SCENARIO: Final = "audit-chaos-responses-stream"
_MESSAGES_JSON_SCENARIO: Final = "audit-chaos-messages"
_MESSAGES_STREAM_SCENARIO: Final = "audit-chaos-messages-stream"
_RESPONSES_BODY: Final = {
"id": "resp_$UNIQUE_ID",
"object": "response",
"created_at": 1700000000,
"status": "completed",
"model": "openai/gpt-4o-mini",
"output": [{"type": "message", "content": [{"type": "output_text", "text": "done"}]}],
"usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7},
}
_ANTHROPIC_MESSAGE: Final = {
"id": "msg_$UNIQUE_ID",
"type": "message",
"role": "assistant",
"model": "claude-3-5-sonnet-20241022",
"content": [{"type": "text", "text": "scripted reply"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 9, "output_tokens": 5},
}
_ANTHROPIC_EVENTS: Final = (
'event: message_start\ndata: {"type":"message_start","message":{"id":"msg_$UNIQUE_ID","type":"message","role":"assistant","model":"claude-3-5-sonnet-20241022","content":[],"usage":{"input_tokens":9,"output_tokens":0}}}',
'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}',
'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"scripted reply"}}',
'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}',
'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":5}}',
'event: message_stop\ndata: {"type":"message_stop"}',
)
_RESPONSES_EVENTS: Final = (
'event: response.completed\ndata: {"type":"response.completed","response":{"id":"resp_$UNIQUE_ID","object":"response","created_at":1700000000,"status":"completed","model":"openai/gpt-4o-mini","output":[{"type":"message","content":[{"type":"output_text","text":"done"}]}],"usage":{"input_tokens":3,"output_tokens":4,"total_tokens":7}}}',
)
def _register_scenarios(upstream_url: str) -> None:
with httpx.Client(timeout=5, trust_env=False) as client:
for scenario_id, response in (
(
_RESPONSES_JSON_SCENARIO,
{
"content_type": "application/x-routed",
"routes": {"POST /responses": {"content_type": "application/json", "body": _RESPONSES_BODY}},
},
),
(_RESPONSES_STREAM_SCENARIO, {"content_type": "text/event-stream", "frames": _RESPONSES_EVENTS}),
(
_MESSAGES_JSON_SCENARIO,
{
"content_type": "application/x-routed",
"routes": {"POST /v1/messages": {"content_type": "application/json", "body": _ANTHROPIC_MESSAGE}},
},
),
(_MESSAGES_STREAM_SCENARIO, {"content_type": "text/event-stream", "frames": _ANTHROPIC_EVENTS}),
):
result: Final = client.post(
f"{upstream_url}/__scenarios", json={"scenario_id": scenario_id, "response": response}
)
assert result.status_code == 200, result.text
def _stream_response_id(text: str) -> str | None:
for line in text.splitlines():
if not line.startswith("data: ") or line == "data: [DONE]":
continue
value: Final = json.loads(line.removeprefix("data: "))
if isinstance(value, dict):
if isinstance(value.get("id"), str):
return value["id"]
for key_name in ("response", "message"):
nested: Final = value.get(key_name)
if isinstance(nested, dict) and isinstance(nested.get("id"), str):
return nested["id"]
return None
_KINDS: Final = ("chat", "chat-stream", "messages", "messages-stream", "responses", "responses-stream")
_PATHS: Final = MappingProxyType(
{"chat": "/v1/chat/completions", "messages": "/v1/messages", "responses": "/v1/responses"}
)
def _burst_kind(index: int) -> str:
return f"{_KINDS[index % 3 * 2].removesuffix('-stream')}{'-stream' if index % 2 == 0 else ''}"
def _fire(
owned: Gateway, models: Mapping[str, str], key: str, kind: str, marker: str
) -> tuple[str, int | None, str | None]:
stream: Final = kind.endswith("-stream")
path: Final = _PATHS[kind.removesuffix("-stream")]
model: Final = models[f"{path.rsplit('/', 1)[-1]}{'-stream' if stream else ''}"]
if path == "/v1/messages":
body: Final = {
"model": model,
"messages": [{"role": "user", "content": f"burst-{kind}-{marker}"}],
"max_tokens": 64,
"stream": stream,
}
elif path == "/v1/responses":
body = {"model": model, "input": f"burst-{kind}-{marker}", "stream": stream}
else:
body = {
"model": model,
"messages": [{"role": "user", "content": f"burst-{kind}-{marker}"}],
"stream": stream,
**({"stream_options": {"include_usage": True}} if stream else {}),
}
try:
response: Final = owned.request("POST", path, body, key=key)
except Exception:
return kind, None, None
if response.status_code != 200:
return kind, response.status_code, None
if stream:
return kind, 200, _stream_response_id(response.text)
response_id: Final = response.json().get("id")
return kind, 200, response_id if isinstance(response_id, str) else None
_WORKER_PID: Final = re.compile(r"Started server process \[(\d+)\]")
def _worker_pids(log: Path) -> tuple[int, ...]:
return tuple(int(match) for match in _WORKER_PID.findall(log.read_text()))
def test_proxy_survives_worker_kill_mid_burst(tmp_path: Path) -> None:
with gateway_from_environment() as upstream_gateway:
directory: Final = tmp_path
_register_scenarios(upstream_gateway.upstream_url)
with owned_proxy_process(upstream_gateway, directory, {}, workers=2) as owned:
with owned.gateway.scenario() as scenario:
model: Final = scenario.model()
responses_model: Final = scenario.model(
api_base=f"{owned.gateway.upstream_url}/{_RESPONSES_JSON_SCENARIO}"
)
responses_stream_model: Final = scenario.model(
api_base=f"{owned.gateway.upstream_url}/{_RESPONSES_STREAM_SCENARIO}"
)
messages_model: Final = scenario.model(
model="anthropic/claude-3-5-sonnet-20241022",
api_base=f"{owned.gateway.upstream_url}/{_MESSAGES_JSON_SCENARIO}",
)
messages_stream_model: Final = scenario.model(
model="anthropic/claude-3-5-sonnet-20241022",
api_base=f"{owned.gateway.upstream_url}/{_MESSAGES_STREAM_SCENARIO}",
)
key: Final = scenario.key(
models=[model, responses_model, responses_stream_model, messages_model, messages_stream_model]
)
models: Final = MappingProxyType(
{
"completions": model,
"completions-stream": model,
"messages": messages_model,
"messages-stream": messages_stream_model,
"responses": responses_model,
"responses-stream": responses_stream_model,
}
)
workers: Final = eventually(lambda: _worker_pids(owned.log), lambda pids: len(pids) == 2, seconds=30)
records: dict[int, tuple[str, int | None, str | None]] = {}
with ThreadPoolExecutor(max_workers=30) as pool:
futures: Final = [
(index, pool.submit(_fire, owned.gateway, models, key, _burst_kind(index), str(index)))
for index in range(30)
]
psutil.Process(workers[0]).send_signal(signal.SIGKILL)
for index, future in futures:
records[index] = future.result()
proxy_stamped: Final = [
record[2]
for record in records.values()
if record[2] is not None and not record[0].startswith("messages")
]
malformed: Final = {i: r for i, r in records.items() if r[1] == 200 and r[2] is None}
assert not malformed, f"200 responses without a well-formed body: {malformed}"
assert len(proxy_stamped) == len(set(proxy_stamped)), records
assert proxy_stamped, "no burst request completed"
for kind in _KINDS:
_, status, response_id = _fire(owned.gateway, models, key, kind, "probe")
assert status == 200 and response_id is not None, (kind, status, records)

View file

@ -9,10 +9,12 @@ import tempfile
import threading
import time
import typing
from collections.abc import Mapping
from contextlib import asynccontextmanager, contextmanager
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import Final
import httpx
import httpx2
@ -21,9 +23,12 @@ import uvicorn
import yaml
from mcp import ClientSession
from mcp.client.streamable_http import streamable_http_client
from mcp.shared._httpx_utils import create_mcp_http_client
from mcp.types import CallToolResult
from starlette.requests import Request
from tests.integration._support.wire import Reply, Request as WireRequest, Wire, wire_server
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._experimental.mcp_server.tool_search import handle_mcp_proxy_tool
from litellm.proxy._types import LiteLLM_ObjectPermissionTable, ProxyException, UserAPIKeyAuth
@ -45,6 +50,173 @@ PROXY_START_TIMEOUT = 30
PROXY_AUTHORIZATION_HEADER = "Bearer sk-1234"
@pytest.mark.asyncio
async def test_cold_concurrent_schema_validation_accepts_valid_arguments() -> None:
from litellm.proxy._experimental.mcp_server.tool_search import _tool_argument_validation_error
schema: Final = {
"type": "object",
"$defs": {"amount": {"type": "number", "multipleOf": 0.25}},
"properties": {"amount": {"$ref": "#/$defs/amount"}},
}
original_affinity: Final = os.sched_getaffinity(0) if sys.platform == "linux" else None
try:
if original_affinity is not None:
os.sched_setaffinity(0, {min(original_affinity)})
for _ in range(2):
results: Final = await asyncio.gather(
*(_tool_argument_validation_error(schema, {"amount": 0.75}) for _ in range(8))
)
assert results == [None] * 8, "Cold and warm workers must accept valid concurrent tool arguments"
finally:
if original_affinity is not None:
os.sched_setaffinity(0, original_affinity)
@pytest.mark.asyncio
@pytest.mark.parametrize("cancel", (False, True))
@pytest.mark.parametrize("concurrency", (1, 8))
async def test_schema_validation_stops_expensive_work_and_recovers(cancel: bool, concurrency: int) -> None:
import psutil
from litellm.proxy._experimental.mcp_server.tool_search import _tool_argument_validation_error
existing_children: Final = frozenset(child.pid for child in psutil.Process().children())
warm_count: Final = min(concurrency, 4)
assert await asyncio.gather(
*(_tool_argument_validation_error({"type": "object"}, {}) for _ in range(warm_count))
) == [None] * warm_count
started: Final = time.monotonic()
tasks: Final = tuple(
asyncio.create_task(
_tool_argument_validation_error(
{"type": "object", "properties": {"value": {"type": "string", "pattern": "^(a+)+$"}}},
{"value": "a" * 80 + "!"},
)
)
for _ in range(concurrency)
)
group: Final = asyncio.gather(*tasks, return_exceptions=True)
try:
with pytest.raises(TimeoutError):
await asyncio.wait_for(asyncio.shield(group), timeout=0.2)
workers: Final = tuple(
child
for child in psutil.Process().children()
if child.pid not in existing_children and "anyio.to_process" in child.cmdline()
)
assert 1 <= len(workers) <= 4
if cancel:
for task in tasks:
task.cancel()
assert all(isinstance(result, asyncio.CancelledError) for result in await group)
else:
assert await group == ["Tool argument validation exceeded its time limit"] * concurrency
assert time.monotonic() - started < 35
assert all(not worker.is_running() for worker in workers), "Cancelled validation must terminate worker CPU work"
assert await _tool_argument_validation_error({"type": "object"}, {}) is None
finally:
for task in tasks:
task.cancel()
await group
@pytest.fixture(scope="session")
def schema_peer() -> typing.Iterator[Wire]:
def respond(request: WireRequest) -> Reply:
if request.method == "GET":
return Reply(body=b'{"type":"number"}')
body: Final = json.loads(request.body)
if "id" not in body:
return Reply(status=202)
result: Final[Mapping[str, object]]
if body["method"] == "initialize":
result = {
"protocolVersion": body["params"]["protocolVersion"],
"capabilities": {"tools": {}},
"serverInfo": {"name": "schema-peer", "version": "1"},
}
elif body["method"] == "tools/list":
result = {
"tools": [
{
"name": name,
"description": "Schema validation",
"inputSchema": {
"type": "object",
"$defs": {"amount": {"type": "number", "minimum": 0.25, "multipleOf": 0.25}},
"properties": {
"value": {"$ref": peer.url + "/ref" if name == "external" else "#/$defs/amount"}
},
},
}
for name in ("external", "local")
]
}
else:
assert body["method"] == "tools/call"
result = {"content": [{"type": "text", "text": "called"}], "isError": False}
return Reply(body=json.dumps({"jsonrpc": "2.0", "id": body["id"], "result": result}).encode())
with wire_server(respond) as peer:
yield peer
@pytest.mark.parametrize("external", (True, False))
def test_proxy_schema_validation_resolves_only_local_references(
proxy_server_url: str, schema_peer: Wire, external: bool
) -> None:
with httpx.Client(
base_url=proxy_server_url,
headers={
"Authorization": "Bearer sk-schema",
"Accept": "application/json, text/event-stream",
"x-mcp-servers": "schema",
},
timeout=30,
) as client:
search: Final = _rpc_result(
client.post(
"/mcp/proxy",
json={
"jsonrpc": "2.0",
"id": 1,
"method": "tools/call",
"params": {"name": "search_tools", "arguments": {"query": "schema"}},
},
)
)
name: Final = "schema-external" if external else "schema-local"
tool_id: Final = next(hit["tool_id"] for hit in json.loads(search["content"][0]["text"]) if hit["name"] == name)
schema_peer.drain()
called: Final = _rpc_result(
client.post(
"/mcp/proxy",
json={
"jsonrpc": "2.0",
"id": 2,
"method": "tools/call",
"params": {
"name": "call_tool",
"arguments": {"tool_id": tool_id, "arguments": {"value": 0.75}},
},
},
)
)
observed: Final = schema_peer.drain()
assert not any(request.method == "GET" for request in observed), (
"Schema validation must not retrieve external references"
)
calls: Final = tuple(
request
for request in observed
if request.method == "POST" and json.loads(request.body).get("method") == "tools/call"
)
assert len(calls) == (0 if external else 1), "Only valid local schemas may reach upstream execution"
assert called["isError"] is external
if not external:
assert called["content"][0]["text"] == "called"
@pytest.fixture(scope="session", autouse=True)
def _clear_proxy_database_env() -> typing.Iterator[None]:
"""Ensure local proxy DB settings don't leak into tests."""
@ -175,10 +347,12 @@ def _proxy_server(
tmp_path_factory: pytest.TempPathFactory,
math_streamable_http_server: str,
math_restricted_server: str,
schema_peer: Wire,
):
config_dir = tmp_path_factory.mktemp("mcp_e2e")
config_path = config_dir / "config.yaml"
config = yaml.safe_load(CONFIG_TEMPLATE_PATH.read_text())
config["mcp_servers"]["schema"] = {"transport": "http", "url": schema_peer.url + "/mcp"}
config["mcp_servers"]["math_stdio"]["command"] = MCP_PEER_PYTHON
config["mcp_servers"]["math_streamable_http"]["url"] = f"{math_streamable_http_server}/mcp"
config["mcp_servers"]["math_restricted"]["url"] = f"{math_restricted_server}/mcp"
@ -208,7 +382,7 @@ def proxy_server_url(_proxy_server: ProxyRig, setup_and_teardown: None) -> str:
@asynccontextmanager
async def _http_streams(url: str, headers: dict[str, str]):
async with httpx2.AsyncClient(headers=headers) as http_client:
async with create_mcp_http_client(headers=headers) as http_client:
async with streamable_http_client(url, http_client=http_client) as streams:
yield streams
@ -528,6 +702,7 @@ class TestProxyMcpSchemaDiscoveryMode:
async def authorize_proxy_key(request: Request, api_key: str) -> UserAPIKeyAuth:
permissions = {
"sk-schema": LiteLLM_ObjectPermissionTable(object_permission_id="schema", mcp_servers=["schema"]),
"sk-1234": LiteLLM_ObjectPermissionTable(object_permission_id="open", mcp_servers=["math_stdio"]),
"sk-restricted": LiteLLM_ObjectPermissionTable(
object_permission_id="restricted", mcp_servers=["math_restricted"]

View file

@ -3114,8 +3114,8 @@ async def test_modern_upstream_requests_are_self_contained_without_initializatio
assert client._last_initialize_instructions == "modern instructions"
assert tuple(methods.get_nowait() for _ in range(methods.qsize())) == (
"server/discover",
"tools/call",
"tools/list",
"tools/call",
)
else:
with pytest.raises((MCPError, RuntimeError), match="protocol version"):
@ -3123,6 +3123,297 @@ async def test_modern_upstream_requests_are_self_contained_without_initializatio
assert tuple(methods.get_nowait() for _ in range(methods.qsize())) == ("server/discover",)
@pytest.mark.asyncio
@pytest.mark.parametrize("paginated", (False, True))
@pytest.mark.parametrize("valid_annotation", (True, False))
async def test_modern_call_emits_listed_argument_headers(paginated: bool, valid_annotation: bool) -> None:
from collections.abc import Mapping
from queue import SimpleQueue
from mcp.types import DiscoverResult, ToolsCapability
calls: Final[SimpleQueue[str]] = SimpleQueue()
def response_result(request: httpx2.Request, payload: JSONRPCRequest) -> Mapping[str, object]:
assert payload.params is not None
caller: Final = request.headers["authorization"].removeprefix("Bearer ")
if payload.method == "server/discover":
return DiscoverResult(
supported_versions=["2026-07-28"], capabilities=ServerCapabilities(tools=ToolsCapability())
).model_dump(by_alias=True, exclude_none=True)
if payload.method == "tools/list":
if paginated and "cursor" not in payload.params:
return {
"resultType": "complete",
"cacheScope": "private",
"ttlMs": 0,
"tools": [],
"nextCursor": "second",
}
return {
"resultType": "complete",
"cacheScope": "private",
"ttlMs": 0,
"tools": [
{
"name": "quote",
"inputSchema": {
"type": "object",
"properties": {
"workspace": {
"type": "string" if valid_annotation else "number",
"x-mcp-header": caller,
},
"count": {"type": "integer", "x-mcp-header": "Count"},
"preview": {"type": "boolean", "x-mcp-header": "Preview"},
},
},
}
],
}
assert payload.method == "tools/call"
calls.put(caller)
assert valid_annotation, "Invalid header definitions must prevent dispatch"
assert request.headers.get(f"mcp-param-{caller.lower()}") == caller
assert request.headers.get("mcp-param-count") == "3"
assert request.headers.get("mcp-param-preview") == "true"
assert set(key for key in request.headers if key.startswith("mcp-param-")) == {
f"mcp-param-{caller.lower()}",
"mcp-param-count",
"mcp-param-preview",
}
assert payload.params["arguments"] == {"workspace": caller, "count": 3, "preview": True}
return {"resultType": "complete", "content": [{"type": "text", "text": "quoted"}], "isError": False}
def respond(request: httpx2.Request) -> httpx2.Response:
payload: Final = _JSONRPC_MESSAGE_ADAPTER.validate_json(request.content)
assert isinstance(payload, JSONRPCRequest)
return httpx2.Response(
200, json={"jsonrpc": "2.0", "id": payload.id, "result": response_result(request, payload)}
)
async def call_as(caller: str) -> None:
client: Final = _MockTransportClient(
respond,
server_url="https://example.com/mcp",
protocol_version="2026-07-28",
auth_type=MCPAuth.bearer_token,
auth_value=caller,
)
params: Final = CallToolRequestParams(
name="quote", arguments={"workspace": caller, "count": 3, "preview": True}
)
if not valid_annotation:
with pytest.raises(MCPError, match="schema is unavailable"):
await client.call_tool(params, raise_on_error=True)
return
result: Final = await client.call_tool(params, raise_on_error=True)
assert result.is_error is False
assert result.content[0].text == "quoted"
await asyncio.gather(call_as("Engineering"), call_as("Finance"))
assert calls.qsize() == (2 if valid_annotation else 0)
@pytest.mark.asyncio
@pytest.mark.parametrize("stop", ("page_cap", "repeated_cursor"))
@pytest.mark.parametrize("schema_available", (False, True))
async def test_modern_call_requires_schema_from_bounded_listing(
stop: str, schema_available: bool, monkeypatch: pytest.MonkeyPatch
) -> None:
from queue import SimpleQueue
from mcp.types import DiscoverResult, ToolsCapability
monkeypatch.setattr("litellm.experimental_mcp_client.tools.MCP_TOOL_LISTING_MAX_PAGES", 2)
methods: Final[SimpleQueue[str]] = SimpleQueue()
def respond(request: httpx2.Request) -> httpx2.Response:
payload: Final = _JSONRPC_MESSAGE_ADAPTER.validate_json(request.content)
assert isinstance(payload, JSONRPCRequest)
methods.put(payload.method)
if payload.method == "server/discover":
return httpx2.Response(
200,
json={
"jsonrpc": "2.0",
"id": payload.id,
"result": DiscoverResult(
supported_versions=["2026-07-28"], capabilities=ServerCapabilities(tools=ToolsCapability())
).model_dump(by_alias=True, exclude_none=True),
},
)
if payload.method == "tools/list":
assert payload.params is not None
return httpx2.Response(
200,
json={
"jsonrpc": "2.0",
"id": payload.id,
"result": {
"resultType": "complete",
"cacheScope": "private",
"ttlMs": 0,
"nextCursor": "loop"
if stop == "repeated_cursor"
else str(int(payload.params.get("cursor", "0")) + 1),
"tools": [
{
"name": "quote",
"inputSchema": {
"type": "object",
"properties": {"workspace": {"type": "string", "x-mcp-header": "Workspace"}},
},
}
]
if schema_available
else [],
},
},
)
assert payload.method == "tools/call"
assert schema_available, "An unavailable schema must not permit dispatch without required headers"
assert request.headers["mcp-param-workspace"] == "engineering"
return httpx2.Response(
200,
json={
"jsonrpc": "2.0",
"id": payload.id,
"result": {"resultType": "complete", "content": [{"type": "text", "text": "quoted"}], "isError": False},
},
)
client: Final = _MockTransportClient(respond, server_url="https://example.com/mcp", protocol_version="2026-07-28")
params: Final = CallToolRequestParams(name="quote", arguments={"workspace": "engineering"})
if schema_available:
result: Final = await client.call_tool(params, raise_on_error=True)
assert not result.is_error
assert result.content[0].text == "quoted"
else:
with pytest.raises(MCPError, match="schema is unavailable") as error:
await client.call_tool(params, raise_on_error=True)
assert error.value.code == -32603
observed: Final = tuple(methods.get_nowait() for _ in range(methods.qsize()))
assert observed.count("tools/list") == 2
assert observed.count("tools/call") == int(schema_available)
def test_modern_call_uses_discovery_deadline_for_multiple_pages() -> None:
from mcp.types import DiscoverResult, ToolsCapability
loop: Final = _AutojumpClockLoop()
async def respond(request: httpx2.Request) -> httpx2.Response:
payload: Final = _JSONRPC_MESSAGE_ADAPTER.validate_json(request.content)
assert isinstance(payload, JSONRPCRequest)
if payload.method == "server/discover":
return httpx2.Response(
200,
json={
"jsonrpc": "2.0",
"id": payload.id,
"result": DiscoverResult(
supported_versions=["2026-07-28"], capabilities=ServerCapabilities(tools=ToolsCapability())
).model_dump(by_alias=True, exclude_none=True),
},
)
if payload.method == "tools/list":
await asyncio.sleep(0.06)
assert payload.params is not None
page: Final = (
{"tools": [], "nextCursor": "second"}
if "cursor" not in payload.params
else {
"tools": [
{
"name": "quote",
"inputSchema": {
"type": "object",
"properties": {"workspace": {"type": "string", "x-mcp-header": "Workspace"}},
},
}
]
}
)
return httpx2.Response(
200,
json={
"jsonrpc": "2.0",
"id": payload.id,
"result": {"resultType": "complete", "cacheScope": "private", "ttlMs": 0, **page},
},
)
assert payload.method == "tools/call"
assert request.headers["mcp-param-workspace"] == "engineering"
return httpx2.Response(
200,
json={
"jsonrpc": "2.0",
"id": payload.id,
"result": {"resultType": "complete", "content": [{"type": "text", "text": "quoted"}], "isError": False},
},
)
async def run() -> None:
client: Final = _MockTransportClient(
respond, server_url="https://example.com/mcp", protocol_version="2026-07-28", timeout=0.1
)
assert [tool.name for tool in await client.list_tools(raise_on_error=True)] == ["quote"]
result: Final = await client.call_tool(
CallToolRequestParams(name="quote", arguments={"workspace": "engineering"}), raise_on_error=True
)
assert result.is_error is False
assert result.content[0].text == "quoted"
try:
loop.run_until_complete(run())
finally:
loop.run_until_complete(loop.shutdown_asyncgens())
loop.close()
@pytest.mark.asyncio
async def test_cancelled_modern_catalog_load_prevents_tool_execution() -> None:
from queue import SimpleQueue
from mcp.types import DiscoverResult, ToolsCapability
listing_started: Final = asyncio.Event()
hold_listing: Final = asyncio.Event()
methods: Final[SimpleQueue[str]] = SimpleQueue()
async def respond(request: httpx2.Request) -> httpx2.Response:
payload: Final = _JSONRPC_MESSAGE_ADAPTER.validate_json(request.content)
assert isinstance(payload, JSONRPCRequest)
methods.put(payload.method)
if payload.method == "tools/list":
listing_started.set()
await hold_listing.wait()
assert payload.method == "server/discover", "A cancelled listing must not dispatch a tool call"
discovery: Final = DiscoverResult(
supported_versions=["2026-07-28"], capabilities=ServerCapabilities(tools=ToolsCapability())
)
return httpx2.Response(
200,
json={"jsonrpc": "2.0", "id": payload.id, "result": discovery.model_dump(by_alias=True, exclude_none=True)},
)
client: Final = _MockTransportClient(respond, server_url="https://example.com/mcp", protocol_version="2026-07-28")
task: Final = asyncio.create_task(
client.call_tool(CallToolRequestParams(name="quote", arguments={}), raise_on_error=True)
)
observed_listing: Final = asyncio.create_task(listing_started.wait())
try:
await asyncio.wait((task, observed_listing), return_when=asyncio.FIRST_COMPLETED)
assert listing_started.is_set()
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert tuple(methods.get_nowait() for _ in range(methods.qsize())) == ("server/discover", "tools/list")
finally:
task.cancel()
observed_listing.cancel()
await asyncio.gather(task, observed_listing, return_exceptions=True)
def test_modern_upstream_rejects_legacy_sse_transport() -> None:
with pytest.raises(ValueError, match="transport"):
MCPClient(protocol_version="2026-07-28", transport_type=MCPTransport.sse)

View file

@ -7,6 +7,8 @@ API docs: https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.ht
import json
import asyncio
from collections.abc import Mapping
from typing import Final
from unittest.mock import Mock, patch
@ -16,6 +18,7 @@ from botocore.auth import SigV4Auth
from botocore.awsrequest import AWSRequest
import litellm
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.llms.bedrock_mantle.chat.transformation import BedrockMantleChatConfig
from litellm.llms.bedrock.base_aws_llm import sign_request_off_loop_if_aws
from litellm.types.utils import LlmProviders
@ -818,6 +821,196 @@ class TestBedrockMantleProviderResolution:
)
def _row_cost(key: str, input_tokens: int, output_tokens: int) -> float:
row: Final[Mapping[str, float]] = litellm.model_cost[key]
return input_tokens * row["input_cost_per_token"] + output_tokens * row["output_cost_per_token"]
def _anthropic_message(request: httpx.Request) -> httpx.Response:
return httpx.Response(
status_code=200,
json={
"id": "msg_test",
"type": "message",
"role": "assistant",
"model": "anthropic.claude-opus-5-5",
"content": [{"type": "text", "text": "ok"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 10, "output_tokens": 5},
},
request=request,
)
def _anthropic_event_stream(request: httpx.Request) -> httpx.Response:
events = (
(
"message_start",
{
"type": "message_start",
"message": {
"id": "msg_test",
"type": "message",
"role": "assistant",
"model": "anthropic.claude-opus-5-5",
"content": [],
"usage": {"input_tokens": 10, "output_tokens": 0},
},
},
),
(
"content_block_start",
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
),
(
"content_block_delta",
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "streamed"}},
),
(
"message_delta",
{"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 3}},
),
("message_stop", {"type": "message_stop"}),
)
body = "".join(f"event: {name}\ndata: {json.dumps(data)}\n\n" for name, data in events)
return httpx.Response(
status_code=200, content=body.encode(), headers={"content-type": "text/event-stream"}, request=request
)
class TestBedrockMantleClaudeChatRoute:
def test_claude_completion_uses_native_messages_endpoint(self, monkeypatch, local_cost_map):
from litellm.llms.custom_httpx.http_handler import HTTPHandler
monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "mantle-key")
monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False)
handler = Mock(side_effect=_anthropic_message)
response = litellm.completion(
model="bedrock_mantle/anthropic.claude-opus-5-5",
messages=[{"role": "user", "content": "hello"}],
max_tokens=64,
aws_region_name="us-east-2",
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler))),
)
sent = handler.call_args.args[0]
assert str(sent.url) == "https://bedrock-mantle.us-east-2.api.aws/anthropic/v1/messages"
assert sent.headers["Authorization"] == "Bearer mantle-key"
assert json.loads(sent.content) == {
"model": "anthropic.claude-opus-5-5",
"messages": [{"role": "user", "content": [{"type": "text", "text": "hello"}]}],
"max_tokens": 64,
"anthropic_version": "bedrock-2023-05-31",
}
assert response.choices[0].message.content == "ok"
assert response._hidden_params["response_cost"] == pytest.approx(
_row_cost("bedrock_mantle/anthropic.claude-opus-5-5", 10, 5)
)
assert response._hidden_params["response_cost"] != pytest.approx(_row_cost("anthropic.claude-opus-5-5", 10, 5))
def test_claude_streaming_completion_uses_native_messages_endpoint(self, monkeypatch, local_cost_map):
from litellm.llms.custom_httpx.http_handler import HTTPHandler
monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "mantle-key")
monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False)
handler = Mock(side_effect=_anthropic_event_stream)
stream = litellm.completion(
model="bedrock_mantle/anthropic.claude-opus-5-5",
messages=[{"role": "user", "content": "hello"}],
max_tokens=64,
stream=True,
aws_region_name="us-east-2",
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler))),
)
assert isinstance(stream, CustomStreamWrapper)
text = "".join(chunk.choices[0].delta.content or "" for chunk in stream)
sent = handler.call_args.args[0]
assert str(sent.url) == "https://bedrock-mantle.us-east-2.api.aws/anthropic/v1/messages"
assert json.loads(sent.content)["stream"] is True
assert text == "streamed"
def test_claude_region_prefixed_model_sends_bare_model_to_that_region(self, monkeypatch, local_cost_map):
from litellm.llms.custom_httpx.http_handler import HTTPHandler
for var in (
"BEDROCK_MANTLE_API_KEY",
"AWS_BEARER_TOKEN_BEDROCK",
"BEDROCK_MANTLE_API_BASE",
"BEDROCK_MANTLE_REGION",
"AWS_REGION_NAME",
"AWS_REGION",
"AWS_PROFILE",
):
monkeypatch.delenv(var, raising=False)
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAEXAMPLE")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "c2VjcmV0LXRlc3Qtc2VjcmV0LXRlc3Qtc2VjcmV0")
handler = Mock(side_effect=_anthropic_message)
response = litellm.completion(
model="bedrock_mantle/us-gov-west-1/anthropic.claude-opus-5-5",
messages=[{"role": "user", "content": "hello"}],
max_tokens=64,
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler))),
)
sent = handler.call_args.args[0]
assert str(sent.url) == "https://bedrock-mantle.us-gov-west-1.api.aws/anthropic/v1/messages"
assert json.loads(sent.content)["model"] == "anthropic.claude-opus-5-5"
assert "/us-gov-west-1/bedrock/aws4_request" in sent.headers["Authorization"]
assert response._hidden_params["response_cost"] == pytest.approx(
_row_cost("bedrock_mantle/us-gov-west-1/anthropic.claude-opus-5-5", 10, 5)
)
def test_non_claude_completion_stays_on_chat_completions(self, monkeypatch, local_cost_map):
from litellm.llms.custom_httpx.http_handler import HTTPHandler
monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "mantle-key")
monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False)
def respond(request: httpx.Request) -> httpx.Response:
return httpx.Response(
status_code=200,
json={
"id": "chatcmpl-test",
"object": "chat.completion",
"created": 1733529600,
"model": "openai.gpt-oss-120b",
"choices": [
{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}
],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
},
request=request,
)
handler = Mock(side_effect=respond)
response = litellm.completion(
model="bedrock_mantle/openai.gpt-oss-120b",
messages=[{"role": "user", "content": "hello"}],
aws_region_name="us-east-2",
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler))),
)
sent = handler.call_args.args[0]
assert str(sent.url) == "https://bedrock-mantle.us-east-2.api.aws/v1/chat/completions"
assert response.choices[0].message.content == "ok"
@pytest.mark.parametrize("request_type", ["chat_completion", "embeddings"])
def test_supported_openai_params_follow_the_route_the_model_takes(self, request_type):
claude_params = litellm.get_supported_openai_params(
model="anthropic.claude-opus-5-5", custom_llm_provider="bedrock_mantle", request_type=request_type
)
open_weight_params = litellm.get_supported_openai_params(
model="openai.gpt-oss-120b", custom_llm_provider="bedrock_mantle", request_type=request_type
)
assert claude_params is not None and open_weight_params is not None
assert "thinking" in claude_params
assert "thinking" not in open_weight_params
class TestBedrockMantlePricing:
"""Tests that verify Bedrock Mantle uses correct AWS Bedrock pricing, not OpenAI pricing."""

View file

@ -6637,20 +6637,11 @@ class TestMCPServerManager:
server.mcp_info = {"server_name": "test-server"}
# Mock tools returned from manager (3 tools, but only 2 are allowed)
tool1 = MagicMock()
tool1.name = "allowed_tool_1"
tool1.description = "This tool is allowed"
tool1.input_schema = {}
tool1 = MCPTool(name="allowed_tool_1", description="This tool is allowed", inputSchema={})
tool2 = MagicMock()
tool2.name = "blocked_tool"
tool2.description = "This tool is not allowed"
tool2.input_schema = {}
tool2 = MCPTool(name="blocked_tool", description="This tool is not allowed", inputSchema={})
tool3 = MagicMock()
tool3.name = "allowed_tool_2"
tool3.description = "This tool is also allowed"
tool3.input_schema = {}
tool3 = MCPTool(name="allowed_tool_2", description="This tool is also allowed", inputSchema={})
# Mock the global_mcp_server_manager._get_tools_from_server
from litellm.proxy._experimental.mcp_server import rest_endpoints
@ -6687,20 +6678,11 @@ class TestMCPServerManager:
server.mcp_info = {"server_name": "test-server"}
# Mock tools returned from manager
tool1 = MagicMock()
tool1.name = "tool_1"
tool1.description = "Tool 1"
tool1.input_schema = {}
tool1 = MCPTool(name="tool_1", description="Tool 1", inputSchema={})
tool2 = MagicMock()
tool2.name = "tool_2"
tool2.description = "Tool 2"
tool2.input_schema = {}
tool2 = MCPTool(name="tool_2", description="Tool 2", inputSchema={})
tool3 = MagicMock()
tool3.name = "tool_3"
tool3.description = "Tool 3"
tool3.input_schema = {}
tool3 = MCPTool(name="tool_3", description="Tool 3", inputSchema={})
# Mock the global_mcp_server_manager._get_tools_from_server
from litellm.proxy._experimental.mcp_server import rest_endpoints
@ -6737,15 +6719,9 @@ class TestMCPServerManager:
server.mcp_info = {"server_name": "test-server"}
# Mock tools returned from manager
tool1 = MagicMock()
tool1.name = "tool_1"
tool1.description = "Tool 1"
tool1.input_schema = {}
tool1 = MCPTool(name="tool_1", description="Tool 1", inputSchema={})
tool2 = MagicMock()
tool2.name = "tool_2"
tool2.description = "Tool 2"
tool2.input_schema = {}
tool2 = MCPTool(name="tool_2", description="Tool 2", inputSchema={})
# Mock the global_mcp_server_manager._get_tools_from_server
from litellm.proxy._experimental.mcp_server import rest_endpoints

View file

@ -11,12 +11,13 @@ Covers:
"""
import json
from collections.abc import Sequence
from collections.abc import Mapping, Sequence
from types import SimpleNamespace
from typing import Any
from typing import Any, Final
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from pydantic import JsonValue
from mcp.types import Tool
import litellm
@ -60,6 +61,90 @@ SAMPLE_TOOLS = _make_tools(
)
@pytest.mark.parametrize(
("schema", "arguments", "error"),
(
(
{
"type": "object",
"$defs": {"amount": {"type": "number", "minimum": 0.25, "multipleOf": 0.25}},
"properties": {"amount": {"$ref": "#/$defs/amount"}},
"required": ["amount"],
},
{"amount": 0.75},
None,
),
({"type": "object", "anyOf": [{"required": ["amount"]}, {"required": ["trace"]}]}, {"trace": "a"}, None),
(
{"type": "object", "properties": {"amount": {"type": "number", "minimum": 0.25}}},
{"amount": 0.1},
"Invalid arguments:",
),
({"$ref": "https://schemas.example.invalid/amount"}, {}, "Unable to validate"),
({"$ref": "#/$defs/missing"}, {}, "Unable to validate"),
({"$ref": "#"}, {}, "Unable to validate"),
({"type": "not-a-type"}, {}, "Unable to validate"),
({"properties": {"value": {"pattern": "["}}}, {"value": "a"}, "Unable to validate"),
),
)
def test_offline_schema_validation_preserves_supported_constraints(
schema: Mapping[str, JsonValue], arguments: Mapping[str, JsonValue], error: str | None
) -> None:
from litellm.proxy._experimental.mcp_server.tool_search import _validate_tool_arguments
result: Final = _validate_tool_arguments(schema, arguments)
if error is None:
assert result is None
else:
assert result is not None and result.startswith(error)
@pytest.mark.parametrize(("count", "allowed"), ((4_999, True), (5_000, False)))
def test_validation_node_limit(count: int, allowed: bool) -> None:
from litellm.proxy._experimental.mcp_server.tool_search import _validation_limit_error
result: Final = _validation_limit_error({}, {str(index): None for index in range(count)})
assert (result is None) is allowed
@pytest.mark.parametrize(("size", "allowed"), ((1_048_575, True), (1_048_576, False)))
def test_validation_text_limit(size: int, allowed: bool) -> None:
from litellm.proxy._experimental.mcp_server.tool_search import _validation_limit_error
assert (_validation_limit_error({}, {"x": "a" * size}) is None) is allowed
@pytest.mark.parametrize(("depth", "allowed"), ((64, True), (65, False)))
def test_validation_depth_limit(depth: int, allowed: bool) -> None:
from functools import reduce
from litellm.proxy._experimental.mcp_server.tool_search import _validation_limit_error
nested: Final = reduce(lambda value, _: {"x": value}, range(depth), {})
assert (_validation_limit_error({}, nested) is None) is allowed
@pytest.mark.asyncio
async def test_oversized_arguments_are_rejected_before_worker_submission() -> None:
from litellm.proxy._experimental.mcp_server.tool_search import _tool_argument_validation_error
with patch("anyio.to_process.run_sync", new_callable=AsyncMock) as submit:
result: Final = await _tool_argument_validation_error({}, {"value": "x" * 1_048_576})
assert result == "Tool schema or arguments exceed validation size or depth limits"
submit.assert_not_called()
@pytest.mark.asyncio
async def test_failed_validation_worker_returns_tool_error() -> None:
from anyio import BrokenWorkerProcess
from litellm.proxy._experimental.mcp_server.tool_search import _tool_argument_validation_error
with patch("anyio.to_process.run_sync", side_effect=BrokenWorkerProcess):
result: Final = await _tool_argument_validation_error({}, {})
assert result == "Tool argument validation worker failed"
FX_TOOL = Tool(
name="treasury-get_rates",
description="Get foreign exchange rates for a currency pair",

View file

@ -14,7 +14,7 @@ if sys.version_info < (3, 11): # BaseExceptionGroup is a builtin only from 3.11
import httpx
import pytest
from fastapi import HTTPException
from mcp.types import CallToolResult, TextContent
from mcp.types import CallToolResult, TextContent, Tool
from starlette.requests import Request
from litellm.constants import MCP_TOOL_LISTING_TIMEOUT
@ -3404,17 +3404,10 @@ class TestGetToolsForSingleServer:
from litellm.proxy._types import LiteLLM_ObjectPermissionTable
from litellm.types.mcp import MCPTransport
# Create mock tools
class MockTool:
def __init__(self, name, description):
self.name = name
self.description = description
self.input_schema = {}
mock_tools = [
MockTool("tool1", "First tool"),
MockTool("tool2", "Second tool"),
MockTool("tool3", "Third tool"),
Tool(name="tool1", description="First tool", inputSchema={}),
Tool(name="tool2", description="Second tool", inputSchema={}),
Tool(name="tool3", description="Third tool", inputSchema={}),
]
# Mock _get_tools_from_server to return all tools
@ -3466,15 +3459,9 @@ class TestGetToolsForSingleServer:
from litellm.proxy._experimental.mcp_server.server import MCPServer
from litellm.types.mcp import MCPTransport
class MockTool:
def __init__(self, name, description):
self.name = name
self.description = description
self.input_schema = {}
mock_tools = [
MockTool("tool1", "First tool"),
MockTool("tool2", "Second tool"),
Tool(name="tool1", description="First tool", inputSchema={}),
Tool(name="tool2", description="Second tool", inputSchema={}),
]
async def fake_get_tools_from_server(**kwargs):
@ -3514,15 +3501,9 @@ class TestGetToolsForSingleServer:
from litellm.proxy._types import LiteLLM_ObjectPermissionTable
from litellm.types.mcp import MCPTransport
class MockTool:
def __init__(self, name, description):
self.name = name
self.description = description
self.input_schema = {}
mock_tools = [
MockTool("tool1", "First tool"),
MockTool("tool2", "Second tool"),
Tool(name="tool1", description="First tool", inputSchema={}),
Tool(name="tool2", description="Second tool", inputSchema={}),
]
async def fake_get_tools_from_server(**kwargs):
@ -3567,15 +3548,9 @@ class TestGetToolsForSingleServer:
from litellm.proxy._types import LiteLLM_ObjectPermissionTable
from litellm.types.mcp import MCPTransport
class MockTool:
def __init__(self, name, description):
self.name = name
self.description = description
self.input_schema = {}
mock_tools = [
MockTool("tool1", "First tool"),
MockTool("tool2", "Second tool"),
Tool(name="tool1", description="First tool", inputSchema={}),
Tool(name="tool2", description="Second tool", inputSchema={}),
]
async def fake_get_tools_from_server(**kwargs):
@ -3620,17 +3595,11 @@ class TestGetToolsForSingleServer:
from litellm.proxy._types import LiteLLM_ObjectPermissionTable
from litellm.types.mcp import MCPTransport
class MockTool:
def __init__(self, name, description):
self.name = name
self.description = description
self.input_schema = {}
mock_tools = [
MockTool("tool1", "First tool"),
MockTool("tool2", "Second tool"),
MockTool("tool3", "Third tool"),
MockTool("tool4", "Fourth tool"),
Tool(name="tool1", description="First tool", inputSchema={}),
Tool(name="tool2", description="Second tool", inputSchema={}),
Tool(name="tool3", description="Third tool", inputSchema={}),
Tool(name="tool4", description="Fourth tool", inputSchema={}),
]
async def fake_get_tools_from_server(**kwargs):
@ -3682,13 +3651,7 @@ class TestGetToolsForSingleServer:
from litellm.proxy._experimental.mcp_server.server import MCPServer
from litellm.types.mcp import MCPTransport
class MockTool:
def __init__(self, name):
self.name = name
self.description = name
self.input_schema = {}
mock_tools = [MockTool("tool1"), MockTool("tool2"), MockTool("tool3")]
mock_tools = [Tool(name="tool1", description="tool1", inputSchema={}), Tool(name="tool2", description="tool2", inputSchema={}), Tool(name="tool3", description="tool3", inputSchema={})]
async def fake_get_tools_from_server(**kwargs):
return mock_tools
@ -4401,6 +4364,31 @@ class TestToolResponseMcpInfoEnrichment:
"alias": None,
}
def test_preserves_complete_sdk_tool_definition(self) -> None:
from mcp.types import Tool
tool: Final = Tool.model_validate(
{
"name": "quote",
"title": "Quote",
"description": "Return a quote",
"inputSchema": {
"type": "object",
"$defs": {"amount": {"type": "number", "minimum": 0.25}},
"properties": {"amount": {"$ref": "#/$defs/amount"}},
"anyOf": [{"required": ["amount"]}, {"maxProperties": 0}],
},
"outputSchema": {"type": "object", "properties": {"price": {"type": "number", "multipleOf": 0.25}}},
"annotations": {"readOnlyHint": True},
"_meta": {"display": {"priority": 0.75}},
"icons": [{"src": "https://example.com/icon.png"}],
}
)
original: Final = tool.model_dump(by_alias=True)
server: Final = MCPServer(server_id="quotes", name="quotes", transport=MCPTransport.http)
response: Final = rest_endpoints._create_tool_response_objects([tool], server)[0]
assert response.model_dump(by_alias=True, exclude={"mcp_info"}) == original
assert tool.model_dump(by_alias=True) == original
class TestRestListToolsetFiltering:
@pytest.mark.asyncio

View file

@ -0,0 +1,535 @@
import json
from types import MappingProxyType
import pytest
import litellm
from litellm.proxy.common_utils.codex_model_catalog import (
CODEX_CATALOG_BYTE_LIMIT,
CodexCatalogRow,
CodexServiceTier,
CodexStockModel,
CodexStockUpgrade,
bundled_codex_models,
codex_catalog_rows,
codex_model_list_body,
codex_models_response_json,
configured_service_tiers,
)
_PROMPT = "base prompt"
_FAST = {"id": "priority", "name": "Fast", "description": "2x speed, increased usage"}
def _stock(slug, *, visibility="list", supported_in_api=True, upgrade=None, service_tiers=(_FAST,), default_tier=None):
return CodexStockModel(
slug=slug,
display_name=slug.upper(),
priority=99,
visibility=visibility,
supported_in_api=supported_in_api,
upgrade=CodexStockUpgrade(model=upgrade) if upgrade else None,
service_tiers=tuple(CodexServiceTier(**tier) for tier in service_tiers),
default_service_tier=default_tier,
model_messages={"instructions_template": f"{slug} prompt"},
)
_STOCK = MappingProxyType(
{
model.slug: model
for model in (
_stock("gpt-5.5", upgrade="gpt-6-sol"),
_stock("gpt-6-sol", default_tier="priority"),
_stock("gpt-daybreak", visibility="hide", supported_in_api=False),
)
}
)
def _row(model_id, **overrides):
"""A one-deployment row; `service_tiers` is that deployment's raw value, `deployments` a per-deployment tuple."""
per_deployment = (
{"service_tiers": (overrides["service_tiers"],)}
if "service_tiers" in overrides
else {"service_tiers": overrides["deployments"]}
if "deployments" in overrides
else {}
)
listing = {key: value for key, value in overrides.items() if key not in ("service_tiers", "deployments")}
return CodexCatalogRow(
**{"id": model_id, "mode": "responses", "max_input_tokens": 272000, **listing, **per_deployment}
)
def _body(*rows, **kwargs):
return codex_models_response_json(rows, stock=_STOCK, instructions=_PROMPT, **kwargs)
def _models(*rows, **kwargs):
return json.loads(_body(*rows, **kwargs).json)["models"]
def _tiers(entry):
return [(tier["id"], tier["name"], tier["description"]) for tier in entry["service_tiers"]]
def test_body_is_exactly_codex_models_response():
body = json.loads(_body(_row("gpt-6-astra")).json)
assert list(body) == ["models"]
assert [entry["slug"] for entry in body["models"]] == ["gpt-6-astra"]
def test_unknown_model_entry_carries_every_field_codex_deserializes_without_a_default():
"""Codex 0.159.3 `codex-rs/protocol/src/openai_models.rs` ModelInfo, the fields with no
`#[serde(default)]`, read on 2026-10-01; `base_instructions` because its `ModelsResponse`
decoder rejects an entry with neither it nor `model_messages.instructions_template`."""
(entry,) = _models(_row("gpt-6-astra"))
assert entry.keys() >= {
"slug",
"display_name",
"description",
"supported_reasoning_levels",
"shell_type",
"visibility",
"supported_in_api",
"priority",
"availability_nux",
"upgrade",
"support_verbosity",
"default_verbosity",
"apply_patch_tool_type",
"truncation_policy",
"experimental_supported_tools",
}
assert (entry["slug"], entry["display_name"], entry["visibility"], entry["supported_in_api"]) == (
"gpt-6-astra",
"gpt-6-astra",
"list",
True,
)
assert (entry["context_window"], entry["base_instructions"], entry["service_tiers"]) == (272000, _PROMPT, [])
def test_known_model_keeps_codex_stock_entry_listed_under_the_served_id():
(entry,) = _models(_row("gpt-daybreak"))
assert entry["model_messages"] == {"instructions_template": "gpt-daybreak prompt"}
assert "base_instructions" not in entry
assert (entry["slug"], entry["display_name"], entry["visibility"], entry["supported_in_api"]) == (
"gpt-daybreak",
"GPT-DAYBREAK",
"list",
True,
)
assert _tiers(entry) == [("priority", "Fast", "2x speed, increased usage")]
@pytest.mark.parametrize("upstream", ["gpt-5.5", "openai/gpt-5.5"])
def test_alias_of_a_known_upstream_model_takes_stock_metadata_under_its_own_name(upstream):
(entry,) = _models(_row("team-55", upstream_model=upstream))
assert entry["model_messages"] == {"instructions_template": "gpt-5.5 prompt"}
assert (entry["slug"], entry["display_name"]) == ("team-55", "team-55")
@pytest.mark.parametrize(
("row", "expected"),
[
(_row("gpt-5.5", display_name="Our 5.5"), "Our 5.5"),
(_row("gpt-5.5"), "GPT-5.5"),
(_row("fast-55", upstream_model="gpt-5.5"), "fast-55"),
(_row("mystery", display_name="Mystery"), "Mystery"),
(_row("mystery"), "mystery"),
],
)
def test_display_name_is_configured_then_stock_for_its_own_slug_then_the_id(row, expected):
(entry,) = _models(row)
assert entry["display_name"] == expected
def test_priority_is_listing_order():
entries = _models(_row("gpt-6-sol"), _row("mystery"), _row("gpt-5.5"))
assert [(entry["slug"], entry["priority"]) for entry in entries] == [
("gpt-6-sol", 0),
("mystery", 1),
("gpt-5.5", 2),
]
def test_upgrade_nudge_survives_only_when_its_target_is_served():
with_target, without_target = (
_models(_row("gpt-5.5"), _row("gpt-6-sol"))[0],
_models(_row("gpt-5.5"))[0],
)
assert with_target["upgrade"]["model"] == "gpt-6-sol"
assert without_target["upgrade"] is None
@pytest.mark.parametrize(
("row", "expected"),
[
(
_row("gpt-6-astra", service_tiers=["ultrafast"]),
[("ultrafast", "Ultrafast", "Sends service_tier=ultrafast upstream")],
),
(
_row("gpt-6-astra", service_tiers=[{"id": "ultrafast", "name": "Ultra fast", "description": "Fastest"}]),
[("ultrafast", "Ultra fast", "Fastest")],
),
(
_row("gpt-6-astra", service_tiers=[{"id": "ultrafast"}]),
[("ultrafast", "Ultrafast", "Sends service_tier=ultrafast upstream")],
),
(
_row("gpt-5.5", service_tiers=["ultrafast"]),
[("ultrafast", "Ultrafast", "Sends service_tier=ultrafast upstream")],
),
(
_row("gpt-5.5", service_tiers=["priority", "ultrafast"]),
[
("priority", "Fast", "2x speed, increased usage"),
("ultrafast", "Ultrafast", "Sends service_tier=ultrafast upstream"),
],
),
(_row("gpt-5.5", service_tiers=[]), []),
(
_row("gpt-5.5", service_tiers=["ultrafast", "priority", "ultrafast", {"id": "priority"}]),
[
("ultrafast", "Ultrafast", "Sends service_tier=ultrafast upstream"),
("priority", "Fast", "2x speed, increased usage"),
],
),
],
)
def test_configured_service_tiers_replace_stock_tiers_and_a_known_id_keeps_its_codex_name(row, expected):
(entry,) = _models(row)
assert _tiers(entry) == expected
@pytest.mark.parametrize(
"invalid", ["ultrafast", [""], [" "], [1], [{"name": "no id"}], [{"id": "x", "bogus": 1}], {"id": "x"}]
)
def test_invalid_service_tiers_offer_no_tier_even_on_a_model_codex_ships_tiers_for(invalid):
"""Keeping the stock tiers would offer Codex's `/fast` on a model whose operator declared something else."""
stock_entry, fallback_entry = _models(
_row("gpt-5.5", service_tiers=invalid), _row("mystery", service_tiers=invalid)
)
assert _tiers(stock_entry) == []
assert fallback_entry["service_tiers"] == []
@pytest.mark.parametrize(
("deployments", "expected"),
[
((["ultrafast"], ["ultrafast"]), [("ultrafast", "Ultrafast", "Sends service_tier=ultrafast upstream")]),
(
(["priority", "ultrafast"], ["ultrafast", "flex", "priority"]),
[
("priority", "Fast", "2x speed, increased usage"),
("ultrafast", "Ultrafast", "Sends service_tier=ultrafast upstream"),
],
),
(
(["priority", "ultrafast"], ["ultrafast"]),
[("ultrafast", "Ultrafast", "Sends service_tier=ultrafast upstream")],
),
((["ultrafast"], ["priority"]), []),
((["ultrafast"], None), []),
((None, ["ultrafast"], None), []),
],
)
def test_a_tier_is_offered_only_when_every_deployment_of_the_model_lists_it(deployments, expected):
"""Any deployment can serve the request, so the first deployment's order is kept but a tier one of them
lacks, or a deployment that lists none, offers nothing Codex could send to the wrong deployment."""
(entry,) = _models(_row("gpt-5.5", deployments=deployments))
assert _tiers(entry) == expected
def test_deployments_that_all_leave_service_tiers_unset_keep_the_stock_tiers():
stock_entry, fallback_entry = _models(
_row("gpt-5.5", deployments=(None, None)), _row("mystery", deployments=(None, None))
)
assert _tiers(stock_entry) == [("priority", "Fast", "2x speed, increased usage")]
assert fallback_entry["service_tiers"] == []
@pytest.mark.parametrize(
"deployments", [(["ultrafast"], "not-a-list"), ("not-a-list", ["ultrafast"]), (["priority"], "not-a-list")]
)
def test_an_invalid_value_on_one_deployment_offers_no_tier_for_the_whole_model(deployments):
"""The deployment with the typo lists nothing Codex could send, and the stock `/fast` stays off too."""
(entry,) = _models(_row("gpt-5.5", deployments=deployments))
assert _tiers(entry) == []
@pytest.mark.parametrize(
("configured", "expected"),
[(None, "priority"), (["priority", "ultrafast"], "priority"), (["ultrafast"], None), ([], None)],
)
def test_default_service_tier_survives_only_while_still_offered(configured, expected):
(entry,) = _models(_row("gpt-6-sol", service_tiers=configured))
assert entry["default_service_tier"] == expected
def test_non_chat_and_wildcard_rows_are_left_out_of_the_catalog():
body = _body(
_row("gpt-6-astra"),
CodexCatalogRow(id="text-embedding-3-small", mode="embedding"),
CodexCatalogRow(id="openai/*"),
CodexCatalogRow(id="no-mode-model"),
_row("gpt-5.5", mode="chat"),
)
assert body.listed == ("gpt-6-astra", "no-mode-model", "gpt-5.5")
assert body.left_out == ()
def test_body_leaves_out_the_entry_that_would_cross_the_byte_limit():
rows = tuple(_row(f"model-{index}") for index in range(6))
unlimited = _body(*rows, byte_limit=None)
limit = len(unlimited.json.encode()) - 1
body = _body(*rows, byte_limit=limit)
one_more = _body(*rows[: len(body.listed) + 1], byte_limit=None)
assert len(body.json.encode()) <= limit < len(one_more.json.encode())
assert body.listed + body.left_out == tuple(row.id for row in rows)
assert body.left_out == ("model-5",)
assert [entry["slug"] for entry in json.loads(body.json)["models"]] == list(body.listed)
def test_entries_offering_a_tier_survive_the_byte_limit_ahead_of_the_rest_and_keep_their_listing_place():
rows = (
_row("plain-first"),
_row("gpt-5.5"),
_row("plain-middle"),
_row("tiered-last", service_tiers=["ultrafast"]),
)
unlimited = _models(*rows, byte_limit=None)
two_entries_only = len(_body(rows[1], rows[3], byte_limit=None).json.encode())
body = _body(*rows, byte_limit=two_entries_only)
models = json.loads(body.json)["models"]
assert body.listed == ("gpt-5.5", "tiered-last") and body.left_out == ("plain-first", "plain-middle")
assert [entry["slug"] for entry in models] == ["gpt-5.5", "tiered-last"]
assert [entry["priority"] for entry in models] == [1, 3]
assert models == [unlimited[1], unlimited[3]]
assert len(body.json.encode()) <= two_entries_only
def test_an_entry_too_large_for_the_bytes_left_is_passed_over_and_the_smaller_ones_after_it_are_kept():
"""A tier description longer than the whole byte limit sorts first (tiered) and can never fit,
and a long display name fits an empty body but not what is left after the entries ahead of it."""
kept_rows = (
_row("plain-first"),
_row("tiered-small", service_tiers=["ultrafast"]),
_row("plain-last"),
)
limit = len(_body(*kept_rows, byte_limit=None).json.encode())
oversized_tier = {"id": "huge", "name": "Huge", "description": "x" * limit}
rows = (
kept_rows[0],
_row("tiered-oversized", service_tiers=[oversized_tier]),
kept_rows[1],
_row("plain-wide", display_name="w" * 200),
kept_rows[2],
)
unlimited = _models(*rows, byte_limit=None)
assert len(_body(rows[3], byte_limit=None).json.encode()) < limit
body = _body(*rows, byte_limit=limit)
models = json.loads(body.json)["models"]
assert body.listed == ("plain-first", "tiered-small", "plain-last")
assert body.left_out == ("tiered-oversized", "plain-wide")
assert models == [unlimited[0], unlimited[2], unlimited[4]]
assert [entry["priority"] for entry in models] == [0, 2, 4]
assert len(body.json.encode()) <= limit
def test_unlimited_body_keeps_every_entry():
rows = tuple(_row(f"model-{index}") for index in range(3))
body = _body(*rows, byte_limit=None)
assert (body.listed, body.left_out) == (tuple(row.id for row in rows), ())
assert CODEX_CATALOG_BYTE_LIMIT == 1024 * 1024
def test_vendored_codex_catalog_entries_decode_on_codex():
"""Every stock entry served keeps the `model_messages` Codex's `ModelsResponse` decoder
needs in place of `base_instructions`, and the catalog is read from the vendored file."""
stock = bundled_codex_models()
rows = tuple(CodexCatalogRow(id=slug, mode="responses") for slug in stock)
body = codex_models_response_json(rows, instructions=_PROMPT, byte_limit=None)
entries = json.loads(body.json)["models"]
assert len(entries) == len(stock) > 0
assert all(entry["model_messages"]["instructions_template"] for entry in entries)
assert all(entry["visibility"] == "list" and entry["supported_in_api"] is True for entry in entries)
assert all("base_instructions" not in entry for entry in entries)
def test_configured_service_tiers_without_known_tiers_names_strings_after_their_id():
assert configured_service_tiers((["ultrafast"],), "m") == (
CodexServiceTier(id="ultrafast", name="Ultrafast", description="Sends service_tier=ultrafast upstream"),
)
assert configured_service_tiers((None,), "m") is None
assert configured_service_tiers((), "m") is None
def test_catalog_rows_read_the_router_by_each_entry_lookup_id():
router = litellm.Router(
model_list=[
{
"model_name": "model_name_team-1_c0ffee",
"litellm_params": {"model": "openai/gpt-5.5"},
"model_info": {"display_name": "Team 5.5", "service_tiers": ["ultrafast"]},
},
{
"model_name": "model_name_team-1_c0ffee",
"litellm_params": {"model": "openai/gpt-5.5", "api_base": "https://second.example"},
},
{"model_name": "plain", "litellm_params": {"model": "openai/some-unmapped-model"}},
]
)
listing = (
{
"id": "gpt-5.5-team",
"object": "model",
"created": 0,
"owned_by": "openai",
"mode": "chat",
"max_input_tokens": 7,
},
{"id": "plain", "object": "model", "created": 0, "owned_by": "openai"},
)
rows = codex_catalog_rows(listing, (("gpt-5.5-team", "model_name_team-1_c0ffee"), ("plain", "plain")), router)
assert rows == (
CodexCatalogRow(
id="gpt-5.5-team",
mode="chat",
max_input_tokens=7,
upstream_model="openai/gpt-5.5",
display_name="Team 5.5",
service_tiers=(["ultrafast"], None),
),
CodexCatalogRow(id="plain", upstream_model="openai/some-unmapped-model", service_tiers=(None,)),
)
def test_catalog_rows_read_an_alias_off_its_target_under_the_alias_name():
router = litellm.Router(
model_list=[
{
"model_name": "gpt-6-astra",
"litellm_params": {"model": "openai/gpt-6-astra"},
"model_info": {"display_name": "Astra", "service_tiers": ["ultrafast"]},
}
],
model_group_alias={"gpt-6": "gpt-6-astra"},
)
listing = (
{"id": "gpt-6", "object": "model", "created": 0, "owned_by": "openai", "mode": "chat", "max_input_tokens": 9},
)
(row,) = codex_catalog_rows(listing, (("gpt-6", "gpt-6"),), router)
assert row == CodexCatalogRow(
id="gpt-6", mode="chat", max_input_tokens=9, upstream_model="openai/gpt-6-astra", service_tiers=(["ultrafast"],)
)
(entry,) = json.loads(codex_model_list_body(listing, (("gpt-6", "gpt-6"),), router))["models"]
assert (entry["slug"], entry["display_name"], [tier["id"] for tier in entry["service_tiers"]]) == (
"gpt-6",
"gpt-6",
["ultrafast"],
)
assert entry["model_messages"] == bundled_codex_models()["gpt-6-astra"].model_dump(mode="json")["model_messages"]
def test_model_list_body_offers_a_tier_only_off_the_deployments_the_key_team_can_route_to():
router = litellm.Router(
model_list=[
{
"model_name": "gpt-6-astra",
"litellm_params": {"model": "openai/gpt-6-astra"},
"model_info": {"service_tiers": ["ultrafast"]},
},
{
"model_name": "gpt-6-astra",
"litellm_params": {"model": "openai/gpt-6-astra", "api_base": "https://team-2.example"},
"model_info": {"team_id": "team-2"},
},
]
)
listing = ({"id": "gpt-6-astra", "object": "model", "created": 0, "owned_by": "openai", "mode": "chat"},)
entries = (("gpt-6-astra", "gpt-6-astra"),)
def offered(team_id):
(entry,) = json.loads(codex_model_list_body(listing, entries, router, team_id))["models"]
return [tier["id"] for tier in entry["service_tiers"]]
assert offered("team-1") == ["ultrafast"]
assert offered("team-2") == []
assert offered(None) == ["ultrafast"]
def test_model_list_body_takes_stock_metadata_off_the_deployment_the_key_team_routes_to():
"""Two teams own a deployment of one name: Codex's stock gpt-5.5 entry goes only to the team whose
requests reach gpt-5.5, under the name and under a `model_group_alias` of it."""
router = litellm.Router(
model_list=[
{
"model_name": "coding-model",
"litellm_params": {"model": "openai/gpt-5.5"},
"model_info": {"team_id": "team-1"},
},
{
"model_name": "coding-model",
"litellm_params": {"model": "openai/some-unmapped-model"},
"model_info": {"team_id": "team-2", "service_tiers": ["flex"]},
},
],
model_group_alias={"coding": "coding-model"},
)
listing = tuple(
{"id": name, "object": "model", "created": 0, "owned_by": "openai", "mode": "chat"}
for name in ("coding-model", "coding")
)
entries = (("coding-model", "coding-model"), ("coding", "coding"))
stock = bundled_codex_models()["gpt-5.5"].model_dump(mode="json")
def served(team_id):
return json.loads(codex_model_list_body(listing, entries, router, team_id))["models"]
for entry in served("team-1"):
assert entry["model_messages"] == stock["model_messages"]
assert entry["supported_reasoning_levels"] == stock["supported_reasoning_levels"] != []
assert entry["service_tiers"] == stock["service_tiers"] != []
for entry in served("team-2"):
assert "model_messages" not in entry and entry["base_instructions"]
assert entry["supported_reasoning_levels"] == []
assert [tier["id"] for tier in entry["service_tiers"]] == ["flex"]
assert [entry["slug"] for entry in served("team-2")] == ["coding-model", "coding"]
def test_catalog_rows_without_a_router_carry_only_the_listing():
listing = ({"id": "plain", "object": "model", "created": 0, "owned_by": "openai", "mode": "chat"},)
assert codex_catalog_rows(listing, (("plain", "plain"),), None) == (CodexCatalogRow(id="plain", mode="chat"),)

View file

@ -888,6 +888,17 @@ def test_token_refresh_params_keep_the_prisma_tls_dialect_but_not_the_schema():
}
def test_token_refresh_params_keep_the_options_carrying_the_server_timeouts() -> None:
kept: Final = token_refresh_params_from_url(
"postgresql://u:TOKEN@db.example.com:5432/litellm_db"
"?schema=tenant&connection_limit=5&options=-c%20statement_timeout%3D5000%20-c%20idle_in_transaction_session_timeout%3D60000"
)
assert dict(kept) == {
"connection_limit": "5",
"options": "-c statement_timeout=5000 -c idle_in_transaction_session_timeout=60000",
}
def _issue_cert(
subject: str, issuer: x509.Certificate | None, issuer_key: ec.EllipticCurvePrivateKey | None, ca: bool
) -> tuple[x509.Certificate, ec.EllipticCurvePrivateKey]:
@ -1199,3 +1210,42 @@ def test_reader_keeps_its_own_pinned_idle_lifetime(monkeypatch):
assert os.environ["DATABASE_URL_READ_REPLICA"] == (
"postgresql://u:p@reader.example.com:5432/db?max_idle_connection_lifetime=120"
)
@pytest.mark.parametrize(
"writer_limit, reader_limit, num_workers, expected",
[
("10", None, "4", "4 worker(s) x writer connection_limit 10 = up to 40 connections"),
(
"10",
"10",
"4",
"4 worker(s) x (writer connection_limit 10 + reader connection_limit 10) = up to 80 connections",
),
(
"10",
"50",
"4",
"4 worker(s) x (writer connection_limit 10 + reader connection_limit 50) = up to 240 connections",
),
("10", None, "0", "1 worker(s) x writer connection_limit 10 = up to 10 connections"),
],
ids=["writer_only", "reader_doubles_engines", "reader_pins_its_own_limit", "worker_floor"],
)
def test_connection_budget_message_sums_the_engines_each_worker_owns(
writer_limit: str, reader_limit: str | None, num_workers: str, expected: str
) -> None:
from litellm.proxy.db.db_url_settings import postgres_connection_budget_message
message: Final = postgres_connection_budget_message(
writer_limit=writer_limit, reader_limit=reader_limit, num_workers=num_workers
)
assert expected in message
assert "max_connections minus superuser_reserved_connections" in message
def test_connection_budget_message_survives_an_unparseable_limit() -> None:
from litellm.proxy.db.db_url_settings import postgres_connection_budget_message
message: Final = postgres_connection_budget_message(writer_limit="ten", reader_limit=None, num_workers="2")
assert "writer='ten'" in message

View file

@ -8,7 +8,7 @@ import httpx
import pytest
from fastapi import HTTPException, Request
from prisma import errors as prisma_errors
from prisma.engine.errors import BinaryNotFoundError, EngineConnectionError
from prisma.engine.errors import BinaryNotFoundError, EngineConnectionError, EngineRequestError
from prisma.errors import (
ClientNotConnectedError,
DataError,
@ -789,3 +789,159 @@ def test_db_lookup_deadline_is_a_connection_and_unavailability_error_but_never_a
assert PrismaDBExceptionHandler.is_database_service_unavailable_error(deadline) is True
assert PrismaDBExceptionHandler.is_database_transport_error(deadline) is False
assert "temporarily unreachable" in PrismaDBExceptionHandler.database_unavailable_message(deadline)
_TOO_MANY_CLIENTS: Final = "Error in connector: Error querying the database: FATAL: sorry, too many clients already"
@pytest.mark.parametrize(
"error",
[
DataError(data={"user_facing_error": {"message": _TOO_MANY_CLIENTS}}),
DataError(
data={
"user_facing_error": {
"message": "Error querying the database: FATAL: zu viele Verbindungen",
"meta": {"code": "53300", "message": "zu viele Verbindungen"},
}
}
),
DataError(
data={
"user_facing_error": {
"message": "Error in connector: Error querying the database: FATAL: remaining connection slots are reserved for roles with the SUPERUSER attribute"
}
}
),
DataError(
data={
"user_facing_error": {
"message": 'Error occurred during query execution: ConnectorError(ConnectorError { user_facing_error: None, kind: QueryError(PostgresError { code: "53300", message: "too many connections for role \\"litellm\\"", severity: "FATAL" }) })'
}
}
),
DataError(
data={
"user_facing_error": {
"message": 'Error in connector: Error querying the database: FATAL: too many connections for role "litellm"'
}
}
),
DataError(
data={
"user_facing_error": {
"message": 'Error in connector: Error querying the database: FATAL: too many connections for database "litellm"'
}
}
),
],
)
def test_postgres_connection_capacity_refusal_is_service_unavailable_not_a_data_error(error: DataError) -> None:
"""Postgres refusing a new connection (SQLSTATE 53300) reaches the proxy as a
bare ``DataError`` with no SQLSTATE in ``meta``. The server is up but full, so
the failure is service-unavailable (the spend-log flush re-raises and requeues
instead of bisecting the batch row by row against a full server) while not a
transport error, which would make auth and the health check tear the engine
down and open yet more connections against it."""
assert PrismaDBExceptionHandler.is_database_capacity_error(error) is True
assert PrismaDBExceptionHandler.is_database_service_unavailable_error(error) is True
assert PrismaDBExceptionHandler.is_database_transport_error(error) is False
assert PrismaDBExceptionHandler.is_prisma_data_error(error) is True
@pytest.mark.parametrize(
"error",
[
DataError(data={"user_facing_error": {"message": "invalid byte sequence for encoding UTF8: 0x00"}}),
UniqueViolationError(data={"user_facing_error": {"error_code": "P2002", "meta": {"table": "t"}}}),
PrismaError("can't reach database server"),
httpx.ConnectError("connection refused"),
RuntimeError(_TOO_MANY_CLIENTS),
],
)
def test_is_database_capacity_error_excludes_other_failures(error: Exception) -> None:
assert PrismaDBExceptionHandler.is_database_capacity_error(error) is False
def _capacity_error(message: str) -> DataError:
"""The shape prisma-client-py raises when Postgres refuses the session: the
connector message with no SQLSTATE in ``meta`` and no P-code, so it falls
through ``handle_response_errors`` to the base ``DataError``."""
return DataError(
data={
"error": f"Error occurred during query execution:\nConnectorError(ConnectorError {{ user_facing_error: None, kind: QueryError({message}) }})",
"user_facing_error": {
"is_panic": False,
"message": f"Error in connector: Error querying the database: FATAL: {message}",
"backtrace": None,
},
}
)
def _pool_timeout_error() -> DataError:
return DataError(
data={
"error": "Error in connector: Error creating a database connection. (Timed out fetching a connection from the pool (connection limit: 10, in use: 10, pool timeout 60))",
"user_facing_error": {
"is_panic": False,
"message": "Timed out fetching a new connection from the connection pool. More info: http://pris.ly/d/connection-pool (Current connection pool timeout: 60, connection limit: 10)",
"meta": {"connection_limit": 10, "timeout": 60},
"error_code": "P2024",
},
}
)
@pytest.mark.parametrize(
"error",
[
_capacity_error("sorry, too many clients already"),
_pool_timeout_error(),
EngineRequestError(
MagicMock(status=500),
'{"is_panic":false,"message":"Error in connector: Error querying the database: FATAL: sorry, too many clients already","backtrace":null}',
),
RawQueryError(
data={
"user_facing_error": {
"message": 'Raw query failed. Code: `53300`. Message: `db error: FATAL: sorry, too many clients already`',
"meta": {"code": "53300", "message": "FATAL: sorry, too many clients already"},
"error_code": "P2010",
}
}
),
],
ids=["53300_connector_dataerror", "P2024_pool_timeout", "engine_500", "raw_query_53300"],
)
def test_is_database_capacity_error_recognises_postgres_and_pool_exhaustion(error: Exception) -> None:
"""SQLSTATE 53300 and prisma P2024 mean the statement was never sent because
the deployment is over its connection budget. Both are infrastructure
failures (503, never 401), neither is a reason to recreate the engine (that
opens another pool against a server that is already full), and both must
reach the spend-log writer as "requeue", not "bisect"."""
assert PrismaDBExceptionHandler.is_database_capacity_error(error) is True
assert PrismaDBExceptionHandler.is_database_service_unavailable_error(error) is True
assert PrismaDBExceptionHandler.is_database_connection_error(error) is True
assert PrismaDBExceptionHandler.is_database_transport_error(error) is False
assert PrismaDBExceptionHandler.is_permanent_database_fault(error) is False
@pytest.mark.parametrize(
"error",
[
PrismaError("timed out while connecting"),
PrismaError("can't reach database server"),
DataError(data={"user_facing_error": {"message": "Can't reach database server at `127.0.0.1`:`5499`"}}),
httpx.ConnectError("conn refused"),
],
)
def test_capacity_check_leaves_reachability_failures_on_the_reconnect_path(error: Exception) -> None:
assert PrismaDBExceptionHandler.is_database_capacity_error(error) is False
assert PrismaDBExceptionHandler.is_database_transport_error(error) is True
def test_capacity_error_is_seen_through_a_wrapping_exception() -> None:
wrapped: Final = RuntimeError("spend flush failed")
wrapped.__cause__ = _capacity_error("sorry, too many clients already")
assert PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(wrapped) is True

View file

@ -1,13 +1,24 @@
from datetime import datetime, timedelta, timezone
from typing import Final
import pytest
from fastapi import HTTPException
from pydantic import ValidationError
import litellm
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm import Router
from litellm.proxy.lens.endpoints import list_agents, user_scope, validate_model, worker_supports_model
from litellm.proxy.lens.models import LensSettings
from litellm.proxy.lens.endpoints import (
list_agents,
run_settings,
run_window,
user_scope,
watchable,
watching,
validate_model,
worker_supports_model,
)
from litellm.proxy.lens.models import Lens, LensSettings, RunRequest, Scope
@pytest.fixture
@ -167,6 +178,79 @@ def test_regular_keys_cannot_read_lens_results(role: LitellmUserRoles | None) ->
assert error.value.status_code == 403
def saved_lens() -> Lens:
now: Final = datetime(2026, 1, 15, tzinfo=timezone.utc)
return Lens(
id="lens",
scope=Scope(all_teams=True),
settings=LensSettings(name="Support", model="analysis", context="Answer questions", agent_name="support"),
created_at=now,
next_run_at=now,
budget_month="2026-01",
)
def test_run_now_agent_override_only_changes_the_agent_for_that_run() -> None:
lens: Final = saved_lens()
overridden: Final = run_settings(lens, RunRequest(agent_name="billing"))
assert overridden is not None
assert overridden.agent_name == "billing"
assert overridden.model_copy(update={"agent_name": "support"}) == lens.settings
def test_run_now_without_overrides_keeps_the_saved_settings() -> None:
assert run_settings(saved_lens(), RunRequest()) is None
def test_run_now_rejects_a_window_that_is_missing_an_edge_or_backwards() -> None:
now: Final = datetime(2026, 1, 15, tzinfo=timezone.utc)
with pytest.raises(ValidationError, match="both a start and an end"):
RunRequest(start=now)
with pytest.raises(ValidationError, match="before end"):
RunRequest(start=now, end=now - timedelta(hours=1))
def test_watching_switches_a_paused_investigation_on_and_records_the_change() -> None:
paused: Final = saved_lens().model_copy(
update={"settings": saved_lens().settings.model_copy(update={"enabled": False})}
)
watched: Final = watching(paused)
assert watched.settings.enabled is True
assert watched.revision == paused.revision + 1
assert watched.settings.model_copy(update={"enabled": False}) == paused.settings
def test_watching_leaves_an_investigation_that_is_already_on_untouched() -> None:
on: Final = saved_lens()
assert watching(on) is on
async def test_watch_all_skips_an_investigation_whose_model_is_gone_instead_of_failing_them_all(
analysis_router: Router,
) -> None:
stale: Final = saved_lens().model_copy(
update={"settings": saved_lens().settings.model_copy(update={"model": "retired-model", "enabled": False})}
)
skipped: Final = await watchable(stale, UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN))
assert skipped is not None
assert skipped.id == stale.id
assert skipped.reason
def test_run_now_since_last_run_keeps_scanning_only_new_traces_even_with_an_agent_override() -> None:
now: Final = datetime(2026, 1, 15, 12, tzinfo=timezone.utc)
resumed: Final = saved_lens().model_copy(update={"last_scan_at": now - timedelta(hours=1)})
window: Final = run_window(resumed, RunRequest(agent_name="billing"), now)
assert window is not None
assert window[0] == now - timedelta(hours=1)
def test_run_now_with_a_lookback_scans_that_lookback_instead_of_since_last_run() -> None:
now: Final = datetime(2026, 1, 15, 12, tzinfo=timezone.utc)
resumed: Final = saved_lens().model_copy(update={"last_scan_at": now - timedelta(hours=1)})
assert run_window(resumed, RunRequest(lookback_hours=24), now) is None
@pytest.mark.parametrize("provider", (False, True))
def test_model_errors_reach_worker_with_status_and_redacted_provider_message(provider: bool) -> None:
import httpx

View file

@ -4,7 +4,8 @@ import pytest
from fastapi import HTTPException
import litellm
from litellm.proxy.lens.inference import Deployment, DeploymentParams, completion_charge, quote
from litellm.proxy.lens.inference import Deployment, DeploymentParams, completion_charge, model_step, quote
from litellm.proxy.lens.models import ModelRequest
from litellm.types.utils import ModelResponse
@ -120,3 +121,17 @@ def test_unknown_model_capacity_requires_explicit_operator_metadata() -> None:
assert "model_info.max_output_tokens" in error.value.detail
configured: Final = Deployment(litellm_params=params, model_info=ModelCapacity(max_output_tokens=32000))
assert output_tokens(configured) == 32000
def test_a_model_step_records_the_serving_model_and_its_tokens() -> None:
response: Final = ModelResponse(model="gpt-5.6", usage={"prompt_tokens": 1200, "completion_tokens": 80})
step: Final = model_step(response, ModelRequest(prompt="review", purpose="extract"), "analysis", 0.02)
assert (step.model, step.prompt_tokens, step.completion_tokens, step.cost) == ("gpt-5.6", 1200, 80, 0.02)
def test_a_response_without_usage_still_records_a_step_instead_of_failing_settlement() -> None:
response: Final = ModelResponse(model="gpt-5.6")
unpriced: Final = response.model_copy(update={"usage": None})
step: Final = model_step(unpriced, ModelRequest(prompt="review", purpose="cluster"), "analysis", 0.0)
assert (step.prompt_tokens, step.completion_tokens) == (0, 0)
assert step.label == "Compared observations"

View file

@ -1,9 +1,11 @@
from datetime import datetime, timedelta, timezone
from functools import reduce
from typing import Final
import pytest
from litellm.proxy.lens.models import (
MAX_STEPS,
AgentTestCase,
Check,
Evidence,
@ -12,9 +14,19 @@ from litellm.proxy.lens.models import (
Lens,
LensSettings,
Scope,
Step,
Worker,
)
from litellm.proxy.lens.state import can_access, claim_job, current_job, merge_finding, queue_job, renew_budget
from litellm.proxy.lens.state import (
add_step,
can_access,
claim_job,
current_job,
merge_finding,
next_scan_start,
queue_job,
renew_budget,
)
NOW: Final = datetime(2026, 1, 15, tzinfo=timezone.utc)
@ -163,15 +175,42 @@ def test_monthly_budget_renews_without_erasing_job_costs() -> None:
@pytest.mark.parametrize("hours", (24, 168, 720, 4800, 8760))
def test_every_scan_uses_the_configured_lookback_window(hours: int) -> None:
def test_first_scan_covers_the_configured_lookback_window(hours: int) -> None:
original: Final = lens()
configured: Final = original.model_copy(
update={"settings": LensSettings.model_validate({**original.settings.model_dump(), "lookback_hours": hours})}
)
first: Final = queue_job(configured, NOW, "first")
assert first.jobs[0].start == NOW - timedelta(hours=hours)
resumed: Final = configured.model_copy(update={"last_scan_at": NOW - timedelta(hours=1)})
assert queue_job(resumed, NOW, "next").jobs[0].start == NOW - timedelta(hours=hours)
assert first.jobs[0].trigger == "schedule"
def test_later_scheduled_scans_only_cover_traces_since_the_last_scan() -> None:
resumed: Final = lens().model_copy(update={"last_scan_at": NOW - timedelta(hours=1)})
job: Final = queue_job(resumed, NOW, "next").jobs[0]
assert job.start == NOW - timedelta(hours=1)
assert job.end == NOW - timedelta(minutes=2)
def test_a_scan_after_a_long_outage_never_reaches_past_the_lookback_window() -> None:
stale: Final = lens().model_copy(update={"last_scan_at": NOW - timedelta(days=400)})
assert queue_job(stale, NOW, "next").jobs[0].start == NOW - timedelta(hours=stale.settings.lookback_hours)
def test_run_now_with_an_exact_window_scans_that_window_and_is_marked_manual() -> None:
window: Final = (NOW - timedelta(hours=5), NOW - timedelta(hours=3))
job: Final = queue_job(lens(), NOW, "manual", window=window, trigger="manual").jobs[0]
assert (job.start, job.end) == window
assert job.trigger == "manual"
def test_steps_keep_only_the_most_recent_entries() -> None:
job: Final = queue_job(lens(), NOW, "job").jobs[0]
steps: Final = tuple(Step(at=NOW, kind="stage", label=f"step {i}") for i in range(MAX_STEPS + 5))
grown: Final = reduce(add_step, steps, job)
assert len(grown.steps) == MAX_STEPS
assert grown.steps[0].label == "step 5"
assert grown.steps[-1].label == f"step {MAX_STEPS + 4}"
def test_finding_keeps_uncertainty_separate_from_the_main_summary() -> None:
@ -287,6 +326,17 @@ def test_legacy_finding_identity_preserves_feedback_only_for_same_kind_and_check
assert separate.status == "open" and separate.reason == ""
def test_only_successful_scheduled_scans_move_the_next_scan_forward() -> None:
previous: Final = lens().model_copy(update={"last_scan_at": NOW - timedelta(hours=3)})
scheduled: Final = queue_job(previous, NOW, "scheduled").jobs[0]
manual: Final = queue_job(
previous, NOW, "manual", window=(NOW - timedelta(hours=2), NOW - timedelta(hours=1)), trigger="manual"
).jobs[0]
assert next_scan_start(previous, scheduled, failed=False) == scheduled.end
assert next_scan_start(previous, scheduled, failed=True) == previous.last_scan_at
assert next_scan_start(previous, manual, failed=False) == previous.last_scan_at
@pytest.mark.parametrize("field", ("lookback_hours", "interval_minutes"))
def test_calendar_overflow_is_rejected_without_the_old_history_and_interval_caps(field: str) -> None:
from pydantic import ValidationError

View file

@ -46,7 +46,9 @@ def patched_models(monkeypatch):
deployment = MagicMock()
deployment.litellm_params.model = "gpt-4"
router.get_deployment_by_model_group_name = MagicMock(return_value=deployment)
router.get_routable_upstream_model = MagicMock(return_value="gpt-4")
router.get_configured_display_name = MagicMock(return_value=None)
router.get_configured_service_tiers = MagicMock(return_value=())
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "prisma_client", MagicMock())
@ -397,3 +399,118 @@ def test_anthropic_format_keeps_served_ids_for_other_anthropic_clients(client, a
assert response.status_code == 200
assert [m["id"] for m in response.json()["data"]] == ["gpt-4", "claude-sonnet"]
@pytest.mark.parametrize("path", ["/v1/models", "/models"])
@pytest.mark.parametrize("params", [{}, {"scope": "expand"}])
def test_codex_format_when_client_version_present(client, auth_as, patched_models, path, params):
"""Codex CLI fetches a provider's catalog as ``GET /v1/models?client_version=<its version>`` and
decodes Codex's own ``{"models": [...]}`` shape; the same request without the parameter keeps the
OpenAI shape byte for byte."""
with auth_as():
codex_response = client.get(path, params={**params, "client_version": "0.159.3"})
openai_response = client.get(path, params=params)
assert codex_response.status_code == 200
assert codex_response.headers["content-type"] == "application/json"
body = codex_response.json()
assert list(body) == ["models"]
assert [(m["slug"], m["display_name"], m["priority"]) for m in body["models"]] == [
("gpt-4", "gpt-4", 0),
("claude-sonnet", "claude-sonnet", 1),
]
assert all(m["base_instructions"] and m["visibility"] == "list" for m in body["models"])
assert openai_response.status_code == 200
assert normalize(openai_response.json()) == {
"data": [
{"id": "<VOLATILE>", "object": "model", "created": "<VOLATILE>", "owned_by": "openai"},
{"id": "<VOLATILE>", "object": "model", "created": "<VOLATILE>", "owned_by": "openai"},
],
"object": "list",
}
@pytest.mark.parametrize("path", ["/v1/models", "/models"])
def test_codex_format_wins_over_the_anthropic_header(client, auth_as, patched_models, path):
with auth_as():
response = client.get(path, params={"client_version": "0.159.3"}, headers={"anthropic-version": "2023-06-01"})
assert response.status_code == 200
assert list(response.json()) == ["models"]
@pytest.mark.parametrize("path", ["/v1/models", "/models"])
def test_codex_format_carries_configured_service_tiers(client, auth_as, patched_models, path):
"""A deployment's ``model_info.service_tiers`` becomes the entry's ``service_tiers``, which Codex
offers as slash commands; a model without one offers none, and the OpenAI shape gains no field."""
patched_models.get_configured_service_tiers = MagicMock(
side_effect=lambda model_name, team_id=None: (["ultrafast"],) if model_name == "gpt-4" else (None,)
)
with auth_as():
codex_response = client.get(path, params={"client_version": "0.159.3"})
openai_response = client.get(path)
gpt_4, claude = codex_response.json()["models"]
assert gpt_4["service_tiers"] == [
{"id": "ultrafast", "name": "Ultrafast", "description": "Sends service_tier=ultrafast upstream"}
]
assert claude["service_tiers"] == []
assert all("service_tiers" not in m for m in openai_response.json()["data"])
@pytest.mark.parametrize("params", [{}, {"scope": "expand"}])
def test_codex_service_tiers_are_read_for_the_key_team(client, auth_as, patched_models, params):
"""A tier and the upstream model that picks Codex's stock entry are read off the deployments the
key's team can route to, so both listing paths hand the router the key's team, and no team for a
key without one."""
patched_models.get_configured_service_tiers = MagicMock(
side_effect=lambda model_name, team_id=None: (["ultrafast"],) if team_id == "team-1" else (None,)
)
patched_models.get_routable_upstream_model = MagicMock(
side_effect=lambda model_name, team_id=None: "openai/gpt-5.5" if team_id == "team-1" else "gpt-4"
)
with auth_as(team_id="team-1"):
team_response = client.get("/v1/models", params={**params, "client_version": "0.159.3"})
with auth_as():
teamless_response = client.get("/v1/models", params={**params, "client_version": "0.159.3"})
assert [[t["id"] for t in m["service_tiers"]] for m in team_response.json()["models"]] == [["ultrafast"]] * 2
assert [m["service_tiers"] for m in teamless_response.json()["models"]] == [[], []]
assert all(m["supported_reasoning_levels"] for m in team_response.json()["models"])
assert [m["supported_reasoning_levels"] for m in teamless_response.json()["models"]] == [[], []]
@pytest.mark.parametrize("params", [{}, {"scope": "expand"}])
def test_codex_service_tiers_resolved_via_internal_team_key(client, auth_as, patched_models, monkeypatch, params):
"""A team-scoped row's tiers are looked up by the internal routing key while the entry is keyed by
the public name Codex sends back as the model."""
from litellm.proxy import utils as proxy_utils
from litellm.proxy.auth import model_checks
internal_name = "model_name_team-1_c0ffee"
patched_models.get_model_list = MagicMock(
return_value=[
{"model_name": internal_name, "model_info": {"team_id": "team-1", "team_public_model_name": "gpt-4-team"}}
]
)
patched_models.get_model_names = MagicMock(return_value=[internal_name])
patched_models.get_configured_service_tiers = MagicMock(
side_effect=lambda model_name, team_id=None: (["ultrafast"],) if model_name == internal_name else ()
)
async def _fake_get_available_models_for_user(**kwargs):
return [internal_name]
monkeypatch.setattr(proxy_utils, "get_available_models_for_user", _fake_get_available_models_for_user)
monkeypatch.setattr(model_checks, "get_complete_model_list", lambda **kwargs: [internal_name])
with auth_as():
response = client.get("/v1/models", params={**params, "client_version": "0.159.3"})
assert response.status_code == 200
(entry,) = response.json()["models"]
assert (entry["slug"], [tier["id"] for tier in entry["service_tiers"]]) == ("gpt-4-team", ["ultrafast"])

View file

@ -3,6 +3,7 @@ import os
from contextlib import nullcontext
from pathlib import Path
from types import SimpleNamespace
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import click
@ -2964,6 +2965,32 @@ class TestPostgresStatementTimeoutOptions:
assert _pg_options_with_timeouts(existing, statement_timeout, lock_timeout) == expected
@pytest.mark.parametrize(
"existing, idle_timeout, expected",
[
("", 30, "-c statement_timeout=60000 -c lock_timeout=15000 -c idle_in_transaction_session_timeout=30000"),
("", None, "-c statement_timeout=60000 -c lock_timeout=15000"),
(
"-c idle_in_transaction_session_timeout=5000",
30,
"-c idle_in_transaction_session_timeout=5000 -c statement_timeout=60000 -c lock_timeout=15000",
),
],
ids=["idle_set", "idle_unset", "pinned_idle_wins"],
)
def test_pg_options_with_idle_in_transaction_timeout(
self,
existing: str,
idle_timeout: int | None,
expected: str,
) -> None:
"""A transaction that opened and then stalled holds its connection and its
locks for as long as the client stays silent; ``idle_in_transaction_session_timeout``
is the only server-side bound on that, so it rides the same ``options`` string."""
from litellm.proxy.proxy_cli import _pg_options_with_timeouts
assert _pg_options_with_timeouts(existing, 60, 15, idle_timeout) == expected
def test_timeouts_reach_the_database_url_from_general_settings(self, tmp_path):
"""The whole point of the setting: it has to land on DATABASE_URL."""
import yaml
@ -2976,6 +3003,7 @@ class TestPostgresStatementTimeoutOptions:
"general_settings": {
"database_statement_timeout": 60,
"database_lock_timeout": 15,
"database_idle_in_transaction_session_timeout": 30,
},
}
)
@ -2986,6 +3014,7 @@ class TestPostgresStatementTimeoutOptions:
options = urlparse.parse_qs(urlparse.urlparse(modified_url).query)["options"][0]
assert "-c statement_timeout=60000" in options
assert "-c lock_timeout=15000" in options
assert "-c idle_in_transaction_session_timeout=30000" in options
def test_no_options_param_when_unset(self, tmp_path):
"""Unset must mean today's behavior, not an empty options string."""
@ -3066,6 +3095,7 @@ def _run_server_and_capture_urls(
database_url: str = "postgresql://t:t@localhost:5432/t",
direct_url: str | None = None,
read_replica_url: str | None = None,
extra_args: tuple[str, ...] = (),
) -> dict:
loaded_config = yaml.safe_load(Path(config_path).read_text())
mock_proxy_config = MagicMock()
@ -3098,7 +3128,7 @@ def _run_server_and_capture_urls(
patch("litellm.proxy.db.check_migration.check_prisma_schema_diff"),
):
run_server.main(
["--config", config_path, "--local", "--skip_server_startup"],
["--config", config_path, "--local", "--skip_server_startup", *extra_args],
standalone_mode=False,
)
return {k: os.environ[k] for k in _CAPTURED_DB_ENV_VARS if k in os.environ}
@ -3172,6 +3202,47 @@ class TestReadReplicaConnectionParams:
assert query["connection_limit"] == ["50"]
assert query["pool_timeout"] == ["20"]
def test_connection_budget_line_counts_the_limits_the_final_urls_carry(
self,
tmp_path: Path,
capsys: pytest.CaptureFixture[str],
) -> None:
import yaml
config_path: Final = tmp_path / "config.yaml"
config_path.write_text(
yaml.dump({"model_list": [], "general_settings": {"database_connection_pool_limit": 3}})
)
_run_server_and_capture_urls(
str(config_path),
read_replica_url="postgresql://t:t@reader:5432/t?connection_limit=50",
)
assert (
"1 worker(s) x (writer connection_limit 3 + reader connection_limit 50) = up to 53 connections"
in capsys.readouterr().out
)
def test_connection_budget_line_counts_one_worker_under_hypercorn(
self,
tmp_path: Path,
capsys: pytest.CaptureFixture[str],
) -> None:
import yaml
config_path: Final = tmp_path / "config.yaml"
config_path.write_text(
yaml.dump({"model_list": [], "general_settings": {"database_connection_pool_limit": 3}})
)
_run_server_and_capture_urls(
str(config_path),
extra_args=("--run_hypercorn", "--num_workers", "4"),
)
assert "1 worker(s) x writer connection_limit 3 = up to 3 connections" in capsys.readouterr().out
def test_extra_connection_params_never_carry_a_schema_override_to_the_reader(self, tmp_path):
"""database_extra_connection_params is an untyped passthrough, so it can carry a
search_path. The writer keeps it, the reader must not inherit it, or replica

View file

@ -10,8 +10,8 @@ from __future__ import annotations
import asyncio
import json
from collections.abc import Iterator
from typing import Any, Dict, List
from collections.abc import Callable, Iterator
from typing import Any, Dict, Final, List
from unittest.mock import AsyncMock, MagicMock
import pytest
@ -917,3 +917,38 @@ async def test_update_spend_logs_parks_failed_batch_in_redis_with_wire_safe_date
parked = await buffer.get_spend_logs_from_redis_buffer(limit=10)
assert mock_prisma_client.spend_log_transactions == []
assert [(row["request_id"], row["startTime"]) for row in parked] == [("a", started.isoformat())]
@pytest.mark.asyncio
async def test_update_spend_logs_requeues_batch_when_postgres_is_out_of_connections(
mock_prisma_client: MagicMock,
make_spend_log_row: Callable[..., Dict[str, Any]],
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Postgres refusing the pool a new connection (SQLSTATE 53300, "too many
clients already") surfaces as a bare ``DataError``. It is the server being
full, not a row being bad: the flush must stop after the one failed insert
and put the batch back at the head of the queue for the next interval,
rather than bisecting it (hundreds more connection attempts against a full
server) and dropping every row as poisoned."""
sleep: Final = AsyncMock(return_value=None)
monkeypatch.setattr(utils_mod.asyncio, "sleep", sleep)
err: Final = _data_error("Error in connector: Error querying the database: FATAL: sorry, too many clients already")
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=err)
proxy_logging: Final = MagicMock()
proxy_logging.failure_handler = AsyncMock()
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="e")]
logs: Final = [make_spend_log_row(request_id=f"r{i}") for i in range(4)]
with pytest.raises(type(err)):
await ProxyUpdateSpend.update_spend_logs(
n_retry_times=2,
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
logs_to_process=logs,
)
assert mock_prisma_client.db.litellm_spendlogs.create_many.await_count == 1
sleep.assert_not_awaited()
assert [row["request_id"] for row in mock_prisma_client.spend_log_transactions] == ["r0", "r1", "r2", "r3", "e"]

View file

@ -12,7 +12,9 @@ from __future__ import annotations
import asyncio
import json
from collections.abc import Callable
from contextlib import suppress
from datetime import datetime, timezone
from typing import Any, Dict, Final, List
from unittest.mock import AsyncMock, MagicMock
@ -968,3 +970,119 @@ async def test_monitor_spend_logs_queue_pulls_parked_rows_before_each_flush(
)
assert seen == [["parked"]]
def _postgres_out_of_connections() -> Exception:
"""prisma's shape for Postgres SQLSTATE 53300: a base ``DataError`` whose only
hint is the connector message."""
from prisma.errors import DataError
return DataError(
data={
"user_facing_error": {
"is_panic": False,
"message": "Error in connector: Error querying the database: FATAL: sorry, too many clients already",
"backtrace": None,
}
}
)
@pytest.mark.asyncio
async def test_update_spend_logs_job_requeues_whole_batch_when_postgres_is_out_of_connections(
mock_prisma_client: MagicMock,
make_spend_log_row: Callable[..., dict[str, object]],
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""53300 is not a poison row: bisecting it would issue one failing statement
per row (each a fresh connection attempt against a full server) and drop
every row. The batch goes back to the queue head untouched, in one attempt,
without the in-job retry loop hammering the server."""
sleeps: Final[list[float]] = []
async def _no_sleep(seconds: float, *_: object, **__: object) -> None:
sleeps.append(seconds)
monkeypatch.setattr(asyncio, "sleep", _no_sleep)
proxy_logging: Final = MagicMock()
proxy_logging.failure_handler = AsyncMock()
mock_prisma_client.spend_log_transactions = [
make_spend_log_row(request_id="r1"),
make_spend_log_row(request_id="r2"),
make_spend_log_row(request_id="r3"),
]
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_postgres_out_of_connections())
with pytest.raises(Exception, match="too many clients already"):
await update_spend_logs_job(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=proxy_logging,
)
assert {
"create_many_calls": mock_prisma_client.db.litellm_spendlogs.create_many.await_count,
"queue_after": [row["request_id"] for row in mock_prisma_client.spend_log_transactions],
"backoff_sleeps": sleeps,
} == {"create_many_calls": 1, "queue_after": ["r1", "r2", "r3"], "backoff_sleeps": []}
def _tool_usage_transaction(request_id: str) -> object:
from litellm.proxy.db.spend_log_tool_index import ToolUsageTransaction
return ToolUsageTransaction(
request_id=request_id,
date="2026-10-02",
start_time=datetime(2026, 10, 2, tzinfo=timezone.utc),
tool_names=("get_weather",),
spend=0.01,
total_tokens=12,
)
@pytest.mark.asyncio
async def test_tool_usage_flush_requeues_when_postgres_is_out_of_connections(
mock_prisma_client: MagicMock,
) -> None:
"""A tool-usage batch Postgres had no connection for was never sent, so it is
safe to keep; dropping it loses the rollup increments for good. The job stops
there, like the spend-log write, so a drain loop does not re-hit the full server."""
from prisma.errors import DataError
mock_prisma_client.db.litellm_spendlogtoolindex.create_many = AsyncMock(side_effect=_postgres_out_of_connections())
mock_prisma_client.spend_log_transactions = []
first, second = _tool_usage_transaction("r1"), _tool_usage_transaction("r2")
mock_prisma_client.tool_usage_transactions = [first, second]
with pytest.raises(DataError, match="too many clients already"):
await update_spend_logs_job(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=MagicMock(),
)
assert {
"index_writes": mock_prisma_client.db.litellm_spendlogtoolindex.create_many.await_count,
"queue_after": mock_prisma_client.tool_usage_transactions,
} == {"index_writes": 1, "queue_after": [first, second]}
@pytest.mark.asyncio
async def test_tool_usage_flush_still_drops_ambiguous_failures(
mock_prisma_client: MagicMock,
) -> None:
"""Anything other than a connection refusal may have reached the server, and
the rollup increments are not idempotent, so the batch is not replayed."""
mock_prisma_client.db.litellm_spendlogtoolindex.create_many = AsyncMock(
side_effect=RuntimeError("engine returned a malformed payload")
)
mock_prisma_client.spend_log_transactions = []
mock_prisma_client.tool_usage_transactions = [_tool_usage_transaction("r1")]
await update_spend_logs_job(
prisma_client=mock_prisma_client,
db_writer_client=None,
proxy_logging_obj=MagicMock(),
)
assert mock_prisma_client.tool_usage_transactions == []

View file

@ -1,5 +1,6 @@
import datetime
import time
from collections.abc import Mapping
from pathlib import Path
from types import MappingProxyType, SimpleNamespace
from typing import Final, cast
@ -3864,6 +3865,35 @@ def test_completion_cost_mantle_native_messages_prices_unversioned_claude_from_t
) == pytest.approx(expected), model
@pytest.mark.parametrize("model", ["anthropic.claude-opus-5-5", "anthropic.claude-sonnet-5-5"])
def test_completion_cost_region_without_its_own_row_prices_mantle_claude_from_the_mantle_row(
_local_model_cost_map, model: str
):
"""The proxy resolves a Mantle region for every call. A region with no
bedrock_mantle/<region>/<model> row must fall back to the model's own bedrock_mantle/ row, not to the
bare Bedrock row that the bedrock provider family also matches."""
response = litellm.ModelResponse(
id="msg_x",
choices=[{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
model=model,
usage={"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110},
)
mantle: Final[Mapping[str, float]] = litellm.model_cost[f"bedrock_mantle/{model}"]
bedrock: Final[Mapping[str, float]] = litellm.model_cost[model]
expected: Final = 100 * mantle["input_cost_per_token"] + 10 * mantle["output_cost_per_token"]
assert expected != 100 * bedrock["input_cost_per_token"] + 10 * bedrock["output_cost_per_token"]
for deployment in (model, f"bedrock_mantle/{model}", f"bedrock_mantle/us-east-1/{model}"):
assert litellm.completion_cost(
completion_response=response,
model=deployment,
custom_llm_provider="bedrock_mantle",
region_name="us-east-1",
) == pytest.approx(expected), deployment
assert litellm.get_model_info(f"bedrock_mantle/us-east-1/{model}", "bedrock_mantle")["key"] == f"bedrock_mantle/{model}"
@pytest.mark.parametrize("model", ["anthropic.claude-opus-5-5", "anthropic.claude-sonnet-5-5"])
def test_cost_per_token_gov_region_prices_mantle_claude_on_the_gov_row(_local_model_cost_map, model):
"""A bedrock_mantle/ deployment in us-gov-west-1 must price from the

View file

@ -0,0 +1,43 @@
from pathlib import Path
from typing import Final
import pytest
from pydantic import TypeAdapter
from litellm import get_model_info
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
REPO_ROOT: Final = Path(__file__).parents[2]
COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
MODELS: Final = (
"openrouter/openai/gpt-image-2",
"openrouter/openai/gpt-image-2.5-flare",
"openrouter/openai/gpt-image-2.5-sunburst",
)
PRICE_FIELDS: Final = ("input_cost_per_token", "input_cost_per_image_token", "output_cost_per_image_token")
def _cost_map(path: Path) -> dict[str, dict[str, object]]:
return COST_MAP_ADAPTER.validate_json(path.read_bytes())
MAIN_COST_MAP: Final = _cost_map(REPO_ROOT / "model_prices_and_context_window.json")
BACKUP_COST_MAP: Final = _cost_map(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json")
@pytest.mark.parametrize("model", MODELS)
def test_openrouter_gpt_image_row_is_identical_in_main_and_backup(model: str) -> None:
assert model in MAIN_COST_MAP, f"{model} is missing from model_prices_and_context_window.json"
assert BACKUP_COST_MAP.get(model) == MAIN_COST_MAP[model]
@pytest.mark.usefixtures("local_model_cost_map")
@pytest.mark.parametrize("model", MODELS)
def test_openrouter_gpt_image_model_info_comes_from_the_openrouter_row(model: str) -> None:
routed_model, provider, _, _ = get_llm_provider(model=model)
assert (routed_model, provider) == (model.removeprefix("openrouter/"), "openrouter")
info = get_model_info(model=routed_model, custom_llm_provider=provider)
row = MAIN_COST_MAP[model]
assert (info["key"], info["litellm_provider"], info["mode"]) == (model, "openrouter", "image_generation")
assert {field: info[field] for field in PRICE_FIELDS} == {field: row[field] for field in PRICE_FIELDS}

View file

@ -10276,6 +10276,200 @@ def test_get_configured_display_name_skips_wildcard_pattern_matching():
)
@pytest.mark.parametrize(
"configured",
[["ultrafast"], ["priority", {"id": "ultrafast", "name": "Ultrafast", "description": "Fastest"}], [], "not-a-list"],
)
def test_get_configured_service_tiers_returns_the_deployment_model_info_value_as_set(configured):
router = litellm.Router(
model_list=[
{
"model_name": "gpt-6-astra",
"litellm_params": {"model": "openai/gpt-6-astra"},
"model_info": {"service_tiers": configured},
}
]
)
assert router.get_configured_service_tiers("gpt-6-astra") == (configured,)
def test_get_configured_service_tiers_returns_one_value_per_deployment_in_model_list_order():
router = litellm.Router(
model_list=[
{
"model_name": "gpt-6-astra",
"litellm_params": {"model": "openai/gpt-6-astra"},
"model_info": {"service_tiers": ["ultrafast"]},
},
{"model_name": "gpt-6-astra", "litellm_params": {"model": "openai/gpt-6-astra", "api_base": "https://a.example"}},
{
"model_name": "gpt-6-astra",
"litellm_params": {"model": "openai/gpt-6-astra", "api_base": "https://b.example"},
"model_info": {"service_tiers": ["priority", "ultrafast"]},
},
]
)
assert router.get_configured_service_tiers("gpt-6-astra") == (["ultrafast"], None, ["priority", "ultrafast"])
def test_get_configured_service_tiers_returns_none_for_an_unset_deployment_and_nothing_for_an_unknown_name():
router = litellm.Router(
model_list=[
{
"model_name": "no-tiers-model",
"litellm_params": {"model": "openai/some-unmapped-model"},
}
]
)
assert router.get_configured_service_tiers("no-tiers-model") == (None,)
assert router.get_configured_service_tiers("not-a-real-model") == ()
def test_get_configured_service_tiers_does_not_apply_a_wildcard_deployment_to_matched_names():
router = litellm.Router(
model_list=[
{
"model_name": "openai/*",
"litellm_params": {"model": "openai/*"},
"model_info": {"service_tiers": ["ultrafast"]},
}
]
)
assert router.get_configured_service_tiers("openai/gpt-6-astra") == ()
def test_get_configured_service_tiers_reads_only_the_deployments_a_request_can_route_to():
router = litellm.Router(
model_list=[
{
"model_name": "gpt-6-astra",
"litellm_params": {"model": "openai/gpt-6-astra"},
"model_info": {"service_tiers": ["ultrafast"]},
},
{
"model_name": "gpt-6-astra",
"litellm_params": {"model": "openai/gpt-6-astra", "api_base": "https://paused.example"},
"model_info": {"blocked": True},
},
{
"model_name": "gpt-6-astra",
"litellm_params": {"model": "openai/gpt-6-astra", "api_base": "https://team-2.example"},
"model_info": {"team_id": "team-2"},
},
]
)
assert router.get_configured_service_tiers("gpt-6-astra", team_id="team-1") == (["ultrafast"],)
assert router.get_configured_service_tiers("gpt-6-astra", team_id="team-2") == (["ultrafast"], None)
assert router.get_configured_service_tiers("gpt-6-astra") == (["ultrafast"],)
@pytest.mark.parametrize(
"alias_value, expected_group",
[
("gpt-6-astra", "gpt-6-astra"),
({"model": "gpt-6-astra", "hidden": True}, "gpt-6-astra"),
({"model": "", "hidden": False}, "gpt-6"),
],
ids=["string-alias", "item-alias", "malformed-alias-is-itself"],
)
def test_routable_model_group_is_the_alias_target_else_the_name_itself(alias_value, expected_group):
router = litellm.Router(
model_list=[{"model_name": "gpt-6-astra", "litellm_params": {"model": "openai/gpt-6-astra"}}],
model_group_alias={"gpt-6": alias_value},
)
assert router.routable_model_group("gpt-6") == expected_group
assert router.routable_model_group("gpt-6-astra") == "gpt-6-astra"
assert router.routable_model_group("not-a-real-model") == "not-a-real-model"
def test_get_configured_service_tiers_reads_an_alias_off_its_target_deployments():
router = litellm.Router(
model_list=[
{
"model_name": "gpt-6-astra",
"litellm_params": {"model": "openai/gpt-6-astra"},
"model_info": {"service_tiers": ["ultrafast"]},
},
{
"model_name": "gpt-6-astra",
"litellm_params": {"model": "openai/gpt-6-astra", "api_base": "https://team-2.example"},
"model_info": {"team_id": "team-2", "service_tiers": ["priority"]},
},
],
model_group_alias={"gpt-6": "gpt-6-astra", "gpt-6-quiet": {"model": "gpt-6-astra", "hidden": True}},
)
assert router.get_configured_service_tiers("gpt-6") == router.get_configured_service_tiers("gpt-6-astra")
assert router.get_configured_service_tiers("gpt-6", team_id="team-1") == (["ultrafast"],)
assert router.get_configured_service_tiers("gpt-6", team_id="team-2") == (["ultrafast"], ["priority"])
assert router.get_configured_service_tiers("gpt-6-quiet", team_id="team-1") == (["ultrafast"],)
def _router_with_team_owned_deployments():
return litellm.Router(
model_list=[
{
"model_name": "owned-by-teams",
"litellm_params": {"model": "openai/gpt-5.5"},
"model_info": {"team_id": "team-1", "service_tiers": ["priority"]},
},
{
"model_name": "owned-by-teams",
"litellm_params": {"model": "openai/paused-model"},
"model_info": {"team_id": "team-2", "blocked": True, "service_tiers": ["paused"]},
},
{
"model_name": "owned-by-teams",
"litellm_params": {"model": "openai/team-2-model"},
"model_info": {"team_id": "team-2", "service_tiers": ["flex"]},
},
{
"model_name": "owned-and-shared",
"litellm_params": {"model": "openai/team-1-model"},
"model_info": {"team_id": "team-1", "service_tiers": ["priority"]},
},
{
"model_name": "owned-and-shared",
"litellm_params": {"model": "openai/shared-model"},
"model_info": {"service_tiers": ["flex"]},
},
{"model_name": "openai/*", "litellm_params": {"model": "openai/*"}},
],
model_group_alias={"nickname": "owned-by-teams"},
)
@pytest.mark.parametrize(
"model_name, team_id, upstream_model, service_tiers",
[
("owned-by-teams", "team-1", "openai/gpt-5.5", (["priority"],)),
("owned-by-teams", "team-2", "openai/team-2-model", (["flex"],)),
("nickname", "team-1", "openai/gpt-5.5", (["priority"],)),
("nickname", "team-2", "openai/team-2-model", (["flex"],)),
("owned-by-teams", "team-3", None, ()),
("owned-by-teams", None, "openai/gpt-5.5", (["priority"], ["flex"])),
("owned-and-shared", "team-1", "openai/team-1-model", (["priority"], ["flex"])),
("owned-and-shared", "team-2", "openai/shared-model", (["flex"],)),
("owned-and-shared", None, "openai/shared-model", (["flex"],)),
("openai/gpt-5.5", "team-1", None, ()),
("not-a-real-model", None, None, ()),
],
)
def test_upstream_model_and_service_tiers_are_read_off_the_deployments_the_team_can_route_to(
model_name, team_id, upstream_model, service_tiers
):
router = _router_with_team_owned_deployments()
assert router.get_routable_upstream_model(model_name, team_id) == upstream_model
assert router.get_configured_service_tiers(model_name, team_id) == service_tiers
def test_get_configured_display_name_treats_malformed_values_as_absent():
malformed = ["", " ", 12345, ["Kimi K3"], {"name": "Kimi K3"}, True]
router = litellm.Router(

View file

@ -32,9 +32,9 @@ describe("Lens navigation", () => {
expect(screen.getByText("Trace polling active")).toBeVisible();
});
it("opens existing lens links directly in investigations", () => {
it("opens existing lens links on findings", () => {
renderWithProviders(<LensPage />, { searchParams: "?lens=saved-lens" });
expect(screen.getByRole("tab", { name: "Investigations" })).toHaveAttribute("aria-selected", "true");
expect(screen.getByRole("tab", { name: "Findings" })).toHaveAttribute("aria-selected", "true");
expect(screen.getByText("Manage investigations")).toBeVisible();
});

View file

@ -41,7 +41,7 @@ describe("Lens interactive demo", () => {
await chooseSelectOption(user, screen.getByRole("combobox", { name: "Filter by status" }), "Failed");
expect(within(screen.getByRole("table", { name: "Agent runs" })).getAllByRole("row")).toHaveLength(4);
await user.click(screen.getByRole("tab", { name: "Investigations" }));
expect(await screen.findByRole("button", { name: /Support quality/ })).toBeVisible();
expect(await screen.findByRole("row", { name: /Support quality/ })).toBeVisible();
expect(screen.queryByRole("button", { name: "New investigation" })).not.toBeInTheDocument();
expect(network).not.toHaveBeenCalled();
expect(onUrlUpdate).not.toHaveBeenCalled();
@ -60,9 +60,9 @@ describe("Lens interactive demo", () => {
});
await screen.findByRole("button", { name: "Preview sample" });
await user.click(screen.getByRole("button", { name: "Preview sample" }));
await user.click(await screen.findByRole("button", { name: /Support quality/ }));
await user.click(await screen.findByRole("tab", { name: "Findings" }));
network.mockClear();
await user.click(screen.getByRole("button", { name: /Repeated lookups leave customers without an answer/ }));
await user.click(await screen.findByRole("row", { name: /Repeated lookups leave customers without an answer/ }));
const finding = screen.getByRole("dialog");
expect(within(finding).getByText(/The support agent retries/)).toBeVisible();
const summaries = within(finding).getAllByText("support_agent", { exact: true });
@ -80,9 +80,7 @@ describe("Lens interactive demo", () => {
within(screen.getByRole("dialog", { name: "Original run" })).getByRole("button", { name: "Close" }),
);
await user.click(within(screen.getByRole("dialog")).getByRole("button", { name: "Close" }));
await user.click(screen.getByRole("tab", { name: "History" }));
await user.click(screen.getAllByRole("button", { name: /runs reviewed/ })[1]);
expect(await screen.findByText(/1 linked run · high priority/)).toBeVisible();
expect(await screen.findByRole("table", { name: "Findings" })).toBeVisible();
expect(network).not.toHaveBeenCalled();
expect(onUrlUpdate).not.toHaveBeenCalled();
await user.click(screen.getByRole("button", { name: "Exit demo" }));
@ -127,7 +125,7 @@ describe("Lens interactive demo", () => {
expect(await screen.findByRole("button", { name: "Preview sample" })).toBeVisible();
const tabs = within(screen.getByRole("tablist", { name: "Lens" }));
await user.click(tabs.getByRole("tab", { name: "Investigations" }));
expect(await screen.findByRole("button", { name: new RegExp(saved.settings.name) })).toBeVisible();
expect(await screen.findByRole("row", { name: new RegExp(saved.settings.name) })).toBeVisible();
expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument();
await user.click(tabs.getByRole("tab", { name: "Traces" }));
await user.click(await screen.findByRole("button", { name: "Preview sample" }));

View file

@ -14,7 +14,7 @@ import { isProxyAdminRole, isProxyAdminTierRole } from "@/utils/roles";
import { InvestigationsView } from "./investigations/InvestigationsView";
import { createLensDemo } from "./demo/createLensDemo";
type Tab = "traces" | "investigations";
type Tab = "traces" | "findings" | "investigations";
type WorkspaceProps = { accessToken: string; userRole: string; readOnly: boolean };
export function LensWorkspace(props: WorkspaceProps) {
@ -59,12 +59,12 @@ function LensContent({
const demo = useLensDemo();
const [tab, setTab] = useQueryState(
"tab",
parseAsStringLiteral(["traces", "investigations"]).withOptions({ history: "push" }),
parseAsStringLiteral(["traces", "findings", "investigations"]).withOptions({ history: "push" }),
);
const [lensId] = useQueryState("lens", parseAsString);
const [demoTab, setDemoTab] = useState(initialTab);
const [previewTarget, setPreviewTarget] = useState<HTMLDivElement | null>(null);
const defaultTab = lensId ? "investigations" : "traces";
const defaultTab = lensId ? "findings" : "traces";
const activeTab = demo ? demoTab : tab ?? defaultTab;
const openDemo = onDemo ? () => onDemo(activeTab) : undefined;
return (
@ -86,6 +86,9 @@ function LensContent({
<TabsTrigger value="traces" className="px-3">
Traces
</TabsTrigger>
<TabsTrigger value="findings" className="px-3">
Findings
</TabsTrigger>
<TabsTrigger value="investigations" className="px-3">
Investigations
</TabsTrigger>
@ -102,19 +105,23 @@ function LensContent({
onDemo={activeTab === "traces" ? openDemo : undefined}
/>
</TabsContent>
<TabsContent value="investigations" keepMounted={!!demo}>
{demo || isProxyAdminTierRole(userRole) ? (
<InvestigationsView
accessToken={accessToken}
readOnly={readOnly || !isProxyAdminRole(userRole)}
onDemo={activeTab === "investigations" ? openDemo : undefined}
/>
) : (
<p className="py-6 text-sm text-muted-foreground">
Investigations require proxy administrator access. You can still view your traces.
</p>
)}
</TabsContent>
{(["findings", "investigations"] as const).map((view) => (
<TabsContent key={view} value={view} keepMounted={!!demo} className="flex min-h-0 flex-col">
{demo || isProxyAdminTierRole(userRole) ? (
<InvestigationsView
view={view}
active={activeTab === view}
accessToken={accessToken}
readOnly={readOnly || !isProxyAdminRole(userRole)}
onDemo={activeTab === view ? openDemo : undefined}
/>
) : (
<p className="py-6 text-sm text-muted-foreground">
Investigations require proxy administrator access. You can still view your traces.
</p>
)}
</TabsContent>
))}
</Tabs>
</main>
</LensPreviewTarget.Provider>

View file

@ -1,4 +1,5 @@
import { z } from "zod";
import type { RunWindow } from "../investigations/detail/RunNowDialog";
import type { ApiClient } from "@/lib/http/client";
import type { components } from "@/lib/http/schema";
import type { ActivitySelection, Job, Lens, LensList, Sample, Settings, WorkerCreated } from "../model/types";
@ -44,7 +45,8 @@ export interface LensApi {
keys(alias: string, page: number, signal: AbortSignal): Promise<KeyPage>;
keyInfo(keyId: string): Promise<KeyInfo>;
saveLens(id: string | undefined, settings: Settings): Promise<Lens>;
startRun(lensId: string): Promise<void>;
startRun(lensId: string, request?: RunWindow): Promise<void>;
watchAll(): Promise<components["schemas"]["WatchAllResult"]>;
cancelRun(lensId: string): Promise<void>;
reviewFinding(lensId: string, findingId: string, status: FindingStatus, reason: string): Promise<void>;
registerWorker(analysisKeyId: string | null): Promise<WorkerCreated>;
@ -109,7 +111,9 @@ export function liveLensApi(apiClient: ApiClient, accessToken: string): LensApi
keyInfoSchema.parse(await apiClient.get("/key/info", { accessToken, query: { key: keyId } })).info,
saveLens: (id, settings) =>
apiClient.request<Lens>(id ? "PUT" : "POST", id ? `/lens/${id}` : "/lens", { accessToken, body: settings }),
startRun: (lensId) => apiClient.post(`/lens/${lensId}/runs`, { accessToken, body: {} }),
startRun: (lensId, request = {}) => apiClient.post(`/lens/${lensId}/runs`, { accessToken, body: request }),
watchAll: () =>
apiClient.post<components["schemas"]["WatchAllResult"]>("/lens/watch-all", { accessToken, body: {} }),
cancelRun: (lensId) => apiClient.post(`/lens/${lensId}/cancel`, { accessToken, body: {} }),
reviewFinding: (lensId, findingId, status, reason) =>
apiClient.patch(`/lens/${lensId}/findings/${findingId}`, { accessToken, body: { status, reason } }),

View file

@ -296,6 +296,8 @@ export function createLensDemoData(now = Date.now()) {
: [],
})),
attempts: 1,
steps: [],
trigger: "schedule" as const,
error: "",
cost: sample.length * 0.012,
coverage: {
@ -359,6 +361,7 @@ function demoLensApi(data: LensDemoData): LensApi {
keyInfo: notInDemo,
saveLens: readOnly,
startRun: readOnly,
watchAll: async () => ({ watching: [], skipped: [] }),
cancelRun: readOnly,
reviewFinding: readOnly,
registerWorker: readOnly,

View file

@ -10,6 +10,7 @@ import { type Finding, type Sample } from "../model/types";
export function FindingSheet({
finding,
agents = [],
sampledRuns,
readOnly,
busy,
@ -18,6 +19,7 @@ export function FindingSheet({
changeFinding,
}: {
finding?: Finding;
agents?: readonly string[];
sampledRuns: Sample["executions"];
readOnly: boolean;
busy: boolean;
@ -46,6 +48,7 @@ export function FindingSheet({
<SheetHeader>
<SheetTitle className="pr-8 text-xl leading-snug">{finding.title}</SheetTitle>
<SheetDescription>
{agents.length > 0 && <span className="font-medium text-foreground">{agents.join(", ")} · </span>}
{finding.kind === "issue" ? `${finding.priority} priority` : "Pattern"} ·{" "}
{finding.occurrences?.length ?? 0} linked {finding.occurrences?.length === 1 ? "run" : "runs"}
</SheetDescription>

View file

@ -0,0 +1,147 @@
"use client";
import { ChevronRight } from "lucide-react";
import { useState } from "react";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { useNow } from "@/hooks/useNow";
import { formatActivityTimestamp } from "@/utils/activityTimestamp";
import { agoLabel } from "@/components/view_logs/TraceView/lensField";
import { cn } from "@/lib/cva.config";
import { ALL_AGENTS, filterInbox, inboxAgents, inboxRows, type InboxRow, type Priority } from "../model/inbox";
import type { Lens } from "../model/types";
const PRIORITY_DOT = { high: "bg-[#e5484d]", medium: "bg-amber-500", low: "bg-muted-foreground/50" } as const;
const PRIORITY_ITEMS: { value: Priority | "all"; label: string }[] = [
{ value: "all", label: "All priorities" },
{ value: "high", label: "High" },
{ value: "medium", label: "Medium" },
{ value: "low", label: "Low" },
];
const TH = "px-3 font-medium";
const TH_NUM = "px-3 text-right font-medium";
function FilterSelect<T extends string>({
label,
value,
items,
onChange,
width,
}: {
label: string;
value: T;
items: { value: T; label: string }[];
onChange: (value: T) => void;
width: string;
}) {
return (
<Select items={items} value={value} onValueChange={(next: T | null) => next !== null && onChange(next)}>
<SelectTrigger size="sm" className={cn("h-7 text-[12px]", width)} aria-label={label}>
<SelectValue />
</SelectTrigger>
<SelectContent>
{items.map((item) => (
<SelectItem key={item.value} value={item.value}>
{item.label}
</SelectItem>
))}
</SelectContent>
</Select>
);
}
export function FindingsInbox({ lenses, onOpen }: { lenses: readonly Lens[]; onOpen: (row: InboxRow) => void }) {
const [agent, setAgent] = useState(ALL_AGENTS);
const [priority, setPriority] = useState<Priority | "all">("all");
const now = useNow(30000);
const all = inboxRows(lenses);
const rows = filterInbox(all, { agent, priority });
const agentItems = [
{ value: ALL_AGENTS, label: "All agents" },
...inboxAgents(all).map((a) => ({ value: a, label: a })),
];
return (
<div
className="flex min-h-[420px] flex-1 flex-col overflow-hidden border-y border-border bg-card"
data-testid="findings-inbox"
>
<div className="flex min-h-10 shrink-0 flex-wrap items-center gap-2 border-b border-border bg-card px-3 py-1.5">
<FilterSelect
label="Filter by agent"
value={agent}
items={agentItems}
onChange={setAgent}
width="min-w-[150px]"
/>
<FilterSelect
label="Filter by priority"
value={priority}
items={PRIORITY_ITEMS}
onChange={setPriority}
width="min-w-[130px]"
/>
</div>
<div className="min-h-0 flex-1 overflow-auto">
<table aria-label="Findings" className="w-full min-w-[900px] table-fixed border-collapse text-left">
<thead className="sticky top-0 z-sticky bg-muted/40 backdrop-blur">
<tr className="h-8 border-b border-border text-[10px] tracking-[0.08em] text-muted-foreground uppercase">
<th className={`w-[96px] ${TH}`}>Priority</th>
<th className={TH}>Finding</th>
<th className={`w-[180px] ${TH}`}>Agent</th>
<th className={`w-[64px] ${TH_NUM}`}>Runs</th>
<th className={`w-[96px] ${TH}`}>Last seen</th>
<th className="w-8" />
</tr>
</thead>
<tbody>
{rows.map((row) => (
<tr
key={row.key}
onClick={() => onOpen(row)}
data-testid="inbox-row"
aria-label={row.title}
className="h-9 cursor-pointer border-b border-border/60 text-[12px] transition-colors duration-150 hover:bg-trace-row-hover motion-reduce:transition-none"
>
<td className="px-3">
<span className="inline-flex items-center gap-1.5 text-muted-foreground">
<span className={cn("size-1.5 rounded-full", PRIORITY_DOT[row.priority])} />
{row.priority}
</span>
</td>
<td className="px-3" title={row.suggestion ? `Fix: ${row.suggestion}` : undefined}>
<span className="block truncate text-foreground">{row.title}</span>
</td>
<td className="truncate px-3 text-muted-foreground" title={row.agents.join(", ")}>
{row.agents.join(", ")}
</td>
<td className="px-3 text-right font-mono tabular-nums text-foreground">{row.runs}</td>
<td
className="px-3 font-mono text-[11px] tabular-nums text-muted-foreground"
title={formatActivityTimestamp(row.lastSeen)}
>
{agoLabel(Date.parse(row.lastSeen), now)}
</td>
<td>
<ChevronRight className="size-3 text-muted-foreground/60" />
</td>
</tr>
))}
</tbody>
</table>
{rows.length === 0 && (
<div className="py-16 text-center text-[12px] text-muted-foreground">
{all.length === 0
? "No open findings yet. New problems show up here as soon as an investigation spots them."
: "No findings match these filters."}
</div>
)}
</div>
<footer className="flex h-8 shrink-0 items-center border-t border-border bg-muted/40 px-3 font-mono text-[11px] text-muted-foreground">
{rows.length} {rows.length === 1 ? "finding" : "findings"}
{rows.length !== all.length && ` of ${all.length}`}
</footer>
</div>
);
}

View file

@ -0,0 +1,13 @@
"use client";
import { useContext, type ReactNode } from "react";
import { createPortal } from "react-dom";
import { LensPreviewTarget } from "../LensPreviewButton";
export function HeaderActions({ children }: { children: ReactNode }) {
const target = useContext(LensPreviewTarget);
const actions = <div className="flex flex-wrap items-center gap-3">{children}</div>;
if (target === null) return null;
return target ? createPortal(actions, target) : actions;
}

Some files were not shown because too many files have changed in this diff Show more