mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge remote-tracking branch 'origin/main' into pr-44290-fixes
This commit is contained in:
commit
91d970a689
119 changed files with 12084 additions and 751 deletions
14
litellm-rust/Cargo.lock
generated
14
litellm-rust/Cargo.lock
generated
|
|
@ -4469,6 +4469,19 @@ dependencies = [
|
|||
"time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-traces-cache"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"litellm-traces",
|
||||
"moka",
|
||||
"rstest",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-traces-clickhouse"
|
||||
version = "0.1.0"
|
||||
|
|
@ -4484,6 +4497,7 @@ dependencies = [
|
|||
"litellm-migrate",
|
||||
"litellm-storage-clickhouse",
|
||||
"litellm-traces",
|
||||
"litellm-traces-cache",
|
||||
"macro_rules_attribute",
|
||||
"moka",
|
||||
"rstest",
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ litellm-config = { path = "crates/config" }
|
|||
litellm-router = { path = "crates/router" }
|
||||
litellm-tracing = { path = "crates/tracing" }
|
||||
litellm-traces = { path = "crates/traces" }
|
||||
litellm-traces-cache = { path = "crates/traces-cache" }
|
||||
litellm-traces-clickhouse = { path = "crates/traces-clickhouse" }
|
||||
litellm-storage-clickhouse = { path = "crates/storage-clickhouse" }
|
||||
litellm-migrate = { path = "crates/migrate" }
|
||||
|
|
|
|||
5
litellm-rust/crates/traces-cache/AGENTS.md
Normal file
5
litellm-rust/crates/traces-cache/AGENTS.md
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
Own resolved-trace snapshot storage, cache identity, weighting, and expiry
|
||||
Depend on trace domain types, never storage, HTTP, or Python
|
||||
Preserve the full source and authorization scope in every cache key
|
||||
Keep snapshots immutable and expose borrowed data
|
||||
Keep cursor formats and database reads in their existing owners
|
||||
17
litellm-rust/crates/traces-cache/Cargo.toml
Normal file
17
litellm-rust/crates/traces-cache/Cargo.toml
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
[package]
|
||||
name = "litellm-traces-cache"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
litellm-traces.workspace = true
|
||||
moka.workspace = true
|
||||
serde_json.workspace = true
|
||||
sha2.workspace = true
|
||||
thiserror.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
rstest.workspace = true
|
||||
tokio.workspace = true
|
||||
7
litellm-rust/crates/traces-cache/src/error.rs
Normal file
7
litellm-rust/crates/traces-cache/src/error.rs
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
#[error("trace snapshot serialization failed")]
|
||||
Serialization(#[from] serde_json::Error),
|
||||
#[error("trace snapshot exceeds the size limit")]
|
||||
ReadTooLarge,
|
||||
}
|
||||
166
litellm-rust/crates/traces-cache/src/lib.rs
Normal file
166
litellm-rust/crates/traces-cache/src/lib.rs
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use litellm_traces::{Trace, query::named::ReadAccessParams};
|
||||
use moka::future::Cache;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
mod error;
|
||||
|
||||
pub use error::Error;
|
||||
|
||||
#[derive(Clone, Eq, Hash, PartialEq)]
|
||||
pub struct SnapshotKey(String);
|
||||
|
||||
impl SnapshotKey {
|
||||
pub fn new(
|
||||
source: &str,
|
||||
access: &ReadAccessParams,
|
||||
trace_id: &str,
|
||||
trace_ref: &str,
|
||||
snapshot_ms: u64,
|
||||
) -> Result<Self, Error> {
|
||||
let encoded = serde_json::to_vec(&(source, access, trace_id, trace_ref, snapshot_ms))?;
|
||||
Ok(Self(format!("{:x}", Sha256::digest(encoded))))
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Snapshot {
|
||||
trace: Trace,
|
||||
version: String,
|
||||
weight: u32,
|
||||
}
|
||||
|
||||
impl Snapshot {
|
||||
pub fn trace(&self) -> &Trace {
|
||||
&self.trace
|
||||
}
|
||||
|
||||
pub fn version(&self) -> &str {
|
||||
&self.version
|
||||
}
|
||||
}
|
||||
|
||||
pub struct SnapshotCache {
|
||||
entries: Cache<SnapshotKey, Arc<Snapshot>>,
|
||||
max_graph_bytes: usize,
|
||||
}
|
||||
|
||||
impl SnapshotCache {
|
||||
pub fn new(max_graph_bytes: usize, ttl: Duration) -> Self {
|
||||
Self {
|
||||
entries: Cache::builder()
|
||||
.max_capacity((max_graph_bytes as u64).saturating_mul(2))
|
||||
.weigher(|_: &SnapshotKey, snapshot: &Arc<Snapshot>| snapshot.weight)
|
||||
.time_to_live(ttl)
|
||||
.build(),
|
||||
max_graph_bytes,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get(&self, key: &SnapshotKey) -> Option<Arc<Snapshot>> {
|
||||
self.entries.get(key).await
|
||||
}
|
||||
|
||||
pub async fn insert(&self, key: SnapshotKey, trace: Trace) -> Result<Arc<Snapshot>, Error> {
|
||||
let encoded = serde_json::to_vec(&trace)?;
|
||||
if encoded.len() > self.max_graph_bytes {
|
||||
return Err(Error::ReadTooLarge);
|
||||
}
|
||||
|
||||
let span_ids: Vec<&str> = trace
|
||||
.spans
|
||||
.iter()
|
||||
.map(|span| span.span_id.as_str())
|
||||
.collect();
|
||||
|
||||
let version = format!("{:x}", Sha256::digest(serde_json::to_vec(&span_ids)?));
|
||||
|
||||
let snapshot = Arc::new(Snapshot {
|
||||
trace,
|
||||
version,
|
||||
weight: u32::try_from(encoded.len().saturating_mul(2)).unwrap_or(u32::MAX),
|
||||
});
|
||||
|
||||
self.entries.insert(key, Arc::clone(&snapshot)).await;
|
||||
Ok(snapshot)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use litellm_traces::{
|
||||
SpanStatus,
|
||||
query::named::{SpendByResponseIdsRow, TraceSpansRow},
|
||||
resolve_trace,
|
||||
};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn trace(span_id: &str) -> Trace {
|
||||
let rows = [TraceSpansRow {
|
||||
trace_id: String::new(),
|
||||
span_id: span_id.into(),
|
||||
parent_span_id: String::new(),
|
||||
name: "run".into(),
|
||||
kind: litellm_traces::ObservationType::Agent,
|
||||
wrapper_candidate: false,
|
||||
agent: "agent".into(),
|
||||
framework: String::new(),
|
||||
status: SpanStatus::Ok,
|
||||
status_message: String::new(),
|
||||
error_truncated: false,
|
||||
start_ns: 1_790_742_989_000_000_000,
|
||||
duration_ns: 10_000_000,
|
||||
service: "agent-demo".into(),
|
||||
input_preview: format!("input of {span_id}"),
|
||||
model: String::new(),
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
litellm_request_id: String::new(),
|
||||
call_keys: Vec::new(),
|
||||
call_evidence: None,
|
||||
tool_call_id: String::new(),
|
||||
team_id: String::new(),
|
||||
api_key_hash: String::new(),
|
||||
user_id: String::new(),
|
||||
}];
|
||||
resolve_trace("trace", "ref", &rows, &[] as &[SpendByResponseIdsRow])
|
||||
.expect("fixture should resolve")
|
||||
}
|
||||
|
||||
fn key(suffix: &str) -> SnapshotKey {
|
||||
SnapshotKey::new(
|
||||
"source",
|
||||
&ReadAccessParams {
|
||||
all_teams: false,
|
||||
user_id: String::new(),
|
||||
team_ids: vec!["team".into()],
|
||||
},
|
||||
suffix,
|
||||
"ref",
|
||||
100,
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn weighted_capacity_bounds_retained_snapshots() {
|
||||
let limit = ["first", "second", "third"]
|
||||
.iter()
|
||||
.map(|span_id| serde_json::to_vec(&trace(span_id)).unwrap().len())
|
||||
.max()
|
||||
.unwrap();
|
||||
let cache = SnapshotCache::new(limit, Duration::from_secs(120));
|
||||
|
||||
for (key, span_id) in [
|
||||
(key("a"), "first"),
|
||||
(key("b"), "second"),
|
||||
(key("c"), "third"),
|
||||
] {
|
||||
cache.insert(key, trace(span_id)).await.unwrap();
|
||||
}
|
||||
|
||||
cache.entries.run_pending_tasks().await;
|
||||
assert!(cache.entries.weighted_size() <= (limit as u64) * 2);
|
||||
}
|
||||
}
|
||||
191
litellm-rust/crates/traces-cache/tests/snapshots.rs
Normal file
191
litellm-rust/crates/traces-cache/tests/snapshots.rs
Normal file
|
|
@ -0,0 +1,191 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use litellm_traces::{
|
||||
SpanStatus, Trace,
|
||||
query::named::{ReadAccessParams, SpendByResponseIdsRow, TraceSpansRow},
|
||||
resolve_trace,
|
||||
};
|
||||
use litellm_traces_cache::{Error, SnapshotCache, SnapshotKey};
|
||||
use rstest::{fixture, rstest};
|
||||
|
||||
const T0: i64 = 1_790_742_989_000_000_000;
|
||||
const MS: i64 = 1_000_000;
|
||||
const TTL: Duration = Duration::from_secs(120);
|
||||
|
||||
fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> TraceSpansRow {
|
||||
TraceSpansRow {
|
||||
trace_id: String::new(),
|
||||
span_id: span_id.into(),
|
||||
parent_span_id: parent.into(),
|
||||
name: name.into(),
|
||||
kind: kind.parse().unwrap(),
|
||||
wrapper_candidate: false,
|
||||
agent: agent.into(),
|
||||
framework: String::new(),
|
||||
status: SpanStatus::Ok,
|
||||
status_message: String::new(),
|
||||
error_truncated: false,
|
||||
start_ns: T0,
|
||||
duration_ns: 10 * MS as u64,
|
||||
service: "agent-demo".into(),
|
||||
input_preview: format!("input of {name}"),
|
||||
model: String::new(),
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
litellm_request_id: String::new(),
|
||||
call_keys: Vec::new(),
|
||||
call_evidence: None,
|
||||
tool_call_id: String::new(),
|
||||
team_id: String::new(),
|
||||
api_key_hash: String::new(),
|
||||
user_id: String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn access() -> ReadAccessParams {
|
||||
ReadAccessParams {
|
||||
all_teams: false,
|
||||
user_id: String::new(),
|
||||
team_ids: vec!["team".into()],
|
||||
}
|
||||
}
|
||||
|
||||
fn key(
|
||||
source: &str,
|
||||
access: &ReadAccessParams,
|
||||
trace_id: &str,
|
||||
trace_ref: &str,
|
||||
ms: u64,
|
||||
) -> SnapshotKey {
|
||||
SnapshotKey::new(source, access, trace_id, trace_ref, ms).unwrap()
|
||||
}
|
||||
|
||||
#[fixture]
|
||||
fn trace() -> Trace {
|
||||
resolve_trace(
|
||||
"trace",
|
||||
"ref",
|
||||
&[row("root", "", "run", "agent", "agent")],
|
||||
&[] as &[SpendByResponseIdsRow],
|
||||
)
|
||||
.expect("fixture should resolve")
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::different_team(false, "", "other-team")]
|
||||
#[case::different_user(false, "other-user", "team")]
|
||||
#[case::different_scope(true, "", "team")]
|
||||
#[tokio::test]
|
||||
async fn cached_trace_is_isolated_by_access_scope(
|
||||
trace: Trace,
|
||||
#[case] all_teams: bool,
|
||||
#[case] user_id: &str,
|
||||
#[case] team_id: &str,
|
||||
) {
|
||||
let cache = SnapshotCache::new(1024 * 1024, TTL);
|
||||
let stored = key("source", &access(), "trace", "ref", 100);
|
||||
|
||||
cache.insert(stored.clone(), trace.clone()).await.unwrap();
|
||||
|
||||
let other_access = ReadAccessParams {
|
||||
all_teams,
|
||||
user_id: user_id.into(),
|
||||
team_ids: vec![team_id.into()],
|
||||
};
|
||||
let other = key("source", &other_access, "trace", "ref", 100);
|
||||
|
||||
assert!(cache.get(&other).await.is_none());
|
||||
let cached = cache.get(&stored).await.unwrap();
|
||||
assert_eq!(cached.trace(), &trace);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::different_source("other-source", "trace", "ref", 100)]
|
||||
#[case::different_trace_id("source", "other-trace", "ref", 100)]
|
||||
#[case::different_trace_ref("source", "trace", "other-ref", 100)]
|
||||
#[case::different_snapshot_ms("source", "trace", "ref", 200)]
|
||||
#[tokio::test]
|
||||
async fn cached_trace_is_isolated_by_key_fields(
|
||||
trace: Trace,
|
||||
#[case] source: &str,
|
||||
#[case] trace_id: &str,
|
||||
#[case] trace_ref: &str,
|
||||
#[case] snapshot_ms: u64,
|
||||
) {
|
||||
let cache = SnapshotCache::new(1024 * 1024, TTL);
|
||||
let stored = key("source", &access(), "trace", "ref", 100);
|
||||
|
||||
cache.insert(stored.clone(), trace.clone()).await.unwrap();
|
||||
|
||||
let other = key(source, &access(), trace_id, trace_ref, snapshot_ms);
|
||||
assert!(cache.get(&other).await.is_none());
|
||||
assert!(cache.get(&stored).await.is_some());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn snapshot_at_the_size_limit_is_accepted(trace: Trace) {
|
||||
let size = serde_json::to_vec(&trace).unwrap().len();
|
||||
let cache = SnapshotCache::new(size, TTL);
|
||||
let stored = key("source", &access(), "trace", "ref", 100);
|
||||
|
||||
cache.insert(stored.clone(), trace).await.unwrap();
|
||||
assert!(cache.get(&stored).await.is_some());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn snapshot_one_byte_over_the_size_limit_is_rejected(trace: Trace) {
|
||||
let size = serde_json::to_vec(&trace).unwrap().len();
|
||||
let cache = SnapshotCache::new(size - 1, TTL);
|
||||
let stored = key("source", &access(), "trace", "ref", 100);
|
||||
|
||||
assert!(matches!(
|
||||
cache.insert(stored.clone(), trace).await,
|
||||
Err(Error::ReadTooLarge)
|
||||
));
|
||||
assert!(cache.get(&stored).await.is_none());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::same_ids(&["root", "child"], &["root", "child"], true)]
|
||||
#[case::different_ids(&["root", "child"], &["root", "other"], false)]
|
||||
#[tokio::test]
|
||||
async fn snapshot_version_tracks_the_ordered_span_ids(
|
||||
#[case] first_ids: &[&str],
|
||||
#[case] second_ids: &[&str],
|
||||
#[case] equal: bool,
|
||||
) {
|
||||
let build = |ids: &[&str]| -> Trace {
|
||||
let rows: Vec<TraceSpansRow> = ids
|
||||
.iter()
|
||||
.map(|span_id| row(span_id, "", "run", "agent", "agent"))
|
||||
.collect();
|
||||
resolve_trace("trace", "ref", &rows, &[] as &[SpendByResponseIdsRow])
|
||||
.expect("fixture should resolve")
|
||||
};
|
||||
let cache = SnapshotCache::new(1024 * 1024, TTL);
|
||||
|
||||
let first = cache
|
||||
.insert(key("source", &access(), "a", "ref", 100), build(first_ids))
|
||||
.await
|
||||
.unwrap();
|
||||
let second = cache
|
||||
.insert(key("source", &access(), "b", "ref", 100), build(second_ids))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(first.version() == second.version(), equal);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn snapshots_expire_after_the_ttl(trace: Trace) {
|
||||
let cache = SnapshotCache::new(1024 * 1024, Duration::from_millis(50));
|
||||
let stored = key("source", &access(), "trace", "ref", 100);
|
||||
|
||||
cache.insert(stored.clone(), trace).await.unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
|
||||
assert!(cache.get(&stored).await.is_none());
|
||||
}
|
||||
|
|
@ -21,6 +21,7 @@ litellm-http.workspace = true
|
|||
litellm-migrate.workspace = true
|
||||
litellm-storage-clickhouse.workspace = true
|
||||
litellm-traces.workspace = true
|
||||
litellm-traces-cache.workspace = true
|
||||
moka.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
|
|
|||
|
|
@ -0,0 +1,14 @@
|
|||
SELECT t.TraceId, t.SpanId, s.request_id, s.spend, s.metadata
|
||||
FROM otel_traces AS t
|
||||
INNER JOIN (
|
||||
SELECT *
|
||||
FROM spend_logs FINAL
|
||||
WHERE start_time >= now() - INTERVAL 1 DAY
|
||||
) AS s
|
||||
ON t.LiteLLMRequestId = s.response_id
|
||||
AND t.TeamId = s.team_id
|
||||
AND ((t.UserId != '' AND t.UserId = s.user)
|
||||
OR (t.ApiKeyHash != '' AND t.ApiKeyHash = s.api_key))
|
||||
WHERE t.Timestamp >= now() - INTERVAL 1 DAY
|
||||
AND t.LiteLLMRequestId != ''
|
||||
LIMIT 100
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
SELECT
|
||||
request_id, response_id, model, spend, JSONExtractString(metadata, 'project') AS project
|
||||
FROM spend_logs FINAL
|
||||
WHERE start_time >= now() - INTERVAL 1 DAY
|
||||
AND JSONHas(metadata, 'project')
|
||||
AND JSONExtractString(metadata, 'project') = 'example'
|
||||
ORDER BY start_time DESC
|
||||
LIMIT 100
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
SELECT
|
||||
DISTINCT arrayJoin(JSONExtractKeys(metadata)) AS key
|
||||
FROM spend_logs FINAL
|
||||
WHERE start_time >= now() - INTERVAL 30 DAY
|
||||
ORDER BY key
|
||||
LIMIT 200
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
SELECT TeamId AS team, ApiKeyHash AS api_key, TraceId AS trace_id,
|
||||
SpanId AS span_id, StatusMessage AS message
|
||||
FROM otel_traces
|
||||
WHERE Timestamp >= now() - INTERVAL 1 DAY
|
||||
AND StatusCode = 'STATUS_CODE_ERROR'
|
||||
ORDER BY Timestamp DESC, team, api_key, trace_id, span_id
|
||||
LIMIT 100
|
||||
|
|
@ -1,5 +1,8 @@
|
|||
SELECT team_id AS team, api_key, request_id, spend,
|
||||
JSONExtractString(metadata, 'labels', 'priority') AS priority
|
||||
FROM spend_logs FINAL
|
||||
WHERE JSONExtractString(metadata, 'labels', 'priority') = 'high'
|
||||
WHERE start_time >= now() - INTERVAL 1 DAY
|
||||
AND JSONHas(metadata, 'labels', 'priority')
|
||||
AND JSONExtractString(metadata, 'labels', 'priority') = 'high'
|
||||
ORDER BY team, api_key, request_id
|
||||
LIMIT 100
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
SELECT
|
||||
team_id, model, requests, unknown_cost_requests,
|
||||
if(unknown_cost_requests = 0, recorded_spend, NULL) AS spend,
|
||||
input_tokens, output_tokens
|
||||
FROM (
|
||||
SELECT
|
||||
team_id, model, count() AS requests,
|
||||
countIf(isNull(spend) OR NOT isFinite(spend)) AS unknown_cost_requests,
|
||||
sum(spend) AS recorded_spend,
|
||||
sum(prompt_tokens) AS input_tokens,
|
||||
sum(completion_tokens) AS output_tokens
|
||||
FROM spend_logs FINAL
|
||||
WHERE start_time >= now() - INTERVAL 1 DAY
|
||||
GROUP BY team_id, model
|
||||
)
|
||||
ORDER BY team_id, model
|
||||
LIMIT 100
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
SELECT
|
||||
request_id,
|
||||
JSONType(metadata, 'labels', 'priority') AS type,
|
||||
JSONExtractRaw(metadata, 'labels', 'priority') AS value
|
||||
FROM spend_logs FINAL
|
||||
WHERE start_time >= now() - INTERVAL 1 DAY
|
||||
AND JSONHas(metadata, 'labels', 'priority')
|
||||
LIMIT 100
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
SELECT
|
||||
TraceId, SpanId, Model, InputTokens, OutputTokens,
|
||||
Duration / 1000000 AS duration_ms
|
||||
FROM otel_traces
|
||||
WHERE Timestamp >= now() - INTERVAL 1 DAY
|
||||
AND ObservationType = 'llm'
|
||||
ORDER BY Timestamp DESC
|
||||
LIMIT 100
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
SELECT
|
||||
request_id, response_id, trace_id, span_id, model, spend,
|
||||
prompt_tokens, completion_tokens, status,
|
||||
JSONExtractBool(metadata, 'synthetic_spend') AS synthetic_spend
|
||||
FROM spend_logs FINAL
|
||||
WHERE start_time >= now() - INTERVAL 1 DAY
|
||||
ORDER BY start_time DESC, request_id
|
||||
LIMIT 100
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
SELECT
|
||||
team_id, api_key, trace_id, count() AS requests,
|
||||
countIf(isNull(spend) OR NOT isFinite(spend)) AS unknown_cost_requests,
|
||||
if(unknown_cost_requests = 0, sum(spend), NULL) AS recorded_spend
|
||||
FROM spend_logs FINAL
|
||||
WHERE start_time >= now() - INTERVAL 1 DAY
|
||||
AND trace_id != ''
|
||||
GROUP BY team_id, api_key, trace_id
|
||||
ORDER BY team_id, api_key, trace_id
|
||||
LIMIT 100
|
||||
|
|
@ -7,4 +7,6 @@ SELECT TeamId AS team, ApiKeyHash AS api_key, TraceId AS trace_id,
|
|||
toUInt32(sum(OutputTokens)) AS output_tokens
|
||||
FROM agent_traces_by_key
|
||||
GROUP BY TeamId, ApiKeyHash, TraceId
|
||||
HAVING min(StartTs) >= now() - INTERVAL 1 DAY
|
||||
ORDER BY team, api_key, trace_id
|
||||
LIMIT 100
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
SELECT
|
||||
t.TraceId, t.SpanId, t.Model, t.LiteLLMRequestId,
|
||||
t.InputTokens, t.OutputTokens
|
||||
FROM otel_traces AS t
|
||||
LEFT ANTI JOIN (
|
||||
SELECT *
|
||||
FROM spend_logs FINAL
|
||||
WHERE start_time >= now() - INTERVAL 1 DAY
|
||||
) AS s
|
||||
ON t.TeamId = s.team_id
|
||||
AND ((t.UserId != '' AND t.UserId = s.user)
|
||||
OR (t.ApiKeyHash != '' AND t.ApiKeyHash = s.api_key))
|
||||
AND t.LiteLLMRequestId != ''
|
||||
AND (t.LiteLLMRequestId = s.response_id OR t.LiteLLMRequestId = s.request_id)
|
||||
WHERE t.Timestamp >= now() - INTERVAL 1 DAY
|
||||
AND t.ObservationType = 'llm'
|
||||
ORDER BY t.Timestamp DESC, t.SpanId
|
||||
LIMIT 100
|
||||
|
|
@ -47,3 +47,12 @@ pub enum Error {
|
|||
#[error(transparent)]
|
||||
Cached(#[from] std::sync::Arc<Error>),
|
||||
}
|
||||
|
||||
impl From<litellm_traces_cache::Error> for Error {
|
||||
fn from(error: litellm_traces_cache::Error) -> Self {
|
||||
match error {
|
||||
litellm_traces_cache::Error::Serialization(_) => Self::InvalidResponse,
|
||||
litellm_traces_cache::Error::ReadTooLarge => Self::ReadTooLarge,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -297,7 +297,7 @@ pub struct QueryHelp {
|
|||
attributes: Vec<AttributeCatalog>,
|
||||
relationships: &'static [Relationship],
|
||||
#[cfg_attr(feature = "schema", schemars(with = "Vec<Example>"))]
|
||||
examples: [Example; 9],
|
||||
examples: [Example; 12],
|
||||
#[cfg_attr(feature = "schema", schemars(with = "Vec<String>"))]
|
||||
gotchas: [String; 13],
|
||||
guide: String,
|
||||
|
|
|
|||
|
|
@ -28,6 +28,12 @@ use crate::{Error, NormalizedFieldDefinition, query_access::ReaderLimits};
|
|||
"trace_spend_sql",
|
||||
"unmatched_spans_name",
|
||||
"unmatched_spans_sql",
|
||||
"trace_summary_name",
|
||||
"trace_summary_sql",
|
||||
"failed_spans_name",
|
||||
"failed_spans_sql",
|
||||
"metadata_filter_name",
|
||||
"metadata_filter_sql",
|
||||
"missing_spend",
|
||||
"partial_spend",
|
||||
"time_window",
|
||||
|
|
@ -60,7 +66,7 @@ impl QueryGuide<'_> {
|
|||
])
|
||||
}
|
||||
|
||||
pub fn examples(&self) -> Result<[Example; 9], Error> {
|
||||
pub fn examples(&self) -> Result<[Example; 12], Error> {
|
||||
Ok([
|
||||
Example {
|
||||
name: render(&self.as_recent_spans_name())?,
|
||||
|
|
@ -98,6 +104,18 @@ impl QueryGuide<'_> {
|
|||
name: render(&self.as_unmatched_spans_name())?,
|
||||
sql: render(&self.as_unmatched_spans_sql())?,
|
||||
},
|
||||
Example {
|
||||
name: render(&self.as_trace_summary_name())?,
|
||||
sql: render(&self.as_trace_summary_sql())?,
|
||||
},
|
||||
Example {
|
||||
name: render(&self.as_failed_spans_name())?,
|
||||
sql: render(&self.as_failed_spans_sql())?,
|
||||
},
|
||||
Example {
|
||||
name: render(&self.as_metadata_filter_name())?,
|
||||
sql: render(&self.as_metadata_filter_sql())?,
|
||||
},
|
||||
])
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
//! Scoped trace reads: the trace list, one trace resolved with its spend, and span payloads.
|
||||
|
||||
use std::sync::{Arc, LazyLock};
|
||||
use std::sync::LazyLock;
|
||||
use std::time::Duration;
|
||||
|
||||
use base64::{Engine, engine::general_purpose::URL_SAFE};
|
||||
|
|
@ -12,9 +12,8 @@ use litellm_traces::{
|
|||
SpanDetail, SpanErrorPage, SpendLookup, Trace, TracePage, listed_summary,
|
||||
query::named as contracts, resolve_trace, to_ui_content,
|
||||
};
|
||||
use moka::future::Cache;
|
||||
use litellm_traces_cache::{SnapshotCache, SnapshotKey};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::{
|
||||
Connection, Error,
|
||||
|
|
@ -38,17 +37,11 @@ impl Query for RunCandidates {
|
|||
}
|
||||
|
||||
// Cursor pages share a bounded snapshot so advancing does not resolve the whole graph again.
|
||||
static TRACE_SNAPSHOTS: LazyLock<Cache<String, Arc<Trace>>> = LazyLock::new(|| {
|
||||
Cache::builder()
|
||||
.max_capacity((2 * crate::span_batches::MAX_GRAPH_BYTES) as u64)
|
||||
.weigher(|_: &String, trace: &Arc<Trace>| {
|
||||
serde_json::to_vec(trace.as_ref())
|
||||
.ok()
|
||||
.and_then(|bytes| u32::try_from(bytes.len().saturating_mul(2)).ok())
|
||||
.unwrap_or(u32::MAX)
|
||||
})
|
||||
.time_to_live(Duration::from_secs(120))
|
||||
.build()
|
||||
static TRACE_SNAPSHOTS: LazyLock<SnapshotCache> = LazyLock::new(|| {
|
||||
SnapshotCache::new(
|
||||
crate::span_batches::MAX_GRAPH_BYTES,
|
||||
Duration::from_secs(120),
|
||||
)
|
||||
});
|
||||
|
||||
const NANOS_PER_MS: i64 = 1_000_000;
|
||||
|
|
@ -339,74 +332,57 @@ pub async fn get_trace_page(
|
|||
version: String::new(),
|
||||
},
|
||||
};
|
||||
let key_bytes = serde_json::to_vec(&(
|
||||
let key = SnapshotKey::new(
|
||||
connection.url().as_str(),
|
||||
access,
|
||||
trace_id,
|
||||
&trace_ref,
|
||||
position.snapshot_ms,
|
||||
))
|
||||
)
|
||||
.map_err(|_| Error::InvalidParameters)?;
|
||||
let key = format!("{:x}", Sha256::digest(key_bytes));
|
||||
let snapshot = if let Some(trace) = TRACE_SNAPSHOTS.get(&key).await {
|
||||
trace
|
||||
} else {
|
||||
let params = TraceSpansParams {
|
||||
access: access.clone(),
|
||||
trace_id: trace_id.to_owned(),
|
||||
trace_ref: trace_ref.clone(),
|
||||
};
|
||||
let rows =
|
||||
crate::span_batches::read_spans(client, connection, params, position.snapshot_ms)
|
||||
.await?;
|
||||
let spend_rows = spend(client, connection, access, &rows).await;
|
||||
let Some(trace) = resolve_trace(trace_id, &trace_ref, &rows, &spend_rows) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if serde_json::to_vec(&trace)
|
||||
.map_err(|_| Error::InvalidResponse)?
|
||||
.len()
|
||||
> crate::span_batches::MAX_GRAPH_BYTES
|
||||
{
|
||||
return Err(Error::ReadTooLarge);
|
||||
let snapshot = match TRACE_SNAPSHOTS.get(&key).await {
|
||||
Some(snapshot) => snapshot,
|
||||
None => {
|
||||
let params = TraceSpansParams {
|
||||
access: access.clone(),
|
||||
trace_id: trace_id.to_owned(),
|
||||
trace_ref: trace_ref.clone(),
|
||||
};
|
||||
let rows =
|
||||
crate::span_batches::read_spans(client, connection, params, position.snapshot_ms)
|
||||
.await?;
|
||||
let spend_rows = spend(client, connection, access, &rows).await;
|
||||
let Some(trace) = resolve_trace(trace_id, &trace_ref, &rows, &spend_rows) else {
|
||||
return Ok(None);
|
||||
};
|
||||
TRACE_SNAPSHOTS.insert(key, trace).await?
|
||||
}
|
||||
let trace = Arc::new(trace);
|
||||
TRACE_SNAPSHOTS.insert(key, Arc::clone(&trace)).await;
|
||||
trace
|
||||
};
|
||||
let span_ids: Vec<&str> = snapshot
|
||||
.spans
|
||||
.iter()
|
||||
.map(|span| span.span_id.as_str())
|
||||
.collect();
|
||||
let version = format!(
|
||||
"{:x}",
|
||||
Sha256::digest(serde_json::to_vec(&span_ids).map_err(|_| Error::InvalidResponse)?)
|
||||
);
|
||||
if cursor.is_some() && position.version != version {
|
||||
let spans = &snapshot.trace().spans;
|
||||
if cursor.is_some() && position.version != snapshot.version() {
|
||||
return Err(Error::TraceChanged);
|
||||
}
|
||||
let mut trace = Trace {
|
||||
summary: snapshot.summary.clone(),
|
||||
agents: snapshot.agents.clone(),
|
||||
summary: snapshot.trace().summary.clone(),
|
||||
agents: snapshot.trace().agents.clone(),
|
||||
spans: Vec::new(),
|
||||
next_cursor: None,
|
||||
};
|
||||
if position.offset > snapshot.spans.len() {
|
||||
if position.offset > spans.len() {
|
||||
return Err(Error::InvalidCursor("span"));
|
||||
}
|
||||
let end = position
|
||||
.offset
|
||||
.saturating_add(page_size as usize)
|
||||
.min(snapshot.spans.len());
|
||||
trace.next_cursor = (end < snapshot.spans.len()).then(|| {
|
||||
.min(spans.len());
|
||||
trace.next_cursor = (end < spans.len()).then(|| {
|
||||
encode_cursor(&SpanPosition {
|
||||
offset: end,
|
||||
version: version.clone(),
|
||||
version: snapshot.version().to_owned(),
|
||||
..position
|
||||
})
|
||||
});
|
||||
trace.spans = snapshot.spans[position.offset..end].to_vec();
|
||||
trace.spans = spans[position.offset..end].to_vec();
|
||||
while serde_json::to_vec(&trace)
|
||||
.map_err(|_| Error::InvalidResponse)?
|
||||
.len()
|
||||
|
|
@ -420,7 +396,7 @@ pub async fn get_trace_page(
|
|||
trace_ref: trace_ref.clone(),
|
||||
snapshot_ms: position.snapshot_ms,
|
||||
offset: position.offset + trace.spans.len(),
|
||||
version: version.clone(),
|
||||
version: snapshot.version().to_owned(),
|
||||
}));
|
||||
}
|
||||
Ok(Some(trace))
|
||||
|
|
|
|||
|
|
@ -60,14 +60,7 @@ Recent normalized LLM spans
|
|||
{%- endblock %}
|
||||
|
||||
{% block recent_spans_sql -%}
|
||||
SELECT
|
||||
TraceId, SpanId, Model, InputTokens, OutputTokens,
|
||||
Duration / 1000000 AS duration_ms
|
||||
FROM otel_traces
|
||||
WHERE Timestamp >= now() - INTERVAL 1 DAY
|
||||
AND ObservationType = 'llm'
|
||||
ORDER BY Timestamp DESC
|
||||
LIMIT 100
|
||||
{% include "../query/help/recent_spans.sql" %}
|
||||
{%- endblock %}
|
||||
|
||||
{% block custom_metadata_name -%}
|
||||
|
|
@ -75,14 +68,7 @@ Find calls by custom metadata
|
|||
{%- endblock %}
|
||||
|
||||
{% block custom_metadata_sql -%}
|
||||
SELECT
|
||||
request_id, response_id, model, spend, JSONExtractString(metadata, 'project') AS project
|
||||
FROM spend_logs FINAL
|
||||
WHERE start_time >= now() - INTERVAL 1 DAY
|
||||
AND JSONHas(metadata, 'project')
|
||||
AND JSONExtractString(metadata, 'project') = 'example'
|
||||
ORDER BY start_time DESC
|
||||
LIMIT 100
|
||||
{% include "../query/help/custom_metadata.sql" %}
|
||||
{%- endblock %}
|
||||
|
||||
{% block nested_metadata_name -%}
|
||||
|
|
@ -90,14 +76,7 @@ Nested metadata with unknown types
|
|||
{%- endblock %}
|
||||
|
||||
{% block nested_metadata_sql -%}
|
||||
SELECT
|
||||
request_id,
|
||||
JSONType(metadata, 'labels', 'priority') AS type,
|
||||
JSONExtractRaw(metadata, 'labels', 'priority') AS value
|
||||
FROM spend_logs FINAL
|
||||
WHERE start_time >= now() - INTERVAL 1 DAY
|
||||
AND JSONHas(metadata, 'labels', 'priority')
|
||||
LIMIT 100
|
||||
{% include "../query/help/nested_metadata.sql" %}
|
||||
{%- endblock %}
|
||||
|
||||
{% block correlated_calls_name -%}
|
||||
|
|
@ -105,20 +84,7 @@ Traces correlated with LLM call metadata
|
|||
{%- endblock %}
|
||||
|
||||
{% block correlated_calls_sql -%}
|
||||
SELECT t.TraceId, t.SpanId, s.request_id, s.spend, s.metadata
|
||||
FROM otel_traces AS t
|
||||
INNER JOIN (
|
||||
SELECT *
|
||||
FROM spend_logs FINAL
|
||||
WHERE start_time >= now() - INTERVAL 1 DAY
|
||||
) AS s
|
||||
ON t.LiteLLMRequestId = s.response_id
|
||||
AND t.TeamId = s.team_id
|
||||
AND ((t.UserId != '' AND t.UserId = s.user)
|
||||
OR (t.ApiKeyHash != '' AND t.ApiKeyHash = s.api_key))
|
||||
WHERE t.Timestamp >= now() - INTERVAL 1 DAY
|
||||
AND t.LiteLLMRequestId != ''
|
||||
LIMIT 100
|
||||
{% include "../query/help/correlated_calls.sql" %}
|
||||
{%- endblock %}
|
||||
|
||||
{% block discover_keys_name -%}
|
||||
|
|
@ -126,12 +92,7 @@ Discover metadata keys over a different window
|
|||
{%- endblock %}
|
||||
|
||||
{% block discover_keys_sql -%}
|
||||
SELECT
|
||||
DISTINCT arrayJoin(JSONExtractKeys(metadata)) AS key
|
||||
FROM spend_logs FINAL
|
||||
WHERE start_time >= now() - INTERVAL 30 DAY
|
||||
ORDER BY key
|
||||
LIMIT 200
|
||||
{% include "../query/help/discover_keys.sql" %}
|
||||
{%- endblock %}
|
||||
|
||||
{% block recent_spend_name -%}
|
||||
|
|
@ -139,14 +100,7 @@ Recent spend records
|
|||
{%- endblock %}
|
||||
|
||||
{% block recent_spend_sql -%}
|
||||
SELECT
|
||||
request_id, response_id, trace_id, span_id, model, spend,
|
||||
prompt_tokens, completion_tokens, status,
|
||||
JSONExtractBool(metadata, 'synthetic_spend') AS synthetic_spend
|
||||
FROM spend_logs FINAL
|
||||
WHERE start_time >= now() - INTERVAL 1 DAY
|
||||
ORDER BY start_time DESC, request_id
|
||||
LIMIT 100
|
||||
{% include "../query/help/recent_spend.sql" %}
|
||||
{%- endblock %}
|
||||
|
||||
{% block model_spend_name -%}
|
||||
|
|
@ -154,23 +108,7 @@ Spend and tokens by model
|
|||
{%- endblock %}
|
||||
|
||||
{% block model_spend_sql -%}
|
||||
SELECT
|
||||
team_id, model, requests, unknown_cost_requests,
|
||||
if(unknown_cost_requests = 0, recorded_spend, NULL) AS spend,
|
||||
input_tokens, output_tokens
|
||||
FROM (
|
||||
SELECT
|
||||
team_id, model, count() AS requests,
|
||||
countIf(isNull(spend) OR NOT isFinite(spend)) AS unknown_cost_requests,
|
||||
sum(spend) AS recorded_spend,
|
||||
sum(prompt_tokens) AS input_tokens,
|
||||
sum(completion_tokens) AS output_tokens
|
||||
FROM spend_logs FINAL
|
||||
WHERE start_time >= now() - INTERVAL 1 DAY
|
||||
GROUP BY team_id, model
|
||||
)
|
||||
ORDER BY team_id, model
|
||||
LIMIT 100
|
||||
{% include "../query/help/model_spend.sql" %}
|
||||
{%- endblock %}
|
||||
|
||||
{% block trace_spend_name -%}
|
||||
|
|
@ -178,16 +116,7 @@ Recorded spend by trace
|
|||
{%- endblock %}
|
||||
|
||||
{% block trace_spend_sql -%}
|
||||
SELECT
|
||||
team_id, api_key, trace_id, count() AS requests,
|
||||
countIf(isNull(spend) OR NOT isFinite(spend)) AS unknown_cost_requests,
|
||||
if(unknown_cost_requests = 0, sum(spend), NULL) AS recorded_spend
|
||||
FROM spend_logs FINAL
|
||||
WHERE start_time >= now() - INTERVAL 1 DAY
|
||||
AND trace_id != ''
|
||||
GROUP BY team_id, api_key, trace_id
|
||||
ORDER BY team_id, api_key, trace_id
|
||||
LIMIT 100
|
||||
{% include "../query/help/trace_spend.sql" %}
|
||||
{%- endblock %}
|
||||
|
||||
{% block unmatched_spans_name -%}
|
||||
|
|
@ -195,24 +124,31 @@ LLM spans without a direct spend match
|
|||
{%- endblock %}
|
||||
|
||||
{% block unmatched_spans_sql -%}
|
||||
SELECT
|
||||
t.TraceId, t.SpanId, t.Model, t.LiteLLMRequestId,
|
||||
t.InputTokens, t.OutputTokens
|
||||
FROM otel_traces AS t
|
||||
LEFT ANTI JOIN (
|
||||
SELECT *
|
||||
FROM spend_logs FINAL
|
||||
WHERE start_time >= now() - INTERVAL 1 DAY
|
||||
) AS s
|
||||
ON t.TeamId = s.team_id
|
||||
AND ((t.UserId != '' AND t.UserId = s.user)
|
||||
OR (t.ApiKeyHash != '' AND t.ApiKeyHash = s.api_key))
|
||||
AND t.LiteLLMRequestId != ''
|
||||
AND (t.LiteLLMRequestId = s.response_id OR t.LiteLLMRequestId = s.request_id)
|
||||
WHERE t.Timestamp >= now() - INTERVAL 1 DAY
|
||||
AND t.ObservationType = 'llm'
|
||||
ORDER BY t.Timestamp DESC, t.SpanId
|
||||
LIMIT 100
|
||||
{% include "../query/help/unmatched_spans.sql" %}
|
||||
{%- endblock %}
|
||||
|
||||
{% block trace_summary_name -%}
|
||||
Trace summaries with tokens and errors
|
||||
{%- endblock %}
|
||||
|
||||
{% block trace_summary_sql -%}
|
||||
{% include "../query/help/trace_summary.sql" %}
|
||||
{%- endblock %}
|
||||
|
||||
{% block failed_spans_name -%}
|
||||
Recent failed spans
|
||||
{%- endblock %}
|
||||
|
||||
{% block failed_spans_sql -%}
|
||||
{% include "../query/help/failed_spans.sql" %}
|
||||
{%- endblock %}
|
||||
|
||||
{% block metadata_filter_name -%}
|
||||
Filter calls by nested metadata
|
||||
{%- endblock %}
|
||||
|
||||
{% block metadata_filter_sql -%}
|
||||
{% include "../query/help/metadata_filter.sql" %}
|
||||
{%- endblock %}
|
||||
|
||||
{% block time_window -%}
|
||||
|
|
|
|||
|
|
@ -1683,7 +1683,15 @@ async fn query_help_discovers_live_schema_and_runs_its_examples(
|
|||
let values: serde_json::Value = serde_json::from_str(&body)?;
|
||||
assert_eq!(
|
||||
values["data"].as_array().ok_or("missing data")?.is_empty(),
|
||||
!populated || example["name"] == "LLM spans without a direct spend match",
|
||||
!populated
|
||||
|| matches!(
|
||||
example["name"].as_str(),
|
||||
Some(
|
||||
"LLM spans without a direct spend match"
|
||||
| "Recent failed spans"
|
||||
| "Filter calls by nested metadata"
|
||||
)
|
||||
),
|
||||
"{sql}"
|
||||
);
|
||||
if populated && example["name"] == "Traces correlated with LLM call metadata" {
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ use litellm_traces::query::named as contracts;
|
|||
use litellm_traces_clickhouse::{
|
||||
QueryScope,
|
||||
query::named::{ListTraces, ListTracesParams, TraceSpans, TraceSpansParams},
|
||||
query_sql,
|
||||
query_help, query_sql,
|
||||
};
|
||||
use rstest::{fixture, rstest};
|
||||
use serde::Deserialize;
|
||||
|
|
@ -48,12 +48,9 @@ struct QueryResult {
|
|||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::rollups(include_str!("queries/rollups.sql"), include_str!("queries/rollups.expected.json"))]
|
||||
#[case::costs(include_str!("queries/trace_costs.sql"), include_str!("queries/trace_costs.expected.json"))]
|
||||
#[case::errors(include_str!("queries/failed_spans.sql"), include_str!("queries/failed_spans.expected.json"))]
|
||||
#[case::metadata(include_str!("queries/metadata_filters.sql"), include_str!("queries/metadata_filters.expected.json"))]
|
||||
#[tokio::test]
|
||||
async fn curated_queries_return_expected_rows(
|
||||
async fn storage_queries_return_expected_rows(
|
||||
#[future(awt)] seeded_database: TestResult<SeededDatabase>,
|
||||
#[case] sql: &str,
|
||||
#[case] expected_json: &str,
|
||||
|
|
@ -78,6 +75,64 @@ async fn curated_queries_return_expected_rows(
|
|||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::trace_summary("Trace summaries with tokens and errors", include_str!("../query/help/trace_summary.sql"), include_str!("queries/rollups.expected.json"))]
|
||||
#[case::failed_spans("Recent failed spans", include_str!("../query/help/failed_spans.sql"), include_str!("queries/failed_spans.expected.json"))]
|
||||
#[case::metadata_filter("Filter calls by nested metadata", include_str!("../query/help/metadata_filter.sql"), include_str!("queries/metadata_filters.expected.json"))]
|
||||
#[tokio::test]
|
||||
async fn documented_queries_render_and_return_expected_rows(
|
||||
#[future(awt)] seeded_database: TestResult<SeededDatabase>,
|
||||
fixture_clock: TestResult<u64>,
|
||||
#[case] name: &str,
|
||||
#[case] expected_sql: &str,
|
||||
#[case] expected_json: &str,
|
||||
#[values(ScopeCase::Admin, ScopeCase::Team, ScopeCase::OtherTeam)] scope: ScopeCase,
|
||||
) -> TestResult {
|
||||
let fixture = seeded_database?;
|
||||
let reader = fixture
|
||||
.readers
|
||||
.connection(&fixture.database.client, &scope.scope(), "fixture-secret")
|
||||
.await?;
|
||||
let help = serde_json::to_value(query_help(&fixture.database.client, &reader).await?)?;
|
||||
let example = help["examples"]
|
||||
.as_array()
|
||||
.ok_or("missing examples")?
|
||||
.iter()
|
||||
.find(|example| example["name"] == name)
|
||||
.ok_or("missing documented query")?;
|
||||
let sql = example["sql"].as_str().ok_or("missing example SQL")?;
|
||||
assert_eq!(sql.trim(), expected_sql.trim());
|
||||
assert!(
|
||||
help["guide"]
|
||||
.as_str()
|
||||
.ok_or("missing guide")?
|
||||
.contains(&format!("{name}\n{sql}"))
|
||||
);
|
||||
let sql_at_fixture_time = sql.replace("now()", &format!("toDateTime({})", fixture_clock?));
|
||||
let result: QueryResult = serde_json::from_str(
|
||||
&query_sql(&fixture.database.client, &reader, &sql_at_fixture_time).await?,
|
||||
)?;
|
||||
let expected: BTreeMap<String, Vec<Value>> = serde_json::from_str(expected_json)?;
|
||||
assert_eq!(
|
||||
&result.data,
|
||||
expected
|
||||
.get(scope.as_ref())
|
||||
.ok_or("missing expected scope")?,
|
||||
"{}: {sql}",
|
||||
scope.as_ref()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[fixture]
|
||||
fn fixture_clock() -> TestResult<u64> {
|
||||
let spans = litellm_traces::decode_otlp(
|
||||
include_bytes!("../../traces/tests/fixtures/query_root.json"),
|
||||
Some("application/json"),
|
||||
)?;
|
||||
Ok(spans.first().ok_or("missing fixture root")?.start_ns / 1_000_000_000)
|
||||
}
|
||||
|
||||
#[fixture]
|
||||
fn admin_access() -> TestResult<contracts::ReadAccessParams> {
|
||||
Ok(serde_json::from_str(include_str!(
|
||||
|
|
|
|||
|
|
@ -1,5 +0,0 @@
|
|||
SELECT TeamId AS team, ApiKeyHash AS api_key, TraceId AS trace_id,
|
||||
SpanId AS span_id, StatusMessage AS message
|
||||
FROM otel_traces
|
||||
WHERE StatusCode = 'STATUS_CODE_ERROR'
|
||||
ORDER BY team, api_key, trace_id, span_id
|
||||
|
|
@ -927,6 +927,12 @@ class MCPClient:
|
|||
|
||||
async def _call_tool_operation(session: ClientSession):
|
||||
verbose_logger.debug("MCP client sending tool call to session")
|
||||
if self.protocol_version == "2026-07-28":
|
||||
tools: Final = await list_tools_with_pagination(
|
||||
session, listing_deadline=max(self.timeout, MCP_TOOL_LISTING_TIMEOUT)
|
||||
)
|
||||
if not any(tool.name == call_tool_request_params.name for tool in tools):
|
||||
raise MCPError(code=-32603, message="Tool schema is unavailable from the bounded upstream catalog")
|
||||
return await session.call_tool(
|
||||
name=call_tool_request_params.name,
|
||||
arguments=call_tool_request_params.arguments,
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from typing import TYPE_CHECKING, Any, Final, Protocol
|
|||
|
||||
import httpx
|
||||
import openai
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.constants import (
|
||||
|
|
@ -46,6 +47,9 @@ if TYPE_CHECKING:
|
|||
from uvicorn import Server
|
||||
|
||||
verbose_logger: Final = logging.getLogger("LiteLLM")
|
||||
_JSON_VALUE_ADAPTER: Final = TypeAdapter(object, config=ConfigDict(strict=True))
|
||||
_JSON_OBJECT_ADAPTER: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
|
||||
_PORT_ADAPTER: Final = TypeAdapter(int, config=ConfigDict(strict=True))
|
||||
|
||||
MISSING_DEPS_MESSAGE = "litellm.harness needs starlette and uvicorn: pip install starlette uvicorn"
|
||||
|
||||
|
|
@ -207,11 +211,11 @@ class SSEUsageParser:
|
|||
if not payload or payload == b"[DONE]":
|
||||
return
|
||||
try:
|
||||
event = json.loads(payload)
|
||||
event: Final[object] = _JSON_VALUE_ADAPTER.validate_python(json.loads(payload))
|
||||
except ValueError:
|
||||
return
|
||||
if isinstance(event, Mapping):
|
||||
self.absorb(event)
|
||||
self.absorb(_JSON_OBJECT_ADAPTER.validate_python(event))
|
||||
|
||||
def absorb(self, event: Mapping[str, object]) -> None:
|
||||
event_type = event.get("type")
|
||||
|
|
@ -434,7 +438,7 @@ class ModelEndpoint:
|
|||
except BaseException:
|
||||
await self.stop()
|
||||
raise
|
||||
self.port = self._server.servers[0].sockets[0].getsockname()[1]
|
||||
self.port = _PORT_ADAPTER.validate_python(self._server.servers[0].sockets[0].getsockname()[1])
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._server is not None:
|
||||
|
|
@ -540,11 +544,12 @@ class ModelEndpoint:
|
|||
if not self._authorized(request):
|
||||
return self._unauthorized()
|
||||
try:
|
||||
body = json.loads(await request.body())
|
||||
parsed_body: Final[object] = _JSON_VALUE_ADAPTER.validate_python(json.loads(await request.body()))
|
||||
except ValueError as e:
|
||||
return self._error(e, 400)
|
||||
if not isinstance(body, dict):
|
||||
if not isinstance(parsed_body, dict):
|
||||
return self._error(ValueError("request body must be a JSON object"), 400)
|
||||
body: Final = _JSON_OBJECT_ADAPTER.validate_python(parsed_body)
|
||||
route = route_of(request.url.path)
|
||||
if self.gateway is not None:
|
||||
return await self._forward(request, route, body)
|
||||
|
|
@ -615,7 +620,8 @@ class ModelEndpoint:
|
|||
tokens = (parser.input_tokens, parser.output_tokens)
|
||||
else:
|
||||
try:
|
||||
tokens = usage_from_body(json.loads(collected))
|
||||
body: Final[object] = _JSON_VALUE_ADAPTER.validate_python(json.loads(collected))
|
||||
tokens = usage_from_body(body)
|
||||
except ValueError:
|
||||
tokens = (0, 0)
|
||||
self._record(model, tokens[0], tokens[1], header_cost(upstream.headers))
|
||||
|
|
|
|||
|
|
@ -186,7 +186,7 @@ class Session:
|
|||
|
||||
def history(
|
||||
self,
|
||||
) -> list[dict[str, Any]]: # mutable-ok: public API returns OpenAI-format message dicts from the handler
|
||||
) -> list[dict[str, object]]: # mutable-ok: public API returns OpenAI-format message dicts from the handler
|
||||
return run_sync(self._inner.history(), "history")
|
||||
|
||||
@property
|
||||
|
|
|
|||
|
|
@ -492,7 +492,7 @@ class GalileoObserve(CustomLogger):
|
|||
@staticmethod
|
||||
def _get_chat_content_for_galileo(response_obj: litellm.ModelResponse) -> object:
|
||||
if response_obj.choices and len(response_obj.choices) > 0:
|
||||
message: Final = response_obj["choices"][0]["message"]
|
||||
message: Final = response_obj.choices[0].message
|
||||
if hasattr(message, "json"):
|
||||
message_json: Final[object] = message.json()
|
||||
if isinstance(message_json, str):
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ from litellm.types.llms.openai import (
|
|||
ResponsesAPIResponse,
|
||||
)
|
||||
from litellm.types.prompts.init_prompts import PromptSpec
|
||||
from litellm.types.utils import CallTypes, StandardCallbackDynamicParams
|
||||
from litellm.types.utils import CallTypes, LLMResponseTypes, ModelResponse, StandardCallbackDynamicParams
|
||||
from litellm.types.vector_stores import (
|
||||
LiteLLM_ManagedVectorStore,
|
||||
VectorStoreSearchFailure,
|
||||
|
|
@ -46,6 +46,7 @@ else:
|
|||
SEARCH_FAILURES_FIELD: Final = "vector_store_search_failures"
|
||||
_DEFAULT_FAILURE_MODE: Final[VectorStoreSearchFailureMode] = "annotate"
|
||||
_FAILURE_MODE_ADAPTER: Final = TypeAdapter(VectorStoreSearchFailureMode)
|
||||
_OBJECT_ADAPTER: Final = TypeAdapter(object)
|
||||
_STR_KEYED_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
_GUARDRAIL_KEYS_THE_PROXY_MERGES_INTO_METADATA: Final = frozenset(
|
||||
{"guardrails", "guardrail_config", "policies", "include_guardrail_response"}
|
||||
|
|
@ -269,7 +270,9 @@ class VectorStorePreCallHook(CustomLogger):
|
|||
verbose_logger.debug("No query found in messages for vector store search")
|
||||
return None
|
||||
|
||||
request_litellm_params: Final = litellm_logging_obj.model_call_details.get("litellm_params", {})
|
||||
request_litellm_params: Final = _OBJECT_ADAPTER.validate_python(
|
||||
litellm_logging_obj.model_call_details.get("litellm_params", {})
|
||||
)
|
||||
request_metadata: Final = (
|
||||
request_litellm_params.get("metadata", {}) if isinstance(request_litellm_params, dict) else {}
|
||||
)
|
||||
|
|
@ -415,9 +418,9 @@ class VectorStorePreCallHook(CustomLogger):
|
|||
async def async_post_call_success_deployment_hook(
|
||||
self,
|
||||
request_data: dict,
|
||||
response: Any,
|
||||
response: LLMResponseTypes,
|
||||
call_type: CallTypes | None,
|
||||
) -> Any | None:
|
||||
) -> LLMResponseTypes | None:
|
||||
"""
|
||||
Add search results to the response after successful LLM call.
|
||||
|
||||
|
|
@ -451,7 +454,7 @@ class VectorStorePreCallHook(CustomLogger):
|
|||
return response
|
||||
|
||||
# Add search results to response object
|
||||
if hasattr(response, "choices") and response.choices:
|
||||
if isinstance(response, ModelResponse) and response.choices:
|
||||
for choice in response.choices:
|
||||
if hasattr(choice, "message") and choice.message:
|
||||
provider_fields = getattr(choice.message, "provider_specific_fields", None) or {}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ from typing import Final, Literal
|
|||
import litellm
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantle_chat_config
|
||||
from litellm.types.utils import LlmProviders, LlmProvidersSet
|
||||
|
||||
|
||||
|
|
@ -104,7 +105,7 @@ def get_supported_openai_params(
|
|||
elif custom_llm_provider == "groq":
|
||||
return litellm.GroqChatConfig().get_supported_openai_params(model=model)
|
||||
elif custom_llm_provider == "bedrock_mantle":
|
||||
return litellm.BedrockMantleChatConfig().get_supported_openai_params(model=model)
|
||||
return bedrock_mantle_chat_config(model).get_supported_openai_params(model=model)
|
||||
elif custom_llm_provider == "hosted_vllm":
|
||||
return litellm.HostedVLLMChatConfig().get_supported_openai_params(model=model)
|
||||
elif custom_llm_provider == "vllm":
|
||||
|
|
|
|||
|
|
@ -1741,11 +1741,14 @@ def _find_server_tool_result(
|
|||
)
|
||||
|
||||
|
||||
_AnthropicToolInvokeItem: TypeAlias = AnthropicMessagesToolUseParam | dict[str, Any]
|
||||
|
||||
|
||||
def convert_to_anthropic_tool_invoke(
|
||||
tool_calls: list[ChatCompletionAssistantToolCall],
|
||||
web_search_results: Sequence[object] | None = None,
|
||||
tool_results: Sequence[object] | None = None,
|
||||
) -> list[AnthropicMessagesToolUseParam | dict[str, Any]]:
|
||||
) -> list[_AnthropicToolInvokeItem]:
|
||||
"""
|
||||
OpenAI tool invokes:
|
||||
{
|
||||
|
|
@ -2565,9 +2568,9 @@ def anthropic_messages_pt(
|
|||
|
||||
# Group tool invoke results into (server_tool_use, result) pairs
|
||||
# and separate regular tool_use blocks
|
||||
server_tool_groups: list[list[Any]] = []
|
||||
regular_tool_uses: list[Any] = []
|
||||
_current_group: list[Any] = []
|
||||
server_tool_groups: list[list[_AnthropicToolInvokeItem]] = []
|
||||
regular_tool_uses: list[_AnthropicToolInvokeItem] = []
|
||||
_current_group: list[_AnthropicToolInvokeItem] = []
|
||||
for item in tool_invoke_results:
|
||||
item_type = item.get("type", "") if isinstance(item, dict) else getattr(item, "type", "")
|
||||
if item_type == "server_tool_use":
|
||||
|
|
|
|||
33
litellm/llms/bedrock_mantle/chat/claude_transformation.py
Normal file
33
litellm/llms/bedrock_mantle/chat/claude_transformation.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
from litellm.llms.base_llm.chat.transformation import BaseConfig
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.bedrock.chat.mantle.transformation import AmazonMantleConfig
|
||||
from litellm.llms.bedrock_mantle.chat.transformation import BedrockMantleChatConfig
|
||||
from litellm.llms.bedrock_mantle.common_utils import BedrockMantleAuthMixin, is_mantle_claude_model
|
||||
from litellm.llms.bedrock_mantle.messages.transformation import build_mantle_native_messages_url
|
||||
|
||||
|
||||
class BedrockMantleClaudeChatConfig(BedrockMantleAuthMixin, AmazonMantleConfig):
|
||||
def __init__(self, aws_signer: BaseAWSLLM | None = None) -> None:
|
||||
AmazonMantleConfig.__init__(self)
|
||||
self._aws_signer = aws_signer or self
|
||||
|
||||
@property
|
||||
def custom_llm_provider(self) -> str | None:
|
||||
return "bedrock_mantle"
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: str | None,
|
||||
api_key: str | None,
|
||||
model: str,
|
||||
optional_params: dict[str, object],
|
||||
litellm_params: dict[str, object],
|
||||
stream: bool | None = None,
|
||||
) -> str:
|
||||
return build_mantle_native_messages_url(api_base=api_base, litellm_params=litellm_params)
|
||||
|
||||
|
||||
def bedrock_mantle_chat_config(model: str) -> BaseConfig:
|
||||
if is_mantle_claude_model(model):
|
||||
return BedrockMantleClaudeChatConfig()
|
||||
return BedrockMantleChatConfig()
|
||||
|
|
@ -127,6 +127,10 @@ class BedrockMantleAuthMixin(SignsRequestsWithAWS):
|
|||
) from e
|
||||
|
||||
|
||||
def is_mantle_claude_model(model: str) -> bool:
|
||||
return "claude" in model.lower()
|
||||
|
||||
|
||||
def mantle_supports_responses(model: str | None, model_cost: dict) -> bool:
|
||||
"""Whether a Bedrock Mantle model can serve the native Responses API.
|
||||
|
||||
|
|
|
|||
|
|
@ -129,7 +129,7 @@ class ClaudeCodeStreamState:
|
|||
result_text: str | None = None
|
||||
is_error: bool = False
|
||||
errors: Sequence[str] = ()
|
||||
structured_output: Any | None = None
|
||||
structured_output: object = None
|
||||
|
||||
@property
|
||||
def final_text(self) -> str:
|
||||
|
|
@ -157,7 +157,7 @@ def _stringify_block(block: object) -> str:
|
|||
return json.dumps(block, ensure_ascii=False)
|
||||
|
||||
|
||||
def _message_blocks(event: Mapping[str, Any]) -> Sequence[Any]:
|
||||
def _message_blocks(event: Mapping[str, object]) -> Sequence[Any]:
|
||||
message: Final = event.get("message")
|
||||
content: Final = message.get("content") if isinstance(message, Mapping) else None
|
||||
if isinstance(content, str):
|
||||
|
|
@ -187,7 +187,7 @@ def _assistant_block_events(block: Mapping[str, Any], state: ClaudeCodeStreamSta
|
|||
return ()
|
||||
|
||||
|
||||
def _assistant_events(event: Mapping[str, Any], state: ClaudeCodeStreamState) -> Sequence[Event]:
|
||||
def _assistant_events(event: Mapping[str, object], state: ClaudeCodeStreamState) -> Sequence[Event]:
|
||||
if event.get("parent_tool_use_id"):
|
||||
return event_list() # subagent traffic
|
||||
message: Final = event.get("message")
|
||||
|
|
@ -201,7 +201,7 @@ def _is_tool_result(block: object) -> bool:
|
|||
return isinstance(block, dict) and block.get("type") == "tool_result"
|
||||
|
||||
|
||||
def _user_events(event: Mapping[str, Any]) -> Sequence[Event]:
|
||||
def _user_events(event: Mapping[str, object]) -> Sequence[Event]:
|
||||
if event.get("parent_tool_use_id"):
|
||||
return event_list()
|
||||
return event_list(
|
||||
|
|
@ -217,7 +217,7 @@ def _user_events(event: Mapping[str, Any]) -> Sequence[Event]:
|
|||
)
|
||||
|
||||
|
||||
def _system_events(event: Mapping[str, Any], state: ClaudeCodeStreamState) -> Sequence[Event]:
|
||||
def _system_events(event: Mapping[str, object], state: ClaudeCodeStreamState) -> Sequence[Event]:
|
||||
subtype = event.get("subtype")
|
||||
if subtype == "init" and event.get("session_id"):
|
||||
state.session_id = str(event["session_id"])
|
||||
|
|
@ -253,7 +253,7 @@ def turn_error_message(state: ClaudeCodeStreamState, exit_code: int, stderr_tail
|
|||
return f"{message}\nstderr:\n{tail}" if tail else message
|
||||
|
||||
|
||||
def build_system_prompt(instructions: str | None, output_schema: Mapping[str, Any] | None) -> str | None:
|
||||
def build_system_prompt(instructions: str | None, output_schema: Mapping[str, object] | None) -> str | None:
|
||||
schema_part: Final = (
|
||||
STRUCTURED_OUTPUT_INSTRUCTION.format(schema=json.dumps(output_schema)) if output_schema is not None else None
|
||||
)
|
||||
|
|
@ -355,7 +355,7 @@ class ClaudeCodeHarnessConfig(BaseCLIHarnessConfig):
|
|||
def create_stream_state(self) -> ClaudeCodeStreamState:
|
||||
return ClaudeCodeStreamState()
|
||||
|
||||
def transform_stream_line(self, line: Mapping[str, Any], state: ClaudeCodeStreamState) -> Sequence[Event]:
|
||||
def transform_stream_line(self, line: Mapping[str, object], state: ClaudeCodeStreamState) -> Sequence[Event]:
|
||||
kind = line.get("type")
|
||||
if kind == "assistant":
|
||||
return _assistant_events(line, state)
|
||||
|
|
|
|||
|
|
@ -117,7 +117,7 @@ class OllamaChatConfig(BaseConfig):
|
|||
system: str | None = None,
|
||||
template: str | None = None,
|
||||
) -> None:
|
||||
locals_: Final = locals().copy()
|
||||
locals_: Final[dict[str, object]] = locals().copy()
|
||||
for key, value in locals_.items():
|
||||
if key != "self" and value is not None:
|
||||
setattr(self.__class__, key, value)
|
||||
|
|
|
|||
|
|
@ -121,6 +121,7 @@ from litellm.llms.bedrock.common_utils import (
|
|||
bedrock_route_for_request,
|
||||
without_bedrock_route_prefix,
|
||||
)
|
||||
from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantle_chat_config
|
||||
from litellm.llms.cohere.common_utils import CohereModelInfo
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled
|
||||
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
|
||||
|
|
@ -2192,7 +2193,7 @@ def _complete_bedrock_mantle(
|
|||
api_base = api_base or litellm.api_base or get_secret("BEDROCK_MANTLE_API_BASE")
|
||||
api_key = api_key or litellm.api_key or get_secret("BEDROCK_MANTLE_API_KEY")
|
||||
headers = headers or litellm.headers
|
||||
config: Final = litellm.BedrockMantleChatConfig.get_config()
|
||||
config: Final = bedrock_mantle_chat_config(model).get_config()
|
||||
for k, v in _provider_config_items(config):
|
||||
if k not in optional_params:
|
||||
optional_params[k] = v
|
||||
|
|
|
|||
|
|
@ -16085,6 +16085,28 @@
|
|||
"input_cost_per_image_token": 4.7e-07,
|
||||
"source": "https://cohere.com/pricing"
|
||||
},
|
||||
"cohere/embed-v5.0-pro": {
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"litellm_provider": "cohere",
|
||||
"max_input_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"output_vector_size": 2048,
|
||||
"supports_embedding_image_input": true,
|
||||
"source": "https://cohere.com/pricing"
|
||||
},
|
||||
"cohere/embed-v5.0-fast": {
|
||||
"input_cost_per_token": 8e-08,
|
||||
"litellm_provider": "cohere",
|
||||
"max_input_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"output_vector_size": 2048,
|
||||
"supports_embedding_image_input": true,
|
||||
"source": "https://cohere.com/pricing"
|
||||
},
|
||||
"cohere/parse-v5.0": {
|
||||
"litellm_provider": "cohere",
|
||||
"mode": "ocr",
|
||||
|
|
@ -61056,6 +61078,9 @@
|
|||
"input_cost_per_audio_token": 2e-06,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 98304,
|
||||
"max_output_tokens": 32768,
|
||||
"max_tokens": 32768,
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_token": 1.2e-05,
|
||||
"source": "https://ai.google.dev/gemini-api/docs/pricing",
|
||||
|
|
@ -61078,6 +61103,9 @@
|
|||
"input_cost_per_audio_token": 3.5e-06,
|
||||
"input_cost_per_token": 3.5e-06,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 65536,
|
||||
"max_tokens": 65536,
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_token": 2.1e-05,
|
||||
"source": "https://ai.google.dev/gemini-api/docs/pricing",
|
||||
|
|
@ -64092,6 +64120,23 @@
|
|||
"video"
|
||||
]
|
||||
},
|
||||
"xai/grok-imagine-video-1.5-lite": {
|
||||
"input_cost_per_image": 0.01,
|
||||
"litellm_provider": "xai",
|
||||
"mode": "video_generation",
|
||||
"output_cost_per_second": 0.02,
|
||||
"output_cost_per_second_1080p": 0.14,
|
||||
"output_cost_per_second_480p": 0.02,
|
||||
"output_cost_per_second_720p": 0.03,
|
||||
"source": "https://docs.x.ai/developers/pricing",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"video"
|
||||
]
|
||||
},
|
||||
"xai/grok-imagine-video-1.5-2026-05-30": {
|
||||
"input_cost_per_image": 0.01,
|
||||
"litellm_provider": "xai",
|
||||
|
|
@ -75511,6 +75556,42 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true
|
||||
},
|
||||
"openrouter/openai/gpt-image-2": {
|
||||
"input_cost_per_image_token": 8e-06,
|
||||
"input_cost_per_token": 5e-06,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "image_generation",
|
||||
"output_cost_per_image_token": 3e-05,
|
||||
"source": "https://openrouter.ai/api/v1/images/models/openai/gpt-image-2/endpoints",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
],
|
||||
"supports_vision": true
|
||||
},
|
||||
"openrouter/openai/gpt-image-2.5-flare": {
|
||||
"input_cost_per_image_token": 8e-06,
|
||||
"input_cost_per_token": 5e-06,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "image_generation",
|
||||
"output_cost_per_image_token": 3e-05,
|
||||
"source": "https://openrouter.ai/api/v1/images/models/openai/gpt-image-2.5-flare/endpoints",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
],
|
||||
"supports_vision": true
|
||||
},
|
||||
"openrouter/openai/gpt-image-2.5-sunburst": {
|
||||
"input_cost_per_image_token": 8e-06,
|
||||
"input_cost_per_token": 5e-06,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "image_generation",
|
||||
"output_cost_per_image_token": 3e-05,
|
||||
"source": "https://openrouter.ai/api/v1/images/models/openai/gpt-image-2.5-sunburst/endpoints",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
],
|
||||
"supports_vision": true
|
||||
},
|
||||
"openrouter/openai/gpt-5.4-mini:batch": {
|
||||
"cache_read_input_token_cost": 3.75e-08,
|
||||
"input_cost_per_token": 3.75e-07,
|
||||
|
|
|
|||
|
|
@ -555,7 +555,7 @@ if MCP_AVAILABLE:
|
|||
verbose_logger.warning("_prefetch_user_oauth_creds: failed to prefetch for user=%s: %s", user_id, e)
|
||||
return {}
|
||||
|
||||
def _create_tool_response_objects(tools, server: MCPServer):
|
||||
def _create_tool_response_objects(tools: Sequence[MCPTool], server: MCPServer):
|
||||
"""Helper function to create tool response objects.
|
||||
|
||||
Enriches the server's ``mcp_info`` with ``server_id`` and ``alias`` so
|
||||
|
|
@ -569,9 +569,7 @@ if MCP_AVAILABLE:
|
|||
}
|
||||
return [
|
||||
ListMCPToolsRestAPIResponseObject(
|
||||
name=tool.name,
|
||||
description=tool.description,
|
||||
inputSchema=tool.input_schema,
|
||||
**tool.model_dump(by_alias=True, exclude={"mcp_info"}),
|
||||
mcp_info=enriched_mcp_info,
|
||||
)
|
||||
for tool in tools
|
||||
|
|
|
|||
|
|
@ -2,13 +2,19 @@ from __future__ import annotations
|
|||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
import re
|
||||
from collections import deque
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from itertools import chain, islice
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, TypedDict
|
||||
|
||||
from pydantic import ValidationError
|
||||
import anyio
|
||||
from anyio import to_process
|
||||
from anyio.lowlevel import RunVar
|
||||
from pydantic import JsonValue, ValidationError
|
||||
from typing_extensions import ReadOnly, Required, assert_never
|
||||
|
||||
import litellm
|
||||
|
|
@ -44,6 +50,77 @@ VIRTUAL_TOOL_NAMES: Final = frozenset(
|
|||
(MCP_TOOL_SEARCH_TOOL_NAME, MCP_TOOL_CALL_TOOL_NAME, AGENT_SEARCH_TOOL_NAME, SKILL_SEARCH_TOOL_NAME)
|
||||
)
|
||||
|
||||
_SCHEMA_VALIDATION_LIMITER: Final = RunVar[anyio.CapacityLimiter]("mcp_schema_validation_limiter")
|
||||
_MAX_VALIDATION_DEPTH: Final = 64
|
||||
_MAX_VALIDATION_NODES: Final = 10_000
|
||||
_MAX_VALIDATION_CHARACTERS: Final = 1_048_576
|
||||
_VALIDATION_TIMEOUT_SECONDS: Final = 30
|
||||
|
||||
|
||||
def _validation_nodes(value: JsonValue | Mapping[str, JsonValue]) -> Iterator[tuple[int, int]]:
|
||||
pending: Final[deque[Iterator[JsonValue | Mapping[str, JsonValue]]]] = deque((iter((value,)),))
|
||||
while pending:
|
||||
try:
|
||||
node, depth = next(pending[-1]), len(pending) - 1
|
||||
except StopIteration:
|
||||
pending.pop()
|
||||
continue
|
||||
yield depth, len(node) if isinstance(node, str) else 0
|
||||
if depth > _MAX_VALIDATION_DEPTH:
|
||||
continue
|
||||
if isinstance(node, Mapping):
|
||||
pending.append(chain(node.keys(), node.values()))
|
||||
elif isinstance(node, list):
|
||||
pending.append(iter(node))
|
||||
|
||||
|
||||
def _validation_limit_error(schema: Mapping[str, JsonValue], arguments: Mapping[str, JsonValue]) -> str | None:
|
||||
nodes: Final = tuple(
|
||||
islice(chain(_validation_nodes(schema), _validation_nodes(arguments)), _MAX_VALIDATION_NODES + 1)
|
||||
)
|
||||
if len(nodes) > _MAX_VALIDATION_NODES or any(depth > _MAX_VALIDATION_DEPTH for depth, _ in nodes):
|
||||
return "Tool schema or arguments exceed validation size or depth limits"
|
||||
if sum(size for _, size in nodes) > _MAX_VALIDATION_CHARACTERS:
|
||||
return "Tool schema or arguments exceed validation size or depth limits"
|
||||
return None
|
||||
|
||||
|
||||
def _validate_tool_arguments(schema: Mapping[str, JsonValue], arguments: Mapping[str, JsonValue]) -> str | None:
|
||||
from jsonschema import validate
|
||||
from jsonschema.exceptions import SchemaError
|
||||
from jsonschema.exceptions import ValidationError as JsonSchemaValidationError
|
||||
from referencing import Registry
|
||||
from referencing.exceptions import Unresolvable
|
||||
|
||||
try:
|
||||
validate(instance=arguments, schema=dict(schema), registry=Registry())
|
||||
except JsonSchemaValidationError as exc:
|
||||
return f"Invalid arguments: {exc.message}"
|
||||
except (SchemaError, Unresolvable, RecursionError, re.error):
|
||||
return "Unable to validate tool arguments against the supplied schema"
|
||||
return None
|
||||
|
||||
|
||||
async def _tool_argument_validation_error(
|
||||
schema: Mapping[str, JsonValue], arguments: Mapping[str, JsonValue]
|
||||
) -> str | None:
|
||||
limit_error: Final = _validation_limit_error(schema, arguments)
|
||||
if limit_error is not None:
|
||||
return limit_error
|
||||
existing: Final = _SCHEMA_VALIDATION_LIMITER.get(None)
|
||||
limiter: Final = existing if existing is not None else anyio.CapacityLimiter(4)
|
||||
if existing is None:
|
||||
_SCHEMA_VALIDATION_LIMITER.set(limiter)
|
||||
try:
|
||||
with anyio.fail_after(_VALIDATION_TIMEOUT_SECONDS):
|
||||
return await to_process.run_sync(
|
||||
_validate_tool_arguments, schema, arguments, cancellable=True, limiter=limiter
|
||||
)
|
||||
except TimeoutError:
|
||||
return "Tool argument validation exceeded its time limit"
|
||||
except anyio.BrokenWorkerProcess:
|
||||
return "Tool argument validation worker failed"
|
||||
|
||||
|
||||
def coerce_top_k(value: Any, default: int = 5) -> int:
|
||||
try:
|
||||
|
|
@ -502,7 +579,7 @@ async def handle_mcp_tool_search(
|
|||
|
||||
async def handle_mcp_proxy_tool(
|
||||
name: str,
|
||||
arguments: dict[str, object], # mutable-ok: MCP dispatcher passes mutable call arguments
|
||||
arguments: dict[str, JsonValue], # mutable-ok: MCP dispatcher passes mutable JSON call arguments
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
client_ip: str | None = None,
|
||||
mcp_servers: list[str] | None = None, # mutable-ok: preserve MCP scope container for existing resolver
|
||||
|
|
@ -513,8 +590,6 @@ async def handle_mcp_proxy_tool(
|
|||
litellm_logging_obj: LiteLLMLoggingObj | None = None,
|
||||
) -> CallToolResult:
|
||||
from fastapi import HTTPException
|
||||
from jsonschema import ValidationError as JsonSchemaValidationError
|
||||
from jsonschema import validate
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._experimental.mcp_server.operations import (
|
||||
|
|
@ -571,10 +646,9 @@ async def handle_mcp_proxy_tool(
|
|||
tool_arguments: Final = arguments.get("arguments", {})
|
||||
if not isinstance(tool_arguments, dict):
|
||||
return _text_tool_result("arguments must be an object", is_error=True)
|
||||
try:
|
||||
validate(instance=tool_arguments, schema=tool.input_schema)
|
||||
except JsonSchemaValidationError as exc:
|
||||
return _text_tool_result(f"Invalid arguments: {exc.message}", is_error=True)
|
||||
validation_error: Final = await _tool_argument_validation_error(tool.input_schema, tool_arguments)
|
||||
if validation_error is not None:
|
||||
return _text_tool_result(validation_error, is_error=True)
|
||||
|
||||
return await handle_mcp_tool_call(
|
||||
tool_name=_mcp_proxy_identity(tool)["tool_name"],
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import re
|
|||
import secrets
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Final, NamedTuple, Protocol, Union, cast
|
||||
from typing import Final, NamedTuple, Protocol, Union, cast
|
||||
|
||||
import fastapi
|
||||
import orjson
|
||||
|
|
@ -215,7 +215,7 @@ def _get_model_from_request_context(
|
|||
request_data: dict,
|
||||
route: str,
|
||||
request: Request | None,
|
||||
llm_router: Any | None = None,
|
||||
llm_router: litellm.Router | None = None,
|
||||
team_id: str | None = None,
|
||||
) -> str | list[str] | None:
|
||||
return get_model_from_request(
|
||||
|
|
@ -520,8 +520,8 @@ def _get_bearer_token_or_received_api_key(api_key: str) -> str:
|
|||
|
||||
|
||||
def _routing_selector_matches_claim(
|
||||
selector_value: Any | None,
|
||||
claim_value: Any | None,
|
||||
selector_value: object,
|
||||
claim_value: object,
|
||||
*,
|
||||
split_space_delimited: bool = False,
|
||||
) -> bool:
|
||||
|
|
@ -661,7 +661,7 @@ async def user_api_key_auth_websocket_for_model(websocket: WebSocket, model: str
|
|||
# ``websocket.url``, which Starlette reconstructs from the (poisonable)
|
||||
# Host header. Carry the ASGI scope's path / root_path so the lookup
|
||||
# never reaches the fallback.
|
||||
synthetic_scope: Final[dict[str, Any]] = {
|
||||
synthetic_scope: Final[dict[str, object]] = {
|
||||
"type": "http",
|
||||
"method": "GET",
|
||||
"query_string": ws_scope.get("query_string", b""),
|
||||
|
|
@ -3211,7 +3211,7 @@ async def _reserve_budget_after_common_checks(
|
|||
user_api_key_auth_obj: UserAPIKeyAuth,
|
||||
request_data: dict,
|
||||
route: str,
|
||||
llm_router: Any | None,
|
||||
llm_router: litellm.Router | None,
|
||||
team_object: LiteLLM_TeamTableCachedObj | None,
|
||||
user_object: LiteLLM_UserTable | None,
|
||||
prisma_client: PrismaClient | None,
|
||||
|
|
@ -3257,7 +3257,7 @@ def _should_skip_budget_checks(
|
|||
request_data: dict,
|
||||
route: str,
|
||||
request: Request | None,
|
||||
llm_router: Any | None,
|
||||
llm_router: litellm.Router | None,
|
||||
team_id: str | None = None,
|
||||
) -> bool:
|
||||
model: Final = _get_model_from_request_context(
|
||||
|
|
@ -3760,7 +3760,7 @@ async def _enforce_key_and_fallback_model_access(
|
|||
route: str,
|
||||
request: Request | None,
|
||||
llm_model_list: list | None,
|
||||
llm_router: Any | None,
|
||||
llm_router: litellm.Router | None,
|
||||
) -> None:
|
||||
"""
|
||||
Key-level model allowlist and client fallbacks (same as standard auth).
|
||||
|
|
|
|||
|
|
@ -9,11 +9,19 @@ from collections.abc import Callable, Mapping, Sequence
|
|||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypeAlias
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
import click
|
||||
import requests
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.proxy.common_utils.codex_model_catalog import (
|
||||
CODEX_BASE_INSTRUCTIONS_PATH,
|
||||
CodexCatalogRow,
|
||||
CodexStockCatalog,
|
||||
CodexStockModel,
|
||||
codex_models_response_json,
|
||||
)
|
||||
|
||||
from .auth import CliContextObj, context_secret_vault, get_stored_api_key, login
|
||||
from .claude_settings import ClaudeSettingsError, install_statusline_script
|
||||
|
|
@ -68,7 +76,6 @@ _HIDDEN_AGENTS: Final = frozenset({"pi"})
|
|||
CODEX_PROXY_PROVIDER: Final = "litellm"
|
||||
CODEX_HOME_ENV: Final = "CODEX_HOME"
|
||||
CODEX_MODEL_CATALOG_FILENAME: Final = "litellm-models.json"
|
||||
_CODEX_BASE_INSTRUCTIONS_PATH: Final = Path(__file__).with_name("codex_base_instructions.md")
|
||||
_CODEX_PREFLIGHT_TIMEOUT_SECONDS: Final = 10.0
|
||||
|
||||
|
||||
|
|
@ -390,96 +397,8 @@ def opencode_model_sync_env(
|
|||
return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing)})
|
||||
|
||||
|
||||
class _CodexTruncationPolicy(BaseModel):
|
||||
mode: Literal["bytes"] = "bytes"
|
||||
limit: int = 10_000
|
||||
|
||||
|
||||
class _CodexModel(BaseModel):
|
||||
"""One `ModelInfo` entry of a Codex model catalog for a model the installed Codex does not know.
|
||||
|
||||
Every field that some Codex release since `model_catalog_json` appeared
|
||||
(0.105.0) deserializes without a default is spelled out here, so one catalog
|
||||
parses on all of them; the values match the fallback metadata Codex uses for
|
||||
a model slug it does not know, so picking such a proxy model behaves the
|
||||
same as `codex -m` did.
|
||||
"""
|
||||
|
||||
slug: str
|
||||
display_name: str
|
||||
description: None = None
|
||||
supported_reasoning_levels: tuple[()] = ()
|
||||
shell_type: Literal["unified_exec"] = "unified_exec"
|
||||
visibility: Literal["list"] = "list"
|
||||
supported_in_api: Literal[True] = True
|
||||
priority: int
|
||||
availability_nux: None = None
|
||||
upgrade: None = None
|
||||
support_verbosity: Literal[False] = False
|
||||
supports_reasoning_summaries: Literal[False] = False
|
||||
supports_parallel_tool_calls: Literal[False] = False
|
||||
default_verbosity: None = None
|
||||
apply_patch_tool_type: None = None
|
||||
truncation_policy: _CodexTruncationPolicy = _CodexTruncationPolicy()
|
||||
experimental_supported_tools: tuple[()] = ()
|
||||
context_window: int | None
|
||||
base_instructions: str
|
||||
|
||||
|
||||
class _StockCodexUpgrade(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
model: str
|
||||
|
||||
|
||||
class _StockCodexModel(BaseModel):
|
||||
"""One `ModelInfo` entry as the installed Codex prints it from `codex debug models`.
|
||||
|
||||
Only the fields the sync rewrites are named; everything else that release
|
||||
knows about the model (its reasoning levels, prompt, tool support) rides
|
||||
along untouched, whatever the release's schema.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
slug: str
|
||||
priority: int
|
||||
visibility: str
|
||||
supported_in_api: bool = True
|
||||
upgrade: _StockCodexUpgrade | None = None
|
||||
|
||||
|
||||
class _StockCodexCatalog(BaseModel):
|
||||
models: tuple[_StockCodexModel, ...]
|
||||
|
||||
|
||||
class _CodexCatalog(BaseModel):
|
||||
models: tuple[_CodexModel | _StockCodexModel, ...]
|
||||
|
||||
|
||||
def _codex_catalog_entry(
|
||||
priority: int,
|
||||
listed: ListedModel,
|
||||
stock: _StockCodexModel | None,
|
||||
served: frozenset[str],
|
||||
instructions: str,
|
||||
) -> _CodexModel | _StockCodexModel:
|
||||
if stock is None:
|
||||
return _CodexModel(
|
||||
slug=listed.id,
|
||||
display_name=listed.id,
|
||||
priority=priority,
|
||||
context_window=listed.max_input_tokens,
|
||||
base_instructions=instructions,
|
||||
)
|
||||
upgrade: Final = stock.upgrade if stock.upgrade is not None and stock.upgrade.model in served else None
|
||||
return stock.model_copy(
|
||||
update={"priority": priority, "visibility": "list", "supported_in_api": True, "upgrade": upgrade}
|
||||
)
|
||||
|
||||
|
||||
def codex_model_catalog(
|
||||
models: Sequence[ListedModel], stock: Sequence[_StockCodexModel], instructions: str
|
||||
models: Sequence[ListedModel], stock: Sequence[CodexStockModel], instructions: str
|
||||
) -> str | None:
|
||||
"""The `model_catalog_json` body listing the proxy's chat models, or None if there are none.
|
||||
|
||||
|
|
@ -490,19 +409,14 @@ def codex_model_catalog(
|
|||
Codex hides it or keeps it off the API, and keeps Codex's upgrade nudge only
|
||||
when the model it points at is served too. A model Codex does not know gets the fallback
|
||||
entry, with the same base instructions Codex itself uses so the agent never
|
||||
runs without a system prompt.
|
||||
runs without a system prompt. The catalog is the one `/v1/models?client_version=...`
|
||||
serves, built from the installed Codex's own entries instead of the proxy's vendored ones.
|
||||
"""
|
||||
chat_models: Final = _chat_models(models)
|
||||
if not chat_models:
|
||||
return None
|
||||
served: Final = frozenset(m.id for m in chat_models)
|
||||
known: Final = MappingProxyType({m.slug: m for m in stock})
|
||||
catalog: Final = _CodexCatalog(
|
||||
models=tuple(
|
||||
_codex_catalog_entry(index, m, known.get(m.id), served, instructions) for index, m in enumerate(chat_models)
|
||||
)
|
||||
rows: Final = tuple(CodexCatalogRow(id=m.id, mode=m.mode, max_input_tokens=m.max_input_tokens) for m in models)
|
||||
body: Final = codex_models_response_json(
|
||||
rows, stock=MappingProxyType({m.slug: m for m in stock}), instructions=instructions, byte_limit=None
|
||||
)
|
||||
return catalog.model_dump_json()
|
||||
return body.json if body.listed else None
|
||||
|
||||
|
||||
def codex_model_catalog_path(env: Mapping[str, str], *, home: Callable[[], Path] = Path.home) -> Path:
|
||||
|
|
@ -558,12 +472,12 @@ def _codex_debug_models(
|
|||
|
||||
def _stock_codex_models(
|
||||
binary: str, env: Mapping[str, str], *, run: Callable[..., subprocess.CompletedProcess[str]]
|
||||
) -> tuple[_StockCodexModel, ...] | ModelSyncSkipped:
|
||||
) -> tuple[CodexStockModel, ...] | ModelSyncSkipped:
|
||||
printed: Final = _codex_debug_models(binary, (), env, run=run)
|
||||
if isinstance(printed, ModelSyncSkipped):
|
||||
return printed
|
||||
try:
|
||||
return _StockCodexCatalog.model_validate_json(printed).models
|
||||
return CodexStockCatalog.model_validate_json(printed).models
|
||||
except ValidationError as e:
|
||||
name: Final = os.path.basename(binary)
|
||||
return ModelSyncSkipped(f"`{name} debug models` printed no model catalog: {e.errors()[0]['msg']}")
|
||||
|
|
@ -578,7 +492,7 @@ def codex_model_sync_args(
|
|||
get: Callable[..., requests.Response] = requests.get,
|
||||
run: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run,
|
||||
home: Callable[[], Path] = Path.home,
|
||||
instructions_path: Path = _CODEX_BASE_INSTRUCTIONS_PATH,
|
||||
instructions_path: Path = CODEX_BASE_INSTRUCTIONS_PATH,
|
||||
) -> ModelSyncArgs | ModelSyncSkipped:
|
||||
"""`-c model_catalog_json=...` pointing Codex at the proxy's model list, or why it was skipped.
|
||||
|
||||
|
|
|
|||
1613
litellm/proxy/common_utils/codex_bundled_models_0.159.3.json
Normal file
1613
litellm/proxy/common_utils/codex_bundled_models_0.159.3.json
Normal file
File diff suppressed because one or more lines are too long
420
litellm/proxy/common_utils/codex_model_catalog.py
Normal file
420
litellm/proxy/common_utils/codex_model_catalog.py
Normal file
|
|
@ -0,0 +1,420 @@
|
|||
"""Codex-native model catalog for the model listing endpoints.
|
||||
|
||||
Codex CLI discovers a provider's models with ``GET <model_catalog_url>?client_version=<its version>``
|
||||
(the provider's ``model_catalog_url`` pointed at ``/v1/models``, with its ``api_key_model_discovery``
|
||||
feature on) and decodes the body as its own ``ModelsResponse``, ``{"models": [ModelInfo, ...]}``,
|
||||
never the OpenAI list shape. This module builds that body from the rows ``/v1/models`` already
|
||||
lists: a model Codex knows keeps Codex's own entry for it, any other model gets the fallback entry
|
||||
Codex itself uses for an unknown slug, and ``model_info.service_tiers`` becomes the entry's
|
||||
``service_tiers``, which Codex turns into slash commands (``/ultrafast``) that send ``service_tier``
|
||||
upstream.
|
||||
|
||||
The stock entries are Codex 0.159.3's bundled catalog, vendored unchanged from
|
||||
https://github.com/openai/codex/blob/rust-v0.159.3/codex-rs/models-manager/models.json (Apache-2.0)
|
||||
as ``codex_bundled_models_0.159.3.json``, and ``codex_base_instructions.md`` is
|
||||
``codex-rs/models-manager/prompt.md`` at the same tag. Refresh both by downloading those two paths at
|
||||
a newer tag.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import cache
|
||||
from itertools import accumulate, pairwise
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType, NoneType
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, StringConstraints, TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
from litellm.types.proxy.model_listing import ModelInfoResponse
|
||||
|
||||
CODEX_BASE_INSTRUCTIONS_PATH: Final = Path(__file__).with_name("codex_base_instructions.md")
|
||||
CODEX_BUNDLED_MODELS_PATH: Final = Path(__file__).with_name("codex_bundled_models_0.159.3.json")
|
||||
CODEX_CATALOG_BYTE_LIMIT: Final = 1024 * 1024
|
||||
CODEX_CHAT_MODES: Final = frozenset({"chat", "responses"})
|
||||
|
||||
_TierId = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)]
|
||||
_BODY_PREFIX: Final = '{"models":['
|
||||
_BODY_SUFFIX: Final = "]}"
|
||||
|
||||
|
||||
class CodexServiceTier(BaseModel):
|
||||
"""A `ModelServiceTier` as Codex reads it: the slash command is `name` lowercased, and toggling
|
||||
it sends `id` as the request's `service_tier`."""
|
||||
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
id: _TierId
|
||||
name: str
|
||||
description: str
|
||||
|
||||
|
||||
class _ConfiguredTier(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
id: _TierId
|
||||
name: str | None = None
|
||||
description: str | None = None
|
||||
|
||||
|
||||
_CONFIGURED_TIERS: Final = TypeAdapter(tuple[_TierId | _ConfiguredTier, ...])
|
||||
|
||||
|
||||
class CodexTruncationPolicy(BaseModel):
|
||||
mode: Literal["bytes"] = "bytes"
|
||||
limit: int = 10_000
|
||||
|
||||
|
||||
class CodexFallbackModel(BaseModel):
|
||||
"""One `ModelInfo` entry of a Codex model catalog for a model Codex does not know.
|
||||
|
||||
Every field that some Codex release since `model_catalog_json` appeared
|
||||
(0.105.0) deserializes without a default is spelled out here, so one catalog
|
||||
parses on all of them; the values match the fallback metadata Codex uses for
|
||||
a model slug it does not know, so picking such a proxy model behaves the
|
||||
same as `codex -m` did.
|
||||
"""
|
||||
|
||||
slug: str
|
||||
display_name: str
|
||||
description: NoneType = None
|
||||
supported_reasoning_levels: tuple[()] = ()
|
||||
shell_type: Literal["unified_exec"] = "unified_exec"
|
||||
visibility: Literal["list"] = "list"
|
||||
supported_in_api: Literal[True] = True
|
||||
priority: int
|
||||
service_tiers: tuple[CodexServiceTier, ...] = ()
|
||||
availability_nux: NoneType = None
|
||||
upgrade: NoneType = None
|
||||
support_verbosity: Literal[False] = False
|
||||
supports_reasoning_summaries: Literal[False] = False
|
||||
supports_parallel_tool_calls: Literal[False] = False
|
||||
default_verbosity: NoneType = None
|
||||
apply_patch_tool_type: NoneType = None
|
||||
truncation_policy: CodexTruncationPolicy = CodexTruncationPolicy()
|
||||
experimental_supported_tools: tuple[()] = ()
|
||||
context_window: int | None
|
||||
base_instructions: str
|
||||
|
||||
|
||||
class CodexStockUpgrade(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
model: str
|
||||
|
||||
|
||||
class CodexStockModel(BaseModel):
|
||||
"""One `ModelInfo` entry as Codex ships it or prints it from `codex debug models`.
|
||||
|
||||
Only the fields the listing rewrites are named; everything else that release
|
||||
knows about the model (its reasoning levels, prompt, tool support) rides
|
||||
along untouched, whatever the release's schema.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
slug: str
|
||||
display_name: str
|
||||
priority: int
|
||||
visibility: str
|
||||
supported_in_api: bool = True
|
||||
upgrade: CodexStockUpgrade | None = None
|
||||
service_tiers: tuple[CodexServiceTier, ...] = ()
|
||||
default_service_tier: str | None = None
|
||||
|
||||
|
||||
class CodexStockCatalog(BaseModel):
|
||||
models: tuple[CodexStockModel, ...]
|
||||
|
||||
|
||||
CodexCatalogEntry = CodexFallbackModel | CodexStockModel
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CodexCatalogRow:
|
||||
"""What the catalog needs to know about one listed model: its public id, the listing's mode and
|
||||
input limit, the upstream model behind it, and the `model_info` values the operator set;
|
||||
`service_tiers` holds one raw `model_info.service_tiers` value per deployment behind the id, None
|
||||
where a deployment sets none."""
|
||||
|
||||
id: str
|
||||
mode: str | None = None
|
||||
max_input_tokens: int | None = None
|
||||
upstream_model: str | None = None
|
||||
display_name: str | None = None
|
||||
service_tiers: tuple[object, ...] = ()
|
||||
|
||||
|
||||
@cache
|
||||
def bundled_codex_models() -> Mapping[str, CodexStockModel]:
|
||||
"""Codex's bundled catalog by slug, read once from the vendored file."""
|
||||
catalog: Final = CodexStockCatalog.model_validate_json(CODEX_BUNDLED_MODELS_PATH.read_text(encoding="utf-8"))
|
||||
return MappingProxyType({model.slug: model for model in catalog.models})
|
||||
|
||||
|
||||
@cache
|
||||
def codex_base_instructions() -> str:
|
||||
return CODEX_BASE_INSTRUCTIONS_PATH.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _tier(item: str | _ConfiguredTier, known: Mapping[str, CodexServiceTier]) -> CodexServiceTier:
|
||||
if isinstance(item, str) and item in known:
|
||||
return known[item]
|
||||
configured: Final = _ConfiguredTier(id=item) if isinstance(item, str) else item
|
||||
return CodexServiceTier(
|
||||
id=configured.id,
|
||||
name=configured.name if configured.name is not None else configured.id.capitalize(),
|
||||
description=(
|
||||
configured.description
|
||||
if configured.description is not None
|
||||
else f"Sends service_tier={configured.id} upstream"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _first_by_id(tiers: Sequence[CodexServiceTier]) -> tuple[CodexServiceTier, ...]:
|
||||
return tuple(tier for index, tier in enumerate(tiers) if tier.id not in {seen.id for seen in tiers[:index]})
|
||||
|
||||
|
||||
_NO_KNOWN_TIERS: Final[Mapping[str, CodexServiceTier]] = MappingProxyType({})
|
||||
|
||||
|
||||
def _deployment_tiers(
|
||||
raw: object, model_id: str, known: Mapping[str, CodexServiceTier]
|
||||
) -> tuple[CodexServiceTier, ...]:
|
||||
try:
|
||||
items: Final = _CONFIGURED_TIERS.validate_python(raw)
|
||||
except ValidationError as e:
|
||||
verbose_proxy_logger.warning(
|
||||
"model_info.service_tiers for %s offers no tier, expected a list of tier ids or {id, name, description} objects: %s",
|
||||
model_id,
|
||||
e.errors()[0]["msg"],
|
||||
)
|
||||
return ()
|
||||
return _first_by_id(tuple(_tier(item, known) for item in items))
|
||||
|
||||
|
||||
def configured_service_tiers(
|
||||
per_deployment: Sequence[object], model_id: str, known: Mapping[str, CodexServiceTier] = _NO_KNOWN_TIERS
|
||||
) -> tuple[CodexServiceTier, ...] | None:
|
||||
"""The tiers `model_info.service_tiers` configures for `model_id`, given each of its deployments'
|
||||
raw value (None where unset), or None when no deployment sets one.
|
||||
|
||||
Accepts tier id strings and `{id, name, description}` objects. A string naming one of the
|
||||
model's `known` tiers (the ones Codex ships for it) keeps that tier's name and description, so
|
||||
`"priority"` stays Codex's `/fast`; any other string `x` is the command `/x` described as
|
||||
"Sends service_tier=x upstream". A duplicate id keeps its first entry. A request to `model_id`
|
||||
can route to any of its deployments, so a tier is offered only when every deployment lists it,
|
||||
in the first deployment's order; a deployment that sets none leaves nothing to offer. Each of
|
||||
those cases is logged, and an invalid value on any deployment is logged and offers no tier for
|
||||
the whole model, stock tiers included, so one typo never fails the listing and never offers a
|
||||
tier nobody declared.
|
||||
"""
|
||||
declared: Final = tuple(raw for raw in per_deployment if raw is not None)
|
||||
if not declared:
|
||||
return None
|
||||
parsed: Final = tuple(_deployment_tiers(raw, model_id, known) for raw in declared)
|
||||
if len(parsed) < len(per_deployment):
|
||||
verbose_proxy_logger.warning(
|
||||
"model_info.service_tiers for %s is set on %d of its %d deployments, so no tier is offered: "
|
||||
"a tier is offered only when every deployment lists it",
|
||||
model_id,
|
||||
len(parsed),
|
||||
len(per_deployment),
|
||||
)
|
||||
return ()
|
||||
tier_ids: Final = tuple(frozenset(tier.id for tier in tiers) for tiers in parsed)
|
||||
shared: Final = tier_ids[0].intersection(*tier_ids[1:])
|
||||
offered: Final = tuple(tier for tier in parsed[0] if tier.id in shared)
|
||||
if len(offered) < len(parsed[0]):
|
||||
verbose_proxy_logger.warning(
|
||||
"model_info.service_tiers for %s differs between its deployments, offering only the tiers every one lists: %s",
|
||||
model_id,
|
||||
", ".join(tier.id for tier in offered) or "none",
|
||||
)
|
||||
return offered
|
||||
|
||||
|
||||
def _codex_can_drive(row: CodexCatalogRow) -> bool:
|
||||
return "*" not in row.id and (row.mode is None or row.mode in CODEX_CHAT_MODES)
|
||||
|
||||
|
||||
def _stock_for(row: CodexCatalogRow, stock: Mapping[str, CodexStockModel]) -> CodexStockModel | None:
|
||||
listed: Final = stock.get(row.id)
|
||||
if listed is not None or row.upstream_model is None:
|
||||
return listed
|
||||
upstream: Final = stock.get(row.upstream_model)
|
||||
return upstream if upstream is not None else stock.get(row.upstream_model.partition("/")[2])
|
||||
|
||||
|
||||
def _entry(
|
||||
index: int,
|
||||
row: CodexCatalogRow,
|
||||
stock: Mapping[str, CodexStockModel],
|
||||
served: frozenset[str],
|
||||
instructions: str,
|
||||
) -> CodexCatalogEntry:
|
||||
stock_model: Final = _stock_for(row, stock)
|
||||
stock_tiers: Final = MappingProxyType(
|
||||
{tier.id: tier for tier in (stock_model.service_tiers if stock_model is not None else ())}
|
||||
)
|
||||
configured: Final = configured_service_tiers(row.service_tiers, row.id, stock_tiers)
|
||||
if stock_model is None:
|
||||
return CodexFallbackModel(
|
||||
slug=row.id,
|
||||
display_name=row.display_name if row.display_name is not None else row.id,
|
||||
priority=index,
|
||||
service_tiers=configured if configured is not None else (),
|
||||
context_window=row.max_input_tokens,
|
||||
base_instructions=instructions,
|
||||
)
|
||||
upgrade: Final = (
|
||||
stock_model.upgrade if stock_model.upgrade is not None and stock_model.upgrade.model in served else None
|
||||
)
|
||||
stock_name: Final = stock_model.display_name if row.id == stock_model.slug else row.id
|
||||
listing: Final = {
|
||||
"slug": row.id,
|
||||
"display_name": row.display_name if row.display_name is not None else stock_name,
|
||||
"priority": index,
|
||||
"visibility": "list",
|
||||
"supported_in_api": True,
|
||||
"upgrade": upgrade,
|
||||
}
|
||||
if configured is None:
|
||||
return stock_model.model_copy(update=listing)
|
||||
configured_ids: Final = frozenset(tier.id for tier in configured)
|
||||
return stock_model.model_copy(
|
||||
update={
|
||||
**listing,
|
||||
"service_tiers": configured,
|
||||
"default_service_tier": (
|
||||
stock_model.default_service_tier if stock_model.default_service_tier in configured_ids else None
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _serialized(entry: CodexCatalogEntry) -> str:
|
||||
"""A stock entry keeps exactly the fields its source printed (an older Codex may know no
|
||||
`service_tiers`), a fallback entry spells out every field Codex requires."""
|
||||
return entry.model_dump_json(exclude_unset=isinstance(entry, CodexStockModel))
|
||||
|
||||
|
||||
def _bytes_left_after(left: int, size: int) -> int:
|
||||
return left - size if size <= left else left
|
||||
|
||||
|
||||
def _fits(serialized: Sequence[str], byte_limit: int | None) -> tuple[bool, ...]:
|
||||
"""Whether each entry, taken in order, fits in what is left of a body of at most `byte_limit`
|
||||
bytes: an entry larger than the bytes left is passed over and a smaller one after it is still
|
||||
taken, each entry counted with its separating comma and the last one's comma given back."""
|
||||
if byte_limit is None:
|
||||
return (True,) * len(serialized)
|
||||
budget: Final = byte_limit - len(_BODY_PREFIX) - len(_BODY_SUFFIX) + 1
|
||||
left: Final = tuple(
|
||||
accumulate((len(entry.encode()) + 1 for entry in serialized), _bytes_left_after, initial=budget)
|
||||
)
|
||||
return tuple(after < before for before, after in pairwise(left))
|
||||
|
||||
|
||||
def _kept_positions(
|
||||
entries: Sequence[CodexCatalogEntry], serialized: Sequence[str], byte_limit: int | None
|
||||
) -> frozenset[int]:
|
||||
"""The listing positions that survive the byte limit: an entry offering a service tier is taken
|
||||
ahead of one offering none, each group in listing order, since a model without a tier is driven
|
||||
by name just as well off Codex's bundled catalog; an entry too large for the bytes left never
|
||||
costs the smaller ones after it their place."""
|
||||
survival: Final = sorted(range(len(entries)), key=lambda position: (not entries[position].service_tiers, position))
|
||||
fits: Final = _fits(tuple(serialized[position] for position in survival), byte_limit)
|
||||
return frozenset(position for position, fit in zip(survival, fits, strict=True) if fit)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CodexCatalogBody:
|
||||
"""A `ModelsResponse` body, the ids it lists, and the chat-capable ids the byte limit left out."""
|
||||
|
||||
json: str
|
||||
listed: tuple[str, ...]
|
||||
left_out: tuple[str, ...]
|
||||
|
||||
|
||||
def codex_models_response_json(
|
||||
rows: Sequence[CodexCatalogRow],
|
||||
*,
|
||||
stock: Mapping[str, CodexStockModel] | None = None,
|
||||
instructions: str | None = None,
|
||||
byte_limit: int | None = CODEX_CATALOG_BYTE_LIMIT,
|
||||
) -> CodexCatalogBody:
|
||||
"""Codex's `ModelsResponse` body for `rows`, in listing order, cut to Codex's byte limit.
|
||||
|
||||
Only chat-capable rows are listed (Codex cannot drive an embedding or image model, and a
|
||||
wildcard id is no model), their order is their `priority`. Codex drops a catalog over its
|
||||
byte limit silently and keeps its bundled one, so the body holds only the entries that fit,
|
||||
the ones offering a service tier taken first and the rest in listing order, an entry too large
|
||||
for the bytes left passed over, every kept entry at its listing position; `byte_limit=None`
|
||||
keeps every entry.
|
||||
"""
|
||||
known: Final = bundled_codex_models() if stock is None else stock
|
||||
prompt: Final = codex_base_instructions() if instructions is None else instructions
|
||||
drivable: Final = tuple(row for row in rows if _codex_can_drive(row))
|
||||
served: Final = frozenset(row.id for row in drivable)
|
||||
entries: Final = tuple(_entry(index, row, known, served, prompt) for index, row in enumerate(drivable))
|
||||
serialized: Final = tuple(_serialized(entry) for entry in entries)
|
||||
kept: Final = _kept_positions(entries, serialized, byte_limit)
|
||||
return CodexCatalogBody(
|
||||
json=f"{_BODY_PREFIX}{','.join(serialized[position] for position in sorted(kept))}{_BODY_SUFFIX}",
|
||||
listed=tuple(row.id for position, row in enumerate(drivable) if position in kept),
|
||||
left_out=tuple(row.id for position, row in enumerate(drivable) if position not in kept),
|
||||
)
|
||||
|
||||
|
||||
def _catalog_row(
|
||||
row: ModelInfoResponse, lookup_id: str, llm_router: Router | None, team_id: str | None
|
||||
) -> CodexCatalogRow:
|
||||
return CodexCatalogRow(
|
||||
id=row["id"],
|
||||
mode=row.get("mode"),
|
||||
max_input_tokens=row.get("max_input_tokens"),
|
||||
upstream_model=llm_router.get_routable_upstream_model(lookup_id, team_id) if llm_router is not None else None,
|
||||
display_name=llm_router.get_configured_display_name(lookup_id) if llm_router is not None else None,
|
||||
service_tiers=llm_router.get_configured_service_tiers(lookup_id, team_id) if llm_router is not None else (),
|
||||
)
|
||||
|
||||
|
||||
def codex_catalog_rows(
|
||||
rows: Sequence[ModelInfoResponse],
|
||||
entries: Sequence[tuple[str, str]],
|
||||
llm_router: Router | None,
|
||||
team_id: str | None = None,
|
||||
) -> tuple[CodexCatalogRow, ...]:
|
||||
"""`rows` joined with the router's configured metadata, looked up by each entry's internal id so
|
||||
team-scoped rows resolve the way the Anthropic listing's display names do, a `model_group_alias`
|
||||
reading its target's deployments under its own name; `team_id` is the requesting key's team, so the
|
||||
upstream model that picks Codex's stock entry and the tiers are read only off the deployments its
|
||||
requests can route to."""
|
||||
lookup_ids: Final = MappingProxyType(dict(entries))
|
||||
return tuple(_catalog_row(row, lookup_ids.get(row["id"], row["id"]), llm_router, team_id) for row in rows)
|
||||
|
||||
|
||||
def codex_model_list_body(
|
||||
rows: Sequence[ModelInfoResponse],
|
||||
entries: Sequence[tuple[str, str]],
|
||||
llm_router: Router | None,
|
||||
team_id: str | None = None,
|
||||
) -> str:
|
||||
"""The `/v1/models?client_version=...` body, logging the models Codex's byte limit left out."""
|
||||
body: Final = codex_models_response_json(codex_catalog_rows(rows, entries, llm_router, team_id))
|
||||
if body.left_out:
|
||||
verbose_proxy_logger.warning(
|
||||
"Codex model catalog cut at %d bytes, left out: %s. Models offering a service tier are kept first, "
|
||||
"then model_list order; list the models Codex users need first in model_list",
|
||||
CODEX_CATALOG_BYTE_LIMIT,
|
||||
", ".join(body.left_out),
|
||||
)
|
||||
return body.json
|
||||
|
|
@ -254,6 +254,40 @@ def translate_libpq_ssl_params(url: str, resolve_root_cert: RootCertResolver = p
|
|||
return urllib.parse.urlunsplit(parsed._replace(query=query))
|
||||
|
||||
|
||||
def postgres_connection_budget_message(writer_limit: str, reader_limit: str | None, num_workers: str) -> str:
|
||||
"""The startup line that states this pod's worst-case Postgres connection demand.
|
||||
|
||||
Prisma's ``connection_limit`` is per query engine, and every uvicorn worker
|
||||
owns one engine per configured database (writer, plus the reader when
|
||||
``DATABASE_URL_READ_REPLICA`` is set). The limits are read off the final URLs,
|
||||
so a reader that pins its own ``connection_limit`` or an operator override in
|
||||
``database_extra_connection_params`` is counted at its real value. The
|
||||
server-side cap is shared by every pod, so the number an operator has to keep
|
||||
under ``max_connections`` minus ``superuser_reserved_connections`` is
|
||||
pods x workers x the per-worker sum, not one engine's limit.
|
||||
"""
|
||||
try:
|
||||
workers: Final = max(1, int(num_workers))
|
||||
writer: Final = int(writer_limit)
|
||||
reader: Final = int(reader_limit) if reader_limit is not None else 0
|
||||
except ValueError:
|
||||
return (
|
||||
"LiteLLM Proxy: Postgres connection budget per pod = workers x (writer connection_limit + reader "
|
||||
f"connection_limit) (workers={num_workers!r}, writer={writer_limit!r}, reader={reader_limit!r}); "
|
||||
"keep pods x that figure under max_connections minus superuser_reserved_connections"
|
||||
)
|
||||
engines: Final = (
|
||||
f"(writer connection_limit {writer} + reader connection_limit {reader})"
|
||||
if reader_limit is not None
|
||||
else f"writer connection_limit {writer}"
|
||||
)
|
||||
per_pod: Final = workers * (writer + reader)
|
||||
return (
|
||||
f"LiteLLM Proxy: Postgres connection budget per pod = {workers} worker(s) x {engines} = up to {per_pod} "
|
||||
"connections; keep pods x that figure under max_connections minus superuser_reserved_connections"
|
||||
)
|
||||
|
||||
|
||||
def reader_shareable_params(params: Mapping[str, str | int | float]) -> Mapping[str, str | int | float]:
|
||||
"""Return the subset of ``params`` the read replica is allowed to inherit."""
|
||||
return MappingProxyType({key: value for key, value in params.items() if key in CONNECTION_PARAM_KEYS})
|
||||
|
|
@ -266,16 +300,26 @@ def connection_params_from_url(url: str) -> Mapping[str, str | int | float]:
|
|||
)
|
||||
|
||||
|
||||
# A re-minted token URL replaces a URL to the same database, so unlike the
|
||||
# reader allowlist it may also carry ``options``: that is where the server-side
|
||||
# timeouts (statement, lock, idle-in-transaction) live, and a refresh that
|
||||
# dropped them would leave the replacement engine's sessions unbounded.
|
||||
TOKEN_REFRESH_PARAM_KEYS: Final[frozenset[str]] = CONNECTION_PARAM_KEYS | PRISMA_TLS_PARAM_KEYS | frozenset({"options"})
|
||||
|
||||
|
||||
def token_refresh_params_from_url(url: str) -> Mapping[str, str | int | float]:
|
||||
"""Return the params a re-minted token URL carries over from the URL it replaces.
|
||||
|
||||
The pool and timeout params plus Prisma's TLS params (already translated from
|
||||
libpq spelling), so a refreshed URL keeps verifying the server the way the
|
||||
first one did.
|
||||
The pool and timeout params, Prisma's TLS params (already translated from
|
||||
libpq spelling) and the ``options`` string, so a refreshed URL keeps verifying
|
||||
the server and bounding its sessions the way the first one did.
|
||||
"""
|
||||
kept: Final = CONNECTION_PARAM_KEYS | PRISMA_TLS_PARAM_KEYS
|
||||
return MappingProxyType(
|
||||
{key: value for key, value in urllib.parse.parse_qsl(urllib.parse.urlsplit(url).query) if key in kept}
|
||||
{
|
||||
key: value
|
||||
for key, value in urllib.parse.parse_qsl(urllib.parse.urlsplit(url).query)
|
||||
if key in TOKEN_REFRESH_PARAM_KEYS
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,13 @@ _TRANSIENT_DB_UNAVAILABLE_MESSAGE: Final = (
|
|||
|
||||
_DATABASE_ERROR_META: Final = TypeAdapter(dict[str, object])
|
||||
_BATCH_POSTGRES_ERROR_CODE: Final = re.compile(r'PostgresError \{ code: "([0-9A-Z]{5})"')
|
||||
_CONNECTION_CAPACITY_PHRASES: Final = (
|
||||
"too many clients already",
|
||||
"too many connections for role",
|
||||
"too many connections for database",
|
||||
"remaining connection slots are reserved",
|
||||
)
|
||||
_PRISMA_POOL_TIMEOUT_CODE: Final = "P2024"
|
||||
|
||||
|
||||
def _exception_chain(e: BaseException) -> Iterator[BaseException]:
|
||||
|
|
@ -109,6 +116,8 @@ class PrismaDBExceptionHandler:
|
|||
return True
|
||||
if isinstance(e, _exception_types(prisma.engine.errors.EngineConnectionError)):
|
||||
return True
|
||||
if PrismaDBExceptionHandler.is_database_capacity_error(e):
|
||||
return True
|
||||
return isinstance(e, ProxyException) and e.type == ProxyErrorTypes.no_db_connection
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -192,6 +201,8 @@ class PrismaDBExceptionHandler:
|
|||
),
|
||||
):
|
||||
return True
|
||||
if PrismaDBExceptionHandler.is_database_capacity_error(e):
|
||||
return False
|
||||
if isinstance(e, _exception_types(prisma.errors.PrismaError)):
|
||||
error_message: Final = str(e).lower()
|
||||
connection_keywords: Final = (
|
||||
|
|
@ -236,6 +247,25 @@ class PrismaDBExceptionHandler:
|
|||
or "write conflict or a deadlock" in error_message
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def is_database_capacity_error(e: Exception) -> bool:
|
||||
"""True iff Postgres refused the pool a new connection (SQLSTATE 53300:
|
||||
``too many clients already``, a reserved slot, or a per-role limit) or
|
||||
prisma's own pool timed out handing one over (P2024). The server is up
|
||||
but full, so the failure is neither a transport error (which would tear
|
||||
the engine down and open yet more connections against it) nor a
|
||||
rejection of the rows being written."""
|
||||
import prisma
|
||||
|
||||
if not isinstance(e, _exception_types(prisma.errors.PrismaError)):
|
||||
return False
|
||||
if isinstance(e, prisma.errors.DataError) and e.code == _PRISMA_POOL_TIMEOUT_CODE:
|
||||
return True
|
||||
if PrismaDBExceptionHandler.postgres_sqlstate(e) == "53300":
|
||||
return True
|
||||
error_message: Final = str(e).lower()
|
||||
return any(phrase in error_message for phrase in _CONNECTION_CAPACITY_PHRASES)
|
||||
|
||||
@staticmethod
|
||||
def postgres_sqlstate(e: Exception) -> str | None:
|
||||
"""The SQLSTATE Postgres attached to a failed statement, as prisma surfaces it, or None."""
|
||||
|
|
@ -325,6 +355,8 @@ class PrismaDBExceptionHandler:
|
|||
return True
|
||||
if PrismaDBExceptionHandler.is_database_transport_error(e):
|
||||
return True
|
||||
if PrismaDBExceptionHandler.is_database_capacity_error(e):
|
||||
return True
|
||||
if PrismaDBExceptionHandler.is_prisma_engine_internal_error(e):
|
||||
return True
|
||||
if "cached plan must not change result type" in str(e).lower():
|
||||
|
|
|
|||
|
|
@ -37,18 +37,24 @@ from litellm.proxy.lens.models import (
|
|||
RunRequest,
|
||||
Sample,
|
||||
Scope,
|
||||
Step,
|
||||
WatchAllResult,
|
||||
WatchSkipped,
|
||||
Worker,
|
||||
WorkerCreated,
|
||||
)
|
||||
from litellm.proxy.lens.repository import LensRepository, WriterDatabase
|
||||
from litellm.proxy.lens.sources import ActivityAvailability, SourceReader, Storage, parse_execution
|
||||
from litellm.proxy.lens.state import (
|
||||
add_step,
|
||||
can_access,
|
||||
claim_job,
|
||||
current_job,
|
||||
merge_finding,
|
||||
next_scan_start,
|
||||
queue_job,
|
||||
replace_job,
|
||||
scheduled_window,
|
||||
snapshot_finding,
|
||||
)
|
||||
from litellm.proxy.tracing_runtime import provide_storage
|
||||
|
|
@ -227,6 +233,40 @@ async def list_agents(auth: Auth, storage: StorageDep) -> tuple[str, ...]:
|
|||
return await source_reader(storage).agents(scope) if storage is not None else ()
|
||||
|
||||
|
||||
def watching(lens: Lens) -> Lens:
|
||||
if lens.settings.enabled:
|
||||
return lens
|
||||
return lens.model_copy(
|
||||
update=MappingProxyType(
|
||||
{
|
||||
"settings": lens.settings.model_copy(update=MappingProxyType({"enabled": True})),
|
||||
"revision": lens.revision + 1,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def watchable(lens: Lens, auth: UserAPIKeyAuth) -> WatchSkipped | None:
|
||||
try:
|
||||
await validate_model(lens.settings.model_copy(update=MappingProxyType({"enabled": True})), auth)
|
||||
except HTTPException as exc:
|
||||
return WatchSkipped(id=lens.id, name=lens.settings.name, reason=str(exc.detail))
|
||||
return None
|
||||
|
||||
|
||||
@router.post("/watch-all", response_model=WatchAllResult)
|
||||
async def watch_all(auth: Auth) -> WatchAllResult:
|
||||
scope: Final = user_scope(auth, write=True)
|
||||
paused: Final = tuple(
|
||||
e for e in await repository().lenses() if can_access(scope, e.scope) and not e.settings.enabled
|
||||
)
|
||||
checks: Final = tuple([(lens, await watchable(lens, auth)) for lens in paused])
|
||||
skipped: Final = tuple(skip for _, skip in checks if skip is not None)
|
||||
ready: Final = tuple(lens for lens, skip in checks if skip is None)
|
||||
updated: Final = tuple([await repository().update(lens.id, watching) for lens in ready])
|
||||
return WatchAllResult(watching=tuple(u.id for u in updated if u is not None), skipped=skipped)
|
||||
|
||||
|
||||
@router.put("/{lens_id}", response_model=Lens)
|
||||
async def update_lens(lens_id: str, settings: LensSettings, auth: Auth) -> Lens:
|
||||
lens: Final = await get_lens(lens_id, user_scope(auth, write=True))
|
||||
|
|
@ -248,6 +288,20 @@ async def update_lens(lens_id: str, settings: LensSettings, auth: Auth) -> Lens:
|
|||
)
|
||||
|
||||
|
||||
def run_window(lens: Lens, body: RunRequest, now: datetime) -> tuple[datetime, datetime] | None:
|
||||
if body.start is not None and body.end is not None:
|
||||
return body.start, body.end
|
||||
if body.lookback_hours is None and body.settings is None:
|
||||
return scheduled_window(lens, now)
|
||||
return None
|
||||
|
||||
|
||||
def run_settings(lens: Lens, body: RunRequest) -> LensSettings | None:
|
||||
if body.agent_name is None:
|
||||
return body.settings
|
||||
return (body.settings or lens.settings).model_copy(update=MappingProxyType({"agent_name": body.agent_name}))
|
||||
|
||||
|
||||
@router.post("/{lens_id}/runs", response_model=Lens)
|
||||
async def run_lens(lens_id: str, body: RunRequest, auth: Auth) -> Lens:
|
||||
lens: Final = await get_lens(lens_id, user_scope(auth, write=True))
|
||||
|
|
@ -257,7 +311,18 @@ async def run_lens(lens_id: str, body: RunRequest, auth: Auth) -> Lens:
|
|||
now: Final = datetime.now(timezone.utc)
|
||||
job_id: Final = str(uuid4())
|
||||
return required(
|
||||
await repository().update(lens_id, lambda e: queue_job(e, now, job_id, body.lookback_hours, body.settings))
|
||||
await repository().update(
|
||||
lens_id,
|
||||
lambda e: queue_job(
|
||||
e,
|
||||
now,
|
||||
job_id,
|
||||
body.lookback_hours,
|
||||
run_settings(e, body),
|
||||
run_window(e, body, now),
|
||||
"manual",
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -425,13 +490,14 @@ async def progress(lens_id: str, job_id: str, body: Progress, worker: WorkerAuth
|
|||
job: Final = current_job(e)
|
||||
if job is None or job.id != job_id or job.worker_id != worker.id:
|
||||
return e
|
||||
renewed: Final = job.model_copy(
|
||||
update=MappingProxyType(
|
||||
{"stage": body.stage, "coverage": body.coverage, "lease_until": now + timedelta(minutes=5)}
|
||||
)
|
||||
)
|
||||
return replace_job(
|
||||
e,
|
||||
job.model_copy(
|
||||
update=MappingProxyType(
|
||||
{"stage": body.stage, "coverage": body.coverage, "lease_until": now + timedelta(minutes=5)}
|
||||
)
|
||||
),
|
||||
renewed if body.stage == job.stage else add_step(renewed, Step(at=now, kind="stage", label=body.stage)),
|
||||
)
|
||||
|
||||
required(await repository().update(lens_id, renew))
|
||||
|
|
@ -575,7 +641,7 @@ async def result(lens_id: str, job_id: str, body: Result, worker: WorkerAuth, st
|
|||
update=MappingProxyType(
|
||||
{
|
||||
"findings": (*merged, *(f for f in e.findings if f.id not in merged_ids)),
|
||||
"last_scan_at": e.last_scan_at if body.error else max(e.last_scan_at or job.end, job.end),
|
||||
"last_scan_at": next_scan_start(e, job, failed=bool(body.error)),
|
||||
"next_run_at": now + timedelta(minutes=e.settings.interval_minutes),
|
||||
}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -13,9 +13,9 @@ from litellm.integrations.clickhouse.context import lens_analysis
|
|||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import inherit_message_logging_privacy
|
||||
from litellm.litellm_core_utils.token_counter import get_modified_max_tokens
|
||||
from litellm.proxy.lens.billing import complete, validate_key
|
||||
from litellm.proxy.lens.models import Job, Lens, ModelRequest, ModelResult, Worker
|
||||
from litellm.proxy.lens.models import Job, Lens, ModelRequest, ModelResult, Step, Worker
|
||||
from litellm.proxy.lens.repository import LensRepository
|
||||
from litellm.proxy.lens.state import current_job, renew_budget, replace_job
|
||||
from litellm.proxy.lens.state import add_step, current_job, renew_budget, replace_job
|
||||
from litellm.types.utils import CostPerToken, ModelResponse
|
||||
|
||||
|
||||
|
|
@ -192,20 +192,17 @@ async def analyze(
|
|||
current, active.model_copy(update=MappingProxyType({"cost": active.cost + estimate}))
|
||||
).model_copy(update=MappingProxyType({"spent": current.spent + estimate}))
|
||||
|
||||
def settle(e: Lens, cost: float) -> Lens:
|
||||
def settle(e: Lens, cost: float, step: Step | None) -> Lens:
|
||||
charged: Final = next((j for j in e.jobs if j.id == job.id), None)
|
||||
adjusted: Final = (
|
||||
e.model_copy(update=MappingProxyType({"spent": max(0, e.spent - estimate + cost)}))
|
||||
if e.budget_month == now.strftime("%Y-%m")
|
||||
else e
|
||||
)
|
||||
return (
|
||||
replace_job(
|
||||
adjusted, charged.model_copy(update=MappingProxyType({"cost": max(0, charged.cost - estimate + cost)}))
|
||||
)
|
||||
if charged
|
||||
else adjusted
|
||||
)
|
||||
if charged is None:
|
||||
return adjusted
|
||||
refunded: Final = charged.model_copy(update=MappingProxyType({"cost": max(0, charged.cost - estimate + cost)}))
|
||||
return replace_job(adjusted, add_step(refunded, step) if step is not None else refunded)
|
||||
|
||||
@asynccontextmanager
|
||||
async def reserve_budget() -> AsyncIterator[None]:
|
||||
|
|
@ -214,7 +211,7 @@ async def analyze(
|
|||
try:
|
||||
yield
|
||||
except BaseException:
|
||||
await repo.update(lens.id, lambda e: settle(e, 0))
|
||||
await repo.update(lens.id, lambda e: settle(e, 0, None))
|
||||
raise
|
||||
|
||||
data: Final[dict[str, object]] = { # mutable-ok: proxy processing enriches request data
|
||||
|
|
@ -241,7 +238,8 @@ async def analyze(
|
|||
response, billed_cost = await complete(worker.analysis_key_id, data, reserve_budget, request)
|
||||
cost: Final = billed_cost if billed_cost is not None else completion_charge(deployments, response, estimate)
|
||||
|
||||
await repo.update(lens.id, lambda e: settle(e, cost))
|
||||
step: Final = model_step(response, body, job.settings.model, cost)
|
||||
await repo.update(lens.id, lambda e: settle(e, cost, step))
|
||||
parsed: Final = Completion.model_validate_json(response.model_dump_json())
|
||||
choice: Final = parsed.choices[0]
|
||||
return ModelResult(
|
||||
|
|
@ -253,6 +251,37 @@ async def analyze(
|
|||
)
|
||||
|
||||
|
||||
class Usage(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
prompt_tokens: int | None = None
|
||||
completion_tokens: int | None = None
|
||||
|
||||
|
||||
class UsageEnvelope(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
model: str | None = None
|
||||
usage: Usage | None = None
|
||||
|
||||
|
||||
_PURPOSE_LABELS: Final = MappingProxyType(
|
||||
{"extract": "Reviewed a run", "cluster": "Compared observations", "investigate": "Checked a pattern"}
|
||||
)
|
||||
|
||||
|
||||
def model_step(response: ModelResponse, body: ModelRequest, requested: str, cost: float) -> Step:
|
||||
envelope: Final = UsageEnvelope.model_validate_json(response.model_dump_json())
|
||||
return Step(
|
||||
at=datetime.now(timezone.utc),
|
||||
kind="model",
|
||||
label=_PURPOSE_LABELS[body.purpose],
|
||||
model=envelope.model or requested,
|
||||
purpose=body.purpose,
|
||||
prompt_tokens=(envelope.usage.prompt_tokens if envelope.usage else None) or 0,
|
||||
completion_tokens=(envelope.usage.completion_tokens if envelope.usage else None) or 0,
|
||||
cost=cost,
|
||||
)
|
||||
|
||||
|
||||
def completion_charge(deployments: tuple[Deployment, ...], response: ModelResponse, estimate: float) -> float:
|
||||
custom: Final = deployments[0].litellm_params if len(deployments) == 1 else None
|
||||
if custom and custom.input_cost_per_token is not None and custom.output_cost_per_token is not None:
|
||||
|
|
|
|||
|
|
@ -190,6 +190,20 @@ class RunAssessment(Record):
|
|||
cannot_assess: bool = False
|
||||
|
||||
|
||||
MAX_STEPS = 200
|
||||
|
||||
|
||||
class Step(Record):
|
||||
at: datetime
|
||||
kind: Literal["stage", "model", "error"]
|
||||
label: str = Field(max_length=200)
|
||||
model: str = Field(default="", max_length=200)
|
||||
purpose: str = Field(default="", max_length=40)
|
||||
prompt_tokens: int = 0
|
||||
completion_tokens: int = 0
|
||||
cost: float = 0
|
||||
|
||||
|
||||
class Job(Record):
|
||||
id: str
|
||||
status: Literal["queued", "running", "completed", "failed", "cancelled"] = "queued"
|
||||
|
|
@ -209,6 +223,8 @@ class Job(Record):
|
|||
cost: float = 0
|
||||
findings: tuple[Finding, ...] | None = None
|
||||
assessments: tuple[RunAssessment, ...] = ()
|
||||
steps: tuple[Step, ...] = ()
|
||||
trigger: Literal["schedule", "manual"] = "schedule"
|
||||
|
||||
|
||||
class Lens(Record):
|
||||
|
|
@ -249,6 +265,28 @@ class LensList(Record):
|
|||
class RunRequest(Record):
|
||||
settings: LensSettings | None = None
|
||||
lookback_hours: LookbackHours | None = None
|
||||
start: datetime | None = None
|
||||
end: datetime | None = None
|
||||
agent_name: str | None = Field(default=None, max_length=200)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def ordered_window(self) -> "RunRequest":
|
||||
if (self.start is None) != (self.end is None):
|
||||
raise ValueError("Choose both a start and an end time")
|
||||
if self.start is not None and self.end is not None and self.start >= self.end:
|
||||
raise ValueError("Start time must be before end time")
|
||||
return self
|
||||
|
||||
|
||||
class WatchSkipped(Record):
|
||||
id: str
|
||||
name: str
|
||||
reason: str
|
||||
|
||||
|
||||
class WatchAllResult(Record):
|
||||
watching: tuple[str, ...]
|
||||
skipped: tuple[WatchSkipped, ...] = ()
|
||||
|
||||
|
||||
class FindingUpdate(Record):
|
||||
|
|
|
|||
|
|
@ -1,9 +1,19 @@
|
|||
import hashlib
|
||||
from datetime import datetime, timedelta
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from typing import Final, Literal
|
||||
|
||||
from litellm.proxy.lens.models import Finding, FindingDraft, Job, Lens, LensSettings, Scope, Worker
|
||||
from litellm.proxy.lens.models import (
|
||||
MAX_STEPS,
|
||||
Finding,
|
||||
FindingDraft,
|
||||
Job,
|
||||
Lens,
|
||||
LensSettings,
|
||||
Scope,
|
||||
Step,
|
||||
Worker,
|
||||
)
|
||||
|
||||
|
||||
def can_access(viewer: Scope, target: Scope) -> bool:
|
||||
|
|
@ -24,27 +34,54 @@ def replace_job(lens: Lens, job: Job) -> Lens:
|
|||
)
|
||||
|
||||
|
||||
SETTLE_DELAY = timedelta(minutes=2)
|
||||
|
||||
|
||||
def scheduled_window(lens: Lens, now: datetime) -> tuple[datetime, datetime]:
|
||||
end: Final = now - SETTLE_DELAY
|
||||
floor: Final = now - timedelta(hours=lens.settings.lookback_hours)
|
||||
start: Final = max(lens.last_scan_at, floor) if lens.last_scan_at else floor
|
||||
return min(start, end), end
|
||||
|
||||
|
||||
def next_scan_start(lens: Lens, job: Job, failed: bool) -> datetime | None:
|
||||
if failed or job.trigger == "manual":
|
||||
return lens.last_scan_at
|
||||
return max(lens.last_scan_at or job.end, job.end)
|
||||
|
||||
|
||||
def queue_job(
|
||||
lens: Lens,
|
||||
now: datetime,
|
||||
job_id: str,
|
||||
lookback_hours: int | None = None,
|
||||
settings: LensSettings | None = None,
|
||||
window: tuple[datetime, datetime] | None = None,
|
||||
trigger: Literal["schedule", "manual"] = "schedule",
|
||||
) -> Lens:
|
||||
if current_job(lens):
|
||||
return lens
|
||||
selected: Final = settings or lens.settings
|
||||
hours: Final = lookback_hours if lookback_hours is not None else (selected.lookback_hours if settings else None)
|
||||
start, end = window or (
|
||||
(now - timedelta(hours=hours), now - SETTLE_DELAY) if hours is not None else scheduled_window(lens, now)
|
||||
)
|
||||
job: Final = Job(
|
||||
id=job_id,
|
||||
created_at=now,
|
||||
start=now - timedelta(hours=lookback_hours if lookback_hours is not None else selected.lookback_hours),
|
||||
end=now - timedelta(minutes=2),
|
||||
start=start,
|
||||
end=end,
|
||||
settings=selected,
|
||||
revision=lens.revision,
|
||||
trigger=trigger,
|
||||
)
|
||||
return lens.model_copy(update=MappingProxyType({"jobs": (job,)}))
|
||||
|
||||
|
||||
def add_step(job: Job, step: Step) -> Job:
|
||||
return job.model_copy(update=MappingProxyType({"steps": (*job.steps, step)[-MAX_STEPS:]}))
|
||||
|
||||
|
||||
def claim_job(lens: Lens, worker: Worker, now: datetime) -> Lens:
|
||||
job: Final = current_job(lens)
|
||||
if job is None or not can_access(worker.scope, lens.scope):
|
||||
|
|
|
|||
|
|
@ -108,12 +108,14 @@ class DatabaseTimeoutSettings(BaseModel):
|
|||
|
||||
database_statement_timeout: float | None = None
|
||||
database_lock_timeout: float | None = None
|
||||
database_idle_in_transaction_session_timeout: float | None = None
|
||||
|
||||
|
||||
def _pg_options_with_timeouts(
|
||||
existing_options: str,
|
||||
statement_timeout: float | None,
|
||||
lock_timeout: float | None,
|
||||
idle_in_transaction_session_timeout: float | None = None,
|
||||
) -> str:
|
||||
"""Return the Postgres ``options`` value carrying the configured timeouts.
|
||||
|
||||
|
|
@ -143,6 +145,7 @@ def _pg_options_with_timeouts(
|
|||
for name, seconds in (
|
||||
("statement_timeout", statement_timeout),
|
||||
("lock_timeout", lock_timeout),
|
||||
("idle_in_transaction_session_timeout", idle_in_transaction_session_timeout),
|
||||
)
|
||||
if seconds is not None and not re.search(rf"(?:-c\s*|--){re.escape(name)}=", existing_options)
|
||||
)
|
||||
|
|
@ -1160,6 +1163,7 @@ def run_server(
|
|||
db_extra_connection_params: dict | None = None
|
||||
db_statement_timeout: float | None = None
|
||||
db_lock_timeout: float | None = None
|
||||
db_idle_in_transaction_timeout: float | None = None
|
||||
general_settings = {}
|
||||
### GET DB TOKEN FOR RDS IAM / AZURE ENTRA AUTH ###
|
||||
|
||||
|
|
@ -1270,6 +1274,7 @@ def run_server(
|
|||
db_timeouts: Final = DatabaseTimeoutSettings.model_validate(general_settings)
|
||||
db_statement_timeout = db_timeouts.database_statement_timeout
|
||||
db_lock_timeout = db_timeouts.database_lock_timeout
|
||||
db_idle_in_transaction_timeout = db_timeouts.database_idle_in_transaction_session_timeout
|
||||
if database_url and database_url.startswith("os.environ/"):
|
||||
original_dir: Final = os.getcwd()
|
||||
# set the working directory to where this script is
|
||||
|
|
@ -1303,6 +1308,7 @@ def run_server(
|
|||
DISABLE_PREPARED_STATEMENTS_ENV_VAR,
|
||||
add_missing_query_params,
|
||||
idle_lifetime_params,
|
||||
postgres_connection_budget_message,
|
||||
reader_shareable_params,
|
||||
translate_libpq_ssl_params,
|
||||
unsupported_db_scheme,
|
||||
|
|
@ -1343,6 +1349,7 @@ def run_server(
|
|||
_url_query_value(resolved_url, "options"),
|
||||
db_statement_timeout,
|
||||
db_lock_timeout,
|
||||
db_idle_in_transaction_timeout,
|
||||
)
|
||||
writer_url: Final = (
|
||||
_with_query_value(resolved_url, "options", pg_options)
|
||||
|
|
@ -1373,6 +1380,7 @@ def run_server(
|
|||
_url_query_value(read_replica_url, "options"),
|
||||
db_statement_timeout,
|
||||
db_lock_timeout,
|
||||
db_idle_in_transaction_timeout,
|
||||
)
|
||||
os.environ["DATABASE_URL_READ_REPLICA"] = translate_libpq_ssl_params(
|
||||
add_missing_query_params(
|
||||
|
|
@ -1385,6 +1393,19 @@ def run_server(
|
|||
lifetime_params,
|
||||
)
|
||||
)
|
||||
print(
|
||||
postgres_connection_budget_message(
|
||||
writer_limit=_url_query_value(os.getenv("DATABASE_URL"), "connection_limit")
|
||||
or f"{db_connection_pool_limit}",
|
||||
reader_limit=(
|
||||
_url_query_value(os.getenv("DATABASE_URL_READ_REPLICA"), "connection_limit")
|
||||
or f"{db_connection_pool_limit}"
|
||||
)
|
||||
if read_replica_url
|
||||
else None,
|
||||
num_workers=f"{1 if run_hypercorn else num_workers}",
|
||||
)
|
||||
)
|
||||
from litellm_proxy_extras.prisma_toolchain import prisma_cli_available
|
||||
|
||||
is_prisma_runnable: Final = prisma_cli_available()
|
||||
|
|
|
|||
|
|
@ -410,6 +410,7 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
|
|||
AuthCacheInvalidationSubscriber,
|
||||
)
|
||||
from litellm.proxy.common_utils.callback_utils import initialize_callbacks_on_proxy
|
||||
from litellm.proxy.common_utils.codex_model_catalog import codex_model_list_body
|
||||
from litellm.proxy.common_utils.config_includes import resolve_include_file_path, resolve_includes
|
||||
from litellm.proxy.common_utils.config_sync_pubsub import ConfigSyncSubscriber
|
||||
from litellm.proxy.common_utils.debug_utils import init_verbose_loggers
|
||||
|
|
@ -9443,7 +9444,7 @@ def _format_fallback_metadata_sse_event(
|
|||
|
||||
def _restamp_streaming_chunk_model(
|
||||
*,
|
||||
chunk: Any,
|
||||
chunk: object,
|
||||
requested_model_from_client: str,
|
||||
request_data: dict,
|
||||
model_mismatch_logged: bool,
|
||||
|
|
@ -11596,6 +11597,7 @@ async def model_list(
|
|||
fallback_type: str | None = None,
|
||||
scope: str | None = None,
|
||||
healthy_only: bool | None = False,
|
||||
client_version: str | None = None,
|
||||
):
|
||||
"""
|
||||
Use `/model/info` - to get detailed model information, example - pricing, mode, etc.
|
||||
|
|
@ -11603,6 +11605,11 @@ async def model_list(
|
|||
This is just for compatibility with openai projects like aider.
|
||||
|
||||
Query Parameters:
|
||||
- client_version: Sent by Codex CLI (`?client_version=0.159.3`) when it fetches a
|
||||
provider's model catalog. When present, the response is Codex's own
|
||||
catalog shape (`{"models": [...]}`) built from the same listing, with
|
||||
each model's `model_info.service_tiers` as its service tiers; absent,
|
||||
the OpenAI shape below is returned
|
||||
- include_metadata: Include additional metadata in the response with fallback information
|
||||
- fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy")
|
||||
Defaults to "general" when include_metadata=true
|
||||
|
|
@ -11745,8 +11752,16 @@ async def model_list(
|
|||
model_info["id"] = response_id
|
||||
model_data.append(model_info)
|
||||
|
||||
admin_listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above
|
||||
if client_version is not None:
|
||||
return Response(
|
||||
content=codex_model_list_body(
|
||||
admin_listing, admin_entries, llm_router, team_id or user_api_key_dict.team_id
|
||||
),
|
||||
media_type="application/json",
|
||||
)
|
||||
|
||||
if wants_anthropic_format:
|
||||
admin_listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above
|
||||
return create_anthropic_model_list_response(
|
||||
admin_listing,
|
||||
display_names=configured_display_names(admin_entries, llm_router),
|
||||
|
|
@ -11805,8 +11820,14 @@ async def model_list(
|
|||
model_info["id"] = response_id
|
||||
model_data.append(model_info)
|
||||
|
||||
listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above
|
||||
if client_version is not None:
|
||||
return Response(
|
||||
content=codex_model_list_body(listing, entries, llm_router, team_id or user_api_key_dict.team_id),
|
||||
media_type="application/json",
|
||||
)
|
||||
|
||||
if wants_anthropic_format:
|
||||
listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above
|
||||
return create_anthropic_model_list_response(
|
||||
listing,
|
||||
display_names=configured_display_names(entries, llm_router),
|
||||
|
|
|
|||
|
|
@ -7661,8 +7661,9 @@ async def _run_spend_logs_job(
|
|||
)
|
||||
|
||||
# Tool usage tracking: drain the request-time queue into the tool index and the
|
||||
# LiteLLM_DailyToolSpend rollup. Never retried; a dropped batch is permanently
|
||||
# absent from the rollup, so failures log at error.
|
||||
# LiteLLM_DailyToolSpend rollup. A batch Postgres had no connection for was never
|
||||
# sent, so it is requeued and the job stops; any other failure is dropped because
|
||||
# a replay could double-count the rollup.
|
||||
async with prisma_client._tool_usage_transactions_lock:
|
||||
tool_usage_to_process: Final = prisma_client.tool_usage_transactions[:MAX_LOGS_PER_INTERVAL]
|
||||
prisma_client.tool_usage_transactions = prisma_client.tool_usage_transactions[len(tool_usage_to_process) :]
|
||||
|
|
@ -7674,11 +7675,25 @@ async def _run_spend_logs_job(
|
|||
transactions=tool_usage_to_process,
|
||||
)
|
||||
except Exception as tool_tracking_err:
|
||||
verbose_proxy_logger.error(
|
||||
"Spend tracking - tool usage flush failed; %s tool usage transactions dropped: %s",
|
||||
len(tool_usage_to_process),
|
||||
tool_tracking_err,
|
||||
)
|
||||
if PrismaDBExceptionHandler.is_database_capacity_error(tool_tracking_err):
|
||||
async with prisma_client._tool_usage_transactions_lock:
|
||||
prisma_client.tool_usage_transactions = [
|
||||
*tool_usage_to_process,
|
||||
*prisma_client.tool_usage_transactions,
|
||||
]
|
||||
verbose_proxy_logger.warning(
|
||||
"Spend tracking - database out of connections during tool usage flush, "
|
||||
"requeued %s tool usage transactions for the next flush: %s",
|
||||
len(tool_usage_to_process),
|
||||
tool_tracking_err,
|
||||
)
|
||||
raise
|
||||
else:
|
||||
verbose_proxy_logger.error(
|
||||
"Spend tracking - tool usage flush failed; %s tool usage transactions dropped: %s",
|
||||
len(tool_usage_to_process),
|
||||
tool_tracking_err,
|
||||
)
|
||||
|
||||
async with prisma_client._model_usage_transactions_lock:
|
||||
model_usage_to_process: Final = prisma_client.model_usage_transactions
|
||||
|
|
|
|||
|
|
@ -479,6 +479,7 @@ def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream])
|
|||
|
||||
_NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType({})
|
||||
_SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str])
|
||||
_ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
|
||||
_DEPLOYMENT_SELECTED_EVENT: Final = "litellm.request.deployment_selected"
|
||||
|
|
@ -503,6 +504,10 @@ def _deployment_pick_attributes(model: str, request_kwargs: Mapping[str, object]
|
|||
)
|
||||
|
||||
|
||||
def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {})
|
||||
|
||||
|
||||
def _as_retry_skipped_deployment_ids(value: object) -> tuple[str, ...]:
|
||||
return tuple(item for item in value if isinstance(item, str)) if isinstance(value, tuple) else ()
|
||||
|
||||
|
|
@ -10650,6 +10655,63 @@ class Router:
|
|||
return display_name
|
||||
return None
|
||||
|
||||
def routable_model_group(self, model_name: str) -> str:
|
||||
"""
|
||||
The model group a request to model_name routes to: its target when
|
||||
model_name is a `model_group_alias`, else model_name itself.
|
||||
"""
|
||||
target: Final = self._get_model_from_alias(model_name)
|
||||
return target if target is not None else model_name
|
||||
|
||||
def _routable_deployments(self, model_name: str, team_id: str | None) -> tuple[DeploymentTypedDict, ...]:
|
||||
"""
|
||||
The deployments a request from team_id to model_name can route to, in
|
||||
model_list order and via the same O(1) index lookup as
|
||||
get_configured_display_name, selected the way routing selects them: a
|
||||
`model_group_alias` reads its target's deployments, another team's
|
||||
deployment of the name is left out, a caller with no team reads the
|
||||
deployments no team owns (every deployment of the name when a team owns
|
||||
each one, which is what an admin's request routes to), and one an admin
|
||||
paused via `LiteLLM_ProxyModelTable.blocked` is left out.
|
||||
|
||||
Returns an empty tuple for wildcard-expanded or unknown names.
|
||||
"""
|
||||
named: Final = self._get_all_deployments(model_name=self.routable_model_group(model_name), team_id=team_id)
|
||||
usable: Final = tuple(
|
||||
deployment for deployment in named if self._deployment_usable_by_team(deployment, team_id)
|
||||
)
|
||||
return tuple(
|
||||
deployment
|
||||
for deployment in (usable or named)
|
||||
if _configured_model_info(deployment).get("blocked") is not True
|
||||
)
|
||||
|
||||
def get_configured_service_tiers(self, model_name: str, team_id: str | None = None) -> tuple[object, ...]:
|
||||
"""
|
||||
Return the service_tiers value each deployment a request from team_id
|
||||
to model_name can route to (see _routable_deployments) configures in
|
||||
its model_info, unvalidated and in model_list order, None for a
|
||||
deployment that sets none; the caller validates the shape it expects
|
||||
and decides how the deployments combine.
|
||||
|
||||
Returns an empty tuple for wildcard-expanded or unknown names.
|
||||
"""
|
||||
return tuple(
|
||||
_configured_model_info(deployment).get("service_tiers")
|
||||
for deployment in self._routable_deployments(model_name, team_id)
|
||||
)
|
||||
|
||||
def get_routable_upstream_model(self, model_name: str, team_id: str | None = None) -> str | None:
|
||||
"""
|
||||
Return the `litellm_params.model` of the first deployment a request
|
||||
from team_id to model_name can route to (see _routable_deployments).
|
||||
|
||||
Returns None for wildcard-expanded or unknown names, and when the team
|
||||
can route to no deployment of the name.
|
||||
"""
|
||||
deployment: Final = next(iter(self._routable_deployments(model_name, team_id)), None)
|
||||
return deployment["litellm_params"].get("model") if deployment is not None else None
|
||||
|
||||
def get_credential_deployment(self, model_id: str, team_id: str | None = None) -> Deployment | None:
|
||||
"""
|
||||
The deployment a passthrough endpoint (files, batches, etc.) resolves for a
|
||||
|
|
|
|||
124
litellm/utils.py
124
litellm/utils.py
|
|
@ -4897,7 +4897,7 @@ def get_optional_params(
|
|||
drop_params=bool(drop_params),
|
||||
)
|
||||
elif custom_llm_provider == "bedrock_mantle":
|
||||
optional_params = litellm.BedrockMantleChatConfig().map_openai_params(
|
||||
optional_params = ProviderConfigManager._get_bedrock_mantle_config(model).map_openai_params(
|
||||
non_default_params=non_default_params,
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
|
|
@ -5669,7 +5669,7 @@ def _get_model_cost_key(potential_key: str) -> str | None:
|
|||
return None
|
||||
|
||||
|
||||
def _get_model_info_from_model_cost(key: str) -> dict:
|
||||
def _get_model_info_from_model_cost(key: str) -> dict[str, Any]:
|
||||
return litellm.model_cost[key]
|
||||
|
||||
|
||||
|
|
@ -5724,12 +5724,26 @@ from typing_extensions import ReadOnly, TypedDict
|
|||
class PotentialModelNamesAndCustomLLMProvider(TypedDict):
|
||||
split_model: str
|
||||
combined_model_name: str
|
||||
region_free_combined_model_name: ReadOnly[str]
|
||||
stripped_model_name: str
|
||||
combined_stripped_model_name: str
|
||||
provider_prefixed_model_name: ReadOnly[str]
|
||||
custom_llm_provider: str
|
||||
|
||||
|
||||
def _first_registered_match(
|
||||
candidates: Sequence[str], custom_llm_provider: str | None
|
||||
) -> tuple[str | None, dict[str, Any] | None]:
|
||||
registered_keys: Final = (key for key in map(_get_model_cost_key, candidates) if key is not None)
|
||||
entries: Final = ((key, _get_model_info_from_model_cost(key=key)) for key in registered_keys)
|
||||
matches: Final = (
|
||||
(key, info)
|
||||
for key, info in entries
|
||||
if _check_provider_match(model_info=info, custom_llm_provider=custom_llm_provider)
|
||||
)
|
||||
return next(matches, (None, None))
|
||||
|
||||
|
||||
def _get_model_info_from_generalization(
|
||||
model: str,
|
||||
potential_model_names: PotentialModelNamesAndCustomLLMProvider,
|
||||
|
|
@ -5751,6 +5765,7 @@ def _get_model_info_from_generalization(
|
|||
candidates: Final = (
|
||||
potential_model_names["combined_model_name"],
|
||||
model,
|
||||
potential_model_names["region_free_combined_model_name"],
|
||||
potential_model_names["split_model"],
|
||||
potential_model_names["combined_stripped_model_name"],
|
||||
potential_model_names["stripped_model_name"],
|
||||
|
|
@ -5828,6 +5843,11 @@ def _get_potential_model_names(model: str, custom_llm_provider: str | None) -> P
|
|||
return PotentialModelNamesAndCustomLLMProvider(
|
||||
split_model=region_free_split_model,
|
||||
combined_model_name=combined_model_name,
|
||||
region_free_combined_model_name=(
|
||||
f"bedrock_mantle/{region_free_split_model}"
|
||||
if custom_llm_provider == "bedrock_mantle"
|
||||
else combined_model_name
|
||||
),
|
||||
stripped_model_name=stripped_model_name,
|
||||
combined_stripped_model_name=region_free_combined_stripped_model_name,
|
||||
provider_prefixed_model_name=provider_cost_key or provider_prefixed_model_name,
|
||||
|
|
@ -6007,78 +6027,29 @@ def _get_model_info_helper(
|
|||
Check if: (in order of specificity)
|
||||
1. 'custom_llm_provider/model' in litellm.model_cost. Checks "groq/llama3-8b-8192" if model="llama3-8b-8192" and custom_llm_provider="groq"
|
||||
2. 'model' in litellm.model_cost. Checks "gemini-1.5-pro-002" in litellm.model_cost if model="gemini-1.5-pro-002" and custom_llm_provider=None
|
||||
3. 'split_model' in litellm.model_cost. Checks "au.anthropic.claude-opus-4-8" in litellm.model_cost if model="bedrock/au.anthropic.claude-opus-4-8"
|
||||
4. 'combined_stripped_model_name' in litellm.model_cost. Checks if 'gemini/gemini-1.5-flash' in model map, if 'gemini/gemini-1.5-flash-001' given.
|
||||
5. 'stripped_model_name' in litellm.model_cost. Checks if 'ft:gpt-3.5-turbo' in model map, if 'ft:gpt-3.5-turbo:my-org:custom_suffix:id' given.
|
||||
6. 'provider_prefixed_model_name' in litellm.model_cost, for providers whose own model ids repeat the
|
||||
3. 'region_free_combined_model_name' in litellm.model_cost. Checks "bedrock_mantle/anthropic.claude-opus-5-5" if
|
||||
model="bedrock_mantle/us-east-1/anthropic.claude-opus-5-5", before 4 reaches the bare Bedrock row. Same as 1 for every other provider.
|
||||
4. 'split_model' in litellm.model_cost. Checks "au.anthropic.claude-opus-4-8" in litellm.model_cost if model="bedrock/au.anthropic.claude-opus-4-8"
|
||||
5. 'combined_stripped_model_name' in litellm.model_cost. Checks if 'gemini/gemini-1.5-flash' in model map, if 'gemini/gemini-1.5-flash-001' given.
|
||||
6. 'stripped_model_name' in litellm.model_cost. Checks if 'ft:gpt-3.5-turbo' in model map, if 'ft:gpt-3.5-turbo:my-org:custom_suffix:id' given.
|
||||
7. 'provider_prefixed_model_name' in litellm.model_cost, for providers whose own model ids repeat the
|
||||
litellm provider name. Checks "perplexity/perplexity/glm-5.2" if model="perplexity/glm-5.2" and
|
||||
custom_llm_provider="perplexity", where 1-5 all read the leading "perplexity/" as the litellm prefix
|
||||
and strip it. Tried last so no model that already resolves through 1-5 can change.
|
||||
custom_llm_provider="perplexity", where 1-6 all read the leading "perplexity/" as the litellm prefix
|
||||
and strip it. Tried last so no model that already resolves through 1-6 can change.
|
||||
"""
|
||||
|
||||
_model_info: dict[str, Any] | None = None
|
||||
key: str | None = None
|
||||
|
||||
# Use case-insensitive lookup for all model name checks
|
||||
_matched_key = _get_model_cost_key(combined_model_name)
|
||||
if _matched_key is not None:
|
||||
key = _matched_key
|
||||
_model_info = _get_model_info_from_model_cost(key=cast(str, key))
|
||||
if not _check_provider_match(
|
||||
model_info=_model_info,
|
||||
custom_llm_provider=model_cost_custom_llm_provider,
|
||||
):
|
||||
_model_info = None
|
||||
if _model_info is None:
|
||||
_matched_key = _get_model_cost_key(model)
|
||||
if _matched_key is not None:
|
||||
key = _matched_key
|
||||
_model_info = _get_model_info_from_model_cost(key=cast(str, key))
|
||||
if not _check_provider_match(
|
||||
model_info=_model_info,
|
||||
custom_llm_provider=model_cost_custom_llm_provider,
|
||||
):
|
||||
_model_info = None
|
||||
if _model_info is None:
|
||||
_matched_key = _get_model_cost_key(split_model)
|
||||
if _matched_key is not None:
|
||||
key = _matched_key
|
||||
_model_info = _get_model_info_from_model_cost(key=cast(str, key))
|
||||
if not _check_provider_match(
|
||||
model_info=_model_info,
|
||||
custom_llm_provider=model_cost_custom_llm_provider,
|
||||
):
|
||||
_model_info = None
|
||||
if _model_info is None:
|
||||
_matched_key = _get_model_cost_key(combined_stripped_model_name)
|
||||
if _matched_key is not None:
|
||||
key = _matched_key
|
||||
_model_info = _get_model_info_from_model_cost(key=cast(str, key))
|
||||
if not _check_provider_match(
|
||||
model_info=_model_info,
|
||||
custom_llm_provider=model_cost_custom_llm_provider,
|
||||
):
|
||||
_model_info = None
|
||||
if _model_info is None:
|
||||
_matched_key = _get_model_cost_key(stripped_model_name)
|
||||
if _matched_key is not None:
|
||||
key = _matched_key
|
||||
_model_info = _get_model_info_from_model_cost(key=cast(str, key))
|
||||
if not _check_provider_match(
|
||||
model_info=_model_info,
|
||||
custom_llm_provider=model_cost_custom_llm_provider,
|
||||
):
|
||||
_model_info = None
|
||||
if _model_info is None:
|
||||
_matched_key = _get_model_cost_key(provider_prefixed_model_name)
|
||||
if _matched_key is not None:
|
||||
key = _matched_key
|
||||
_model_info = _get_model_info_from_model_cost(key=cast(str, key))
|
||||
if not _check_provider_match(
|
||||
model_info=_model_info,
|
||||
custom_llm_provider=model_cost_custom_llm_provider,
|
||||
):
|
||||
_model_info = None
|
||||
lookup_order: Final = (
|
||||
combined_model_name,
|
||||
model,
|
||||
potential_model_names["region_free_combined_model_name"],
|
||||
split_model,
|
||||
combined_stripped_model_name,
|
||||
stripped_model_name,
|
||||
provider_prefixed_model_name,
|
||||
)
|
||||
lookup: Final = _first_registered_match(lookup_order, model_cost_custom_llm_provider)
|
||||
key: str | None = lookup[0]
|
||||
_model_info: dict[str, Any] | None = lookup[1]
|
||||
|
||||
if _model_info is not None and key is not None and _model_info.get("mode", "chat") in _BACKFILL_MODES:
|
||||
fill_missing: Final = match_fill_missing_generalizations(key, _model_info.get("litellm_provider", ""))
|
||||
|
|
@ -8478,10 +8449,7 @@ class ProviderConfigManager:
|
|||
LlmProviders.DEEPSEEK: (lambda: litellm.DeepSeekChatConfig(), False),
|
||||
LlmProviders.TENCENT: (lambda: litellm.TencentChatConfig(), False),
|
||||
LlmProviders.GROQ: (lambda: litellm.GroqChatConfig(), False),
|
||||
LlmProviders.BEDROCK_MANTLE: (
|
||||
lambda: litellm.BedrockMantleChatConfig(),
|
||||
False,
|
||||
),
|
||||
LlmProviders.BEDROCK_MANTLE: (ProviderConfigManager._get_bedrock_mantle_config, True),
|
||||
LlmProviders.A2A: (lambda: litellm.A2AConfig(), False),
|
||||
LlmProviders.BYTEZ: (lambda: litellm.BytezChatConfig(), False),
|
||||
LlmProviders.DATABRICKS: (lambda: litellm.DatabricksConfig(), False),
|
||||
|
|
@ -8669,6 +8637,12 @@ class ProviderConfigManager:
|
|||
|
||||
return get_bedrock_chat_config(model=model)
|
||||
|
||||
@staticmethod
|
||||
def _get_bedrock_mantle_config(model: str) -> BaseConfig:
|
||||
from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantle_chat_config
|
||||
|
||||
return bedrock_mantle_chat_config(model)
|
||||
|
||||
@staticmethod
|
||||
def _get_cohere_config(model: str) -> BaseConfig:
|
||||
"""Get Cohere config based on route."""
|
||||
|
|
|
|||
|
|
@ -16085,6 +16085,28 @@
|
|||
"input_cost_per_image_token": 4.7e-07,
|
||||
"source": "https://cohere.com/pricing"
|
||||
},
|
||||
"cohere/embed-v5.0-pro": {
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"litellm_provider": "cohere",
|
||||
"max_input_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"output_vector_size": 2048,
|
||||
"supports_embedding_image_input": true,
|
||||
"source": "https://cohere.com/pricing"
|
||||
},
|
||||
"cohere/embed-v5.0-fast": {
|
||||
"input_cost_per_token": 8e-08,
|
||||
"litellm_provider": "cohere",
|
||||
"max_input_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "embedding",
|
||||
"output_cost_per_token": 0.0,
|
||||
"output_vector_size": 2048,
|
||||
"supports_embedding_image_input": true,
|
||||
"source": "https://cohere.com/pricing"
|
||||
},
|
||||
"cohere/parse-v5.0": {
|
||||
"litellm_provider": "cohere",
|
||||
"mode": "ocr",
|
||||
|
|
@ -61056,6 +61078,9 @@
|
|||
"input_cost_per_audio_token": 2e-06,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 98304,
|
||||
"max_output_tokens": 32768,
|
||||
"max_tokens": 32768,
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_token": 1.2e-05,
|
||||
"source": "https://ai.google.dev/gemini-api/docs/pricing",
|
||||
|
|
@ -61078,6 +61103,9 @@
|
|||
"input_cost_per_audio_token": 3.5e-06,
|
||||
"input_cost_per_token": 3.5e-06,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 65536,
|
||||
"max_tokens": 65536,
|
||||
"mode": "audio_transcription",
|
||||
"output_cost_per_token": 2.1e-05,
|
||||
"source": "https://ai.google.dev/gemini-api/docs/pricing",
|
||||
|
|
@ -64092,6 +64120,23 @@
|
|||
"video"
|
||||
]
|
||||
},
|
||||
"xai/grok-imagine-video-1.5-lite": {
|
||||
"input_cost_per_image": 0.01,
|
||||
"litellm_provider": "xai",
|
||||
"mode": "video_generation",
|
||||
"output_cost_per_second": 0.02,
|
||||
"output_cost_per_second_1080p": 0.14,
|
||||
"output_cost_per_second_480p": 0.02,
|
||||
"output_cost_per_second_720p": 0.03,
|
||||
"source": "https://docs.x.ai/developers/pricing",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"video"
|
||||
]
|
||||
},
|
||||
"xai/grok-imagine-video-1.5-2026-05-30": {
|
||||
"input_cost_per_image": 0.01,
|
||||
"litellm_provider": "xai",
|
||||
|
|
@ -75511,6 +75556,42 @@
|
|||
"supports_vision": true,
|
||||
"supports_web_search": true
|
||||
},
|
||||
"openrouter/openai/gpt-image-2": {
|
||||
"input_cost_per_image_token": 8e-06,
|
||||
"input_cost_per_token": 5e-06,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "image_generation",
|
||||
"output_cost_per_image_token": 3e-05,
|
||||
"source": "https://openrouter.ai/api/v1/images/models/openai/gpt-image-2/endpoints",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
],
|
||||
"supports_vision": true
|
||||
},
|
||||
"openrouter/openai/gpt-image-2.5-flare": {
|
||||
"input_cost_per_image_token": 8e-06,
|
||||
"input_cost_per_token": 5e-06,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "image_generation",
|
||||
"output_cost_per_image_token": 3e-05,
|
||||
"source": "https://openrouter.ai/api/v1/images/models/openai/gpt-image-2.5-flare/endpoints",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
],
|
||||
"supports_vision": true
|
||||
},
|
||||
"openrouter/openai/gpt-image-2.5-sunburst": {
|
||||
"input_cost_per_image_token": 8e-06,
|
||||
"input_cost_per_token": 5e-06,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "image_generation",
|
||||
"output_cost_per_image_token": 3e-05,
|
||||
"source": "https://openrouter.ai/api/v1/images/models/openai/gpt-image-2.5-sunburst/endpoints",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
],
|
||||
"supports_vision": true
|
||||
},
|
||||
"openrouter/openai/gpt-5.4-mini:batch": {
|
||||
"cache_read_input_token_cost": 3.75e-08,
|
||||
"input_cost_per_token": 3.75e-07,
|
||||
|
|
|
|||
|
|
@ -254,6 +254,7 @@ proxy-dev = [
|
|||
"a2a-sdk==1.1.0",
|
||||
]
|
||||
ci = [
|
||||
"psutil==7.2.2",
|
||||
# These are lazily imported at call sites; keep them out of core deps to
|
||||
# avoid bloating the base SDK install (google-generativeai pulls grpcio +
|
||||
# protobuf, Pillow is a compiled C extension).
|
||||
|
|
@ -314,7 +315,8 @@ include = [
|
|||
"litellm/router_strategy/complexity_router/artifacts/*.json",
|
||||
"litellm/router_strategy/complexity_router/fuse_presets.json",
|
||||
"litellm/proxy/model_insights_tasks.json",
|
||||
"litellm/proxy/client/cli/commands/codex_base_instructions.md",
|
||||
"litellm/proxy/common_utils/codex_base_instructions.md",
|
||||
"litellm/proxy/common_utils/codex_bundled_models_0.159.3.json",
|
||||
"litellm/proxy/lens/prompts/*.md",
|
||||
]
|
||||
exclude = [
|
||||
|
|
|
|||
643
scripts/quickstart.ps1
Normal file
643
scripts/quickstart.ps1
Normal file
|
|
@ -0,0 +1,643 @@
|
|||
# LiteLLM Gateway quickstart for Windows: the gateway, Postgres, and the admin UI in one command.
|
||||
# powershell -c "irm https://raw.githubusercontent.com/BerriAI/litellm/main/scripts/quickstart.ps1 | iex"
|
||||
# On macOS and Linux, scripts/quickstart.sh does the same.
|
||||
#
|
||||
# To read it before running it:
|
||||
# irm https://raw.githubusercontent.com/BerriAI/litellm/main/scripts/quickstart.ps1 -OutFile quickstart.ps1
|
||||
# notepad quickstart.ps1
|
||||
# powershell -ExecutionPolicy Bypass -File quickstart.ps1
|
||||
#
|
||||
# Asks where to keep the files, then offers to copy the admin password and
|
||||
# open the admin UI; every question has a default you accept by pressing Enter.
|
||||
# It asks nothing when input is not a console, under CI or Claude Code, or
|
||||
# with -Yes.
|
||||
#
|
||||
# -Yes, or LITELLM_YES=1 no questions: install to ~\litellm-gateway, don't open a browser
|
||||
# LITELLM_DIR folder to install into (skips the folder question)
|
||||
# LITELLM_PORT port for the gateway (default 4000, or the next free one)
|
||||
# NO_COLOR plain output, which agents, CI, and log files always get
|
||||
#
|
||||
# New installs listen on this machine only (127.0.0.1). To reach the gateway
|
||||
# from other machines, remove LITELLM_BIND from .env and put it behind TLS.
|
||||
#
|
||||
# Keys and the database password are random, written only to .env, which only
|
||||
# your Windows account can read, and never printed. When you ask, the admin
|
||||
# password goes straight to your clipboard. Needs Docker Desktop, Podman, or
|
||||
# Rancher Desktop.
|
||||
# Works in Windows PowerShell 5.1 and PowerShell 7. Everything runs inside
|
||||
# Invoke-LiteLLMQuickstart, so a partial download runs nothing.
|
||||
param([switch]$Yes)
|
||||
|
||||
function Invoke-LiteLLMQuickstart {
|
||||
param([switch]$Yes)
|
||||
$ErrorActionPreference = 'Stop'
|
||||
# Windows PowerShell draws a progress bar for every web request, which makes them crawl.
|
||||
$ProgressPreference = 'SilentlyContinue'
|
||||
Set-StrictMode -Version 2
|
||||
|
||||
$composeUrl = 'https://raw.githubusercontent.com/BerriAI/litellm/main/docker/docker-compose.quickstart.yml'
|
||||
if ($env:LITELLM_COMPOSE_URL) { $composeUrl = $env:LITELLM_COMPOSE_URL }
|
||||
$onWindows = ($PSVersionTable.PSEdition -eq 'Desktop') -or ((Test-Path variable:IsWindows) -and $IsWindows)
|
||||
|
||||
# ------------------------------------------------------------ output
|
||||
|
||||
# A person at a console gets colors, step marks, and a spinner. Agents, CI,
|
||||
# log files, and NO_COLOR get the same lines as plain text.
|
||||
$style = (-not [Console]::IsOutputRedirected) -and (-not $env:NO_COLOR) -and (-not $env:CI) -and
|
||||
(-not $env:CLAUDECODE) -and ($env:TERM -ne 'dumb')
|
||||
# Escape sequences give the brand blue and bold; older consoles get the
|
||||
# sixteen console colors instead.
|
||||
$vt = $style -and $Host.UI.PSObject.Properties['SupportsVirtualTerminal'] -and $Host.UI.SupportsVirtualTerminal
|
||||
# Symbols need a console font that has them: Windows Terminal, VS Code, or
|
||||
# any macOS or Linux terminal. The classic console gets plain ASCII.
|
||||
$unicode = $style -and ((-not $onWindows) -or $env:WT_SESSION -or ($env:TERM_PROGRAM -eq 'vscode'))
|
||||
if ($unicode) {
|
||||
$sym = @{ Ok = [char]0x2713; Warn = '!'; Err = [char]0x2717; Head = [char]0x25C6; Ask = '?'; Pointer = [char]0x276F
|
||||
Frames = @([char]0x280B, [char]0x2819, [char]0x2839, [char]0x2838, [char]0x283C, [char]0x2834, [char]0x2826, [char]0x2827, [char]0x2807, [char]0x280F)
|
||||
Hint = "$([char]0x2191)/$([char]0x2193) to move, Enter to choose"; Dot = [char]0x00B7
|
||||
TL = [char]0x256D; TR = [char]0x256E; BL = [char]0x2570; BR = [char]0x256F; H = [char]0x2500; V = [char]0x2502 }
|
||||
} else {
|
||||
$sym = @{ Ok = '+'; Warn = '!'; Err = 'x'; Head = '*'; Ask = '?'; Pointer = '>'; Frames = @('|', '/', '-', '\')
|
||||
Hint = 'Up/Down to move, Enter to choose'; Dot = '-'; TL = '+'; TR = '+'; BL = '+'; BR = '+'; H = '-'; V = '|' }
|
||||
}
|
||||
$e = [char]27
|
||||
$ansi = @{ a = "$e[38;2;91;108;255m"; b = "$e[1m"; d = "$e[90m"; g = "$e[32m"; y = "$e[33m"; r = "$e[1;31m"; u = "$e[1;38;2;91;108;255m"; off = "$e[0m" }
|
||||
$colors = @{ a = 'Blue'; b = $null; d = 'DarkGray'; g = 'Green'; y = 'Yellow'; r = 'Red'; u = 'Blue' }
|
||||
|
||||
# Say "text with {a:accent} {b:bold} {d:dim} {g:green} {y:yellow} {r:red} {u:link} spans"
|
||||
function Say {
|
||||
param([string]$Text, [switch]$Err, [switch]$NoNewline)
|
||||
$parts = [regex]::Split($Text, '(\{[abdgyru]:[^}]*\})')
|
||||
# Errors go out plain when stderr is redirected, so a log file gets no escape codes.
|
||||
if (-not $style -or ($Err -and [Console]::IsErrorRedirected)) {
|
||||
$plain = ($parts | ForEach-Object { if ($_ -match '^\{[abdgyru]:(.*)\}$') { $Matches[1] } else { $_ } }) -join ''
|
||||
$stream = if ($Err) { [Console]::Error } else { [Console]::Out }
|
||||
if ($NoNewline) { $stream.Write($plain) } else { $stream.WriteLine($plain) }
|
||||
return
|
||||
}
|
||||
if ($vt) {
|
||||
$line = ($parts | ForEach-Object { if ($_ -match '^\{([abdgyru]):(.*)\}$') { $ansi[$Matches[1]] + $Matches[2] + $ansi.off } else { $_ } }) -join ''
|
||||
Write-Host $line -NoNewline:$NoNewline
|
||||
return
|
||||
}
|
||||
foreach ($p in $parts) {
|
||||
if ($p -match '^\{([abdgyru]):(.*)\}$') {
|
||||
$c = $colors[$Matches[1]]
|
||||
if ($c) { Write-Host $Matches[2] -NoNewline -ForegroundColor $c } else { Write-Host $Matches[2] -NoNewline }
|
||||
} elseif ($p) { Write-Host $p -NoNewline }
|
||||
}
|
||||
if (-not $NoNewline) { Write-Host '' }
|
||||
}
|
||||
# Markup characters in a value (a path, a URL) must not open a span.
|
||||
function Lit([string]$s) { return $s.Replace('{', '(').Replace('}', ')') }
|
||||
function Step([string]$Text) { if ($style) { Say ("{g:$($sym.Ok)} " + $Text) } else { Say $Text } }
|
||||
function Warn([string]$Text) { if ($style) { Say ("{y:$($sym.Warn)} " + $Text) } else { Say $Text } }
|
||||
function Fail([string]$Head, [string]$Rest = '') {
|
||||
$tail = if ($Rest) { " $Rest" } else { '' }
|
||||
if ($style) { Say ("{r:$($sym.Err) $Head}" + $tail) -Err } else { Say ($Head + $tail) -Err }
|
||||
}
|
||||
function Tildify([string]$Path) {
|
||||
if ($Path -eq $HOME) { return '~' }
|
||||
if ($Path.StartsWith($HOME + [IO.Path]::DirectorySeparatorChar, [StringComparison]::OrdinalIgnoreCase)) { return '~' + $Path.Substring($HOME.Length) }
|
||||
return $Path
|
||||
}
|
||||
function Elapsed([datetime]$Since) {
|
||||
$s = [int]((Get-Date) - $Since).TotalSeconds
|
||||
if ($s -lt 60) { return "${s}s" } else { return "$([math]::Floor($s / 60))m $($s % 60)s" }
|
||||
}
|
||||
function Clear-Line {
|
||||
if ($vt) { Write-Host "`r$e[2K" -NoNewline } else { Write-Host ("`r" + (' ' * ([Console]::WindowWidth - 1)) + "`r") -NoNewline }
|
||||
}
|
||||
function Show-Cursor([bool]$Shown) { try { [Console]::CursorVisible = $Shown } catch { <# a host without a cursor #> } }
|
||||
|
||||
# Run a native program; returns its exit code and stdout lines. stderr is
|
||||
# dropped so Windows PowerShell does not turn it into a terminating error.
|
||||
function Invoke-Native {
|
||||
param([string]$File, [string[]]$Arguments)
|
||||
$prev = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
try {
|
||||
$out = & $File @Arguments 2>$null
|
||||
$code = $LASTEXITCODE
|
||||
} catch {
|
||||
$out = @(); $code = 1
|
||||
} finally {
|
||||
$ErrorActionPreference = $prev
|
||||
}
|
||||
return [pscustomobject]@{ Code = $code; Out = @($out) }
|
||||
}
|
||||
|
||||
# Write a text file as UTF-8 without a byte order mark, with LF line endings,
|
||||
# which is what Docker Compose and git expect.
|
||||
function Write-PlainText([string]$Path, [string]$Text, [switch]$Append) {
|
||||
$enc = New-Object System.Text.UTF8Encoding $false
|
||||
if ($Append) { [IO.File]::AppendAllText($Path, $Text, $enc) } else { [IO.File]::WriteAllText($Path, $Text, $enc) }
|
||||
}
|
||||
|
||||
# The POSIX cksum of a string's UTF-8 bytes: CRC-32 over the bytes and then
|
||||
# their length. quickstart.sh names projects with the same number, so both
|
||||
# scripts find the same database for a folder. Int64 keeps the arithmetic
|
||||
# unsigned; hex literals with the top bit set would be negative here.
|
||||
function Get-Cksum([string]$Text) {
|
||||
$bytes = New-Object System.Collections.Generic.List[byte]
|
||||
$bytes.AddRange([Text.Encoding]::UTF8.GetBytes($Text))
|
||||
for ($n = [int64]$bytes.Count; $n -gt 0; $n = $n -shr 8) { $bytes.Add([byte]($n -band 255)) }
|
||||
[int64]$crc = 0
|
||||
foreach ($b in $bytes) {
|
||||
$crc = $crc -bxor ([int64]$b -shl 24)
|
||||
for ($i = 0; $i -lt 8; $i++) {
|
||||
$crc = if ($crc -band 2147483648) { (($crc -shl 1) -bxor 79764919) -band 4294967295 } else { ($crc -shl 1) -band 4294967295 }
|
||||
}
|
||||
}
|
||||
return (-bnot $crc) -band 4294967295
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------ questions
|
||||
|
||||
$interactive = (-not $Yes) -and (-not $env:LITELLM_YES) -and (-not $env:CI) -and (-not $env:CLAUDECODE) -and
|
||||
(-not [Console]::IsInputRedirected) -and [Environment]::UserInteractive
|
||||
|
||||
# Menu "Question" Default @(@('label', 'note'), ...) -> the 1-based pick.
|
||||
# ReadKey holds on to Ctrl+C, so take it as a key and stop the way PowerShell
|
||||
# does; finally blocks still run and put the terminal back.
|
||||
function Read-Key {
|
||||
$saved = [Console]::TreatControlCAsInput
|
||||
try {
|
||||
[Console]::TreatControlCAsInput = $true
|
||||
$key = [Console]::ReadKey($true)
|
||||
} finally {
|
||||
[Console]::TreatControlCAsInput = $saved
|
||||
}
|
||||
if ($key.Key -eq 'C' -and ($key.Modifiers -band [ConsoleModifiers]::Control)) {
|
||||
throw [System.Management.Automation.PipelineStoppedException]::new()
|
||||
}
|
||||
return $key
|
||||
}
|
||||
|
||||
function Menu {
|
||||
param([string]$Question, [int]$Default, [object[]]$Options)
|
||||
if (-not $interactive) { return $Default }
|
||||
$count = $Options.Count
|
||||
$pad = ($Options | ForEach-Object { $_[0].Length } | Measure-Object -Maximum).Maximum
|
||||
Write-Host ''
|
||||
if ($style) { Say ("{a:$($sym.Ask)} {b:" + (Lit $Question) + '}') } else { Write-Host $Question }
|
||||
$choice = $Default
|
||||
try {
|
||||
Show-Cursor $false
|
||||
$first = $true
|
||||
$top = 0
|
||||
while ($true) {
|
||||
# Escape-code consoles move up relative to the cursor; the classic
|
||||
# Windows console jumps back to the row the menu started on.
|
||||
if (-not $first) { if ($vt) { Write-Host "$e[$($count + 1)A" -NoNewline } else { [Console]::SetCursorPosition(0, $top) } }
|
||||
$first = $false
|
||||
for ($i = 1; $i -le $count; $i++) {
|
||||
$label = (Lit $Options[$i - 1][0]).PadRight($pad)
|
||||
$note = if ($Options[$i - 1].Count -gt 1) { Lit $Options[$i - 1][1] } else { '' }
|
||||
Clear-Line
|
||||
if ($i -eq $choice) { Say " {u:$($sym.Pointer) $label} {d:$note}" } else { Say " $label {d:$note}" }
|
||||
}
|
||||
Clear-Line
|
||||
Say " {d:$($sym.Hint)}"
|
||||
if (-not $vt) { $top = [Console]::CursorTop - ($count + 1) }
|
||||
$key = Read-Key
|
||||
if ($key.Key -eq 'UpArrow' -or $key.KeyChar -eq 'k') { if ($choice -gt 1) { $choice-- } }
|
||||
elseif ($key.Key -eq 'DownArrow' -or $key.KeyChar -eq 'j') { if ($choice -lt $count) { $choice++ } }
|
||||
elseif ($key.KeyChar -match '^[1-9]$' -and [int]"$($key.KeyChar)" -le $count) { $choice = [int]"$($key.KeyChar)" }
|
||||
elseif ($key.Key -eq 'Enter') { break }
|
||||
}
|
||||
if ($vt) { Write-Host "$e[1A" -NoNewline } else { [Console]::SetCursorPosition(0, $top + $count) }
|
||||
Clear-Line
|
||||
} catch [System.Management.Automation.PipelineStoppedException] {
|
||||
throw
|
||||
} catch {
|
||||
# No cursor control here: fall back to a numbered list.
|
||||
for ($i = 1; $i -le $count; $i++) { Write-Host (" $i) " + $Options[$i - 1][0]) }
|
||||
$answer = Read-Host "Choose [$Default]"
|
||||
if ($answer -match '^[1-9]$' -and [int]$answer -le $count) { $choice = [int]$answer }
|
||||
} finally {
|
||||
Show-Cursor $true
|
||||
}
|
||||
return $choice
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------ steps
|
||||
|
||||
function Test-PortFree([int]$Port) {
|
||||
# Binding answers right away and is accurate. Probing with a connect is
|
||||
# not: the WSL2 localhost relay (Podman, Rancher Desktop) swallows the
|
||||
# refusal on closed ports, so every connect waits out its timeout and
|
||||
# closed ports look taken.
|
||||
$listener = New-Object System.Net.Sockets.TcpListener([Net.IPAddress]::Loopback, $Port)
|
||||
try {
|
||||
$listener.Start()
|
||||
return $true
|
||||
} catch {
|
||||
return $false
|
||||
} finally {
|
||||
$listener.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
function Test-Ready([int]$Port) {
|
||||
try {
|
||||
$r = Invoke-WebRequest -UseBasicParsing -TimeoutSec 2 -Uri "http://127.0.0.1:$Port/health/readiness"
|
||||
return $r.StatusCode -eq 200
|
||||
} catch { return $false }
|
||||
}
|
||||
|
||||
# Run a program with a spinner and the elapsed time, keeping its output to
|
||||
# show if it fails. Without styling it runs in the open.
|
||||
function Invoke-WithSpinner {
|
||||
param([string]$Label, [string]$Detail, [string]$File, [string[]]$Arguments)
|
||||
if (-not $style) {
|
||||
$prev = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
try { & $File @Arguments 2>&1 | ForEach-Object { [Console]::Out.WriteLine("$_") }; $code = $LASTEXITCODE }
|
||||
finally { $ErrorActionPreference = $prev }
|
||||
return $code
|
||||
}
|
||||
$out = [IO.Path]::GetTempFileName(); $err = [IO.Path]::GetTempFileName()
|
||||
try {
|
||||
$p = Start-Process -FilePath $File -ArgumentList $Arguments -NoNewWindow -PassThru -RedirectStandardOutput $out -RedirectStandardError $err
|
||||
$null = $p.Handle # keeps ExitCode available after exit in Windows PowerShell
|
||||
$start = Get-Date
|
||||
$n = 0
|
||||
Show-Cursor $false
|
||||
try {
|
||||
while (-not $p.HasExited) {
|
||||
$s = [int]((Get-Date) - $start).TotalSeconds
|
||||
$clock = '{0}:{1:00}' -f [math]::Floor($s / 60), ($s % 60)
|
||||
$extra = if ($Detail) { "$Detail $($sym.Dot) " } else { '' }
|
||||
Clear-Line
|
||||
Say ("{a:$($sym.Frames[$n % $sym.Frames.Count])} $Label {d:$($sym.Dot) $extra$clock}") -NoNewline
|
||||
$n++
|
||||
Start-Sleep -Milliseconds 100
|
||||
}
|
||||
$p.WaitForExit()
|
||||
Clear-Line
|
||||
} finally { Show-Cursor $true }
|
||||
$code = $p.ExitCode
|
||||
if ($code -ne 0) {
|
||||
Get-Content $out, $err -ErrorAction SilentlyContinue | ForEach-Object { [Console]::Error.WriteLine($_) }
|
||||
}
|
||||
return $code
|
||||
} finally {
|
||||
# Also on Ctrl+C, so a cancelled run leaves no temporary files.
|
||||
Remove-Item -LiteralPath $out, $err -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
|
||||
function Wait-Ready([int]$Port) {
|
||||
# Up to 3 minutes, like the shell version. A spinner when styled.
|
||||
$start = Get-Date
|
||||
$n = 0
|
||||
$nextCheck = Get-Date
|
||||
if ($style) { Show-Cursor $false }
|
||||
try {
|
||||
while (((Get-Date) - $start).TotalSeconds -lt 180) {
|
||||
if ((Get-Date) -ge $nextCheck) {
|
||||
if (Test-Ready $Port) { if ($style) { Clear-Line }; return $true }
|
||||
$nextCheck = (Get-Date).AddSeconds(2)
|
||||
}
|
||||
if ($style) {
|
||||
$s = [int]((Get-Date) - $start).TotalSeconds
|
||||
Clear-Line
|
||||
Say ("{a:$($sym.Frames[$n % $sym.Frames.Count])} Waiting for the gateway to be ready {d:$($sym.Dot) $('{0}:{1:00}' -f [math]::Floor($s / 60), ($s % 60))}") -NoNewline
|
||||
$n++
|
||||
}
|
||||
Start-Sleep -Milliseconds 100
|
||||
}
|
||||
if ($style) { Clear-Line }
|
||||
return $false
|
||||
} finally { if ($style) { Show-Cursor $true } }
|
||||
}
|
||||
|
||||
# The closing summary. Plain output, and a console too narrow for the frame,
|
||||
# get the same lines without it.
|
||||
function Open-Url([string]$Url) {
|
||||
try {
|
||||
if ($onWindows) { Start-Process $Url } elseif (Get-Command open -ErrorAction SilentlyContinue) { & open $Url } else { & xdg-open $Url }
|
||||
Step "Opened {a:$Url} in your browser"
|
||||
} catch {
|
||||
# No browser to open; print the address instead.
|
||||
Say " {d:Open} {a:$Url} {d:when you are ready.}"
|
||||
}
|
||||
}
|
||||
|
||||
# The admin password is the master key. It goes straight to the clipboard,
|
||||
# never to the screen.
|
||||
function Copy-Password([string]$Folder) {
|
||||
$pw = Get-Content -LiteralPath (Join-Path $Folder '.env') |
|
||||
Where-Object { $_.StartsWith('LITELLM_MASTER_KEY=') } | Select-Object -First 1
|
||||
if (-not $pw) { Warn ('No LITELLM_MASTER_KEY in ' + (Lit (Join-Path (Tildify $Folder) '.env')) + '.'); return }
|
||||
try {
|
||||
Set-Clipboard -Value $pw.Substring('LITELLM_MASTER_KEY='.Length) -ErrorAction Stop
|
||||
Step 'Copied the admin password. Paste it into the sign-in page.'
|
||||
} catch {
|
||||
# Another app can hold the Windows clipboard open.
|
||||
Warn ('Could not copy it. The password is LITELLM_MASTER_KEY in ' + (Lit (Join-Path (Tildify $Folder) '.env')) + '.')
|
||||
} finally {
|
||||
$pw = $null
|
||||
}
|
||||
}
|
||||
|
||||
function Show-Box([string]$Title, [object[]]$Rows) {
|
||||
$width = $Title.Length
|
||||
foreach ($r in $Rows) { if (11 + $r[1].Length -gt $width) { $width = 11 + $r[1].Length } }
|
||||
$cols = 0
|
||||
try { $cols = [Console]::WindowWidth } catch { <# no console: print without the frame #> }
|
||||
if (-not $style -or $cols -lt $width + 4) {
|
||||
if ($style) { Say "{u:$Title}" } else { Say "$Title." }
|
||||
foreach ($r in $Rows) { Say (' ' + ($r[0] + ':').PadRight(12) + (Lit $r[1])) }
|
||||
return
|
||||
}
|
||||
$rule = "$($sym.H)" * ($width + 2)
|
||||
Say "{a:$($sym.TL)$rule$($sym.TR)}"
|
||||
Say ("{a:$($sym.V)} {u:" + $Title.PadRight($width) + "} {a:$($sym.V)}")
|
||||
foreach ($r in $Rows) {
|
||||
$value = (Lit $r[1]).PadRight($width - 11)
|
||||
if ($r[0] -eq 'Admin UI') { $value = "{u:$value}" }
|
||||
Say ("{a:$($sym.V)} {d:" + $r[0].PadRight(10) + "} $value {a:$($sym.V)}")
|
||||
}
|
||||
Say "{a:$($sym.BL)$rule$($sym.BR)}"
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------ main
|
||||
|
||||
$savedEncoding = $null
|
||||
try {
|
||||
if ($unicode -and $onWindows) {
|
||||
# Windows PowerShell writes the console in the OEM code page, which has no check marks.
|
||||
$savedEncoding = [Console]::OutputEncoding
|
||||
[Console]::OutputEncoding = New-Object System.Text.UTF8Encoding $false
|
||||
}
|
||||
if ($PSVersionTable.PSEdition -eq 'Desktop') {
|
||||
# Windows PowerShell can default to TLS 1.0, which GitHub refuses.
|
||||
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
|
||||
}
|
||||
|
||||
if ($style) {
|
||||
Write-Host ''
|
||||
Say "{u:$($sym.Head) LiteLLM quickstart}"
|
||||
Say '{d: The gateway, Postgres, and the admin UI in one command}'
|
||||
Write-Host ''
|
||||
} else {
|
||||
Say 'LiteLLM quickstart'
|
||||
}
|
||||
|
||||
# Podman and Rancher Desktop (nerdctl in containerd mode; its dockerd mode
|
||||
# already provides a docker CLI) work the same way through Compose v2.
|
||||
$engineCmd = foreach ($cand in 'docker', 'podman', 'nerdctl') {
|
||||
$c = Get-Command $cand -CommandType Application -ErrorAction SilentlyContinue | Select-Object -First 1
|
||||
if ($c) { $c; break }
|
||||
}
|
||||
if (-not $engineCmd) {
|
||||
Fail 'No container engine found (docker, podman, or nerdctl).' 'The LiteLLM Gateway runs in a container alongside a Postgres database.'
|
||||
$install = if ($onWindows) { 'https://docs.docker.com/desktop/setup/install/windows-install/' } else { 'https://docs.docker.com/get-docker/' }
|
||||
Say '' -Err
|
||||
Say " Install Docker Desktop ($install)," -Err
|
||||
Say ' Podman (https://podman.io), or Rancher Desktop (https://rancherdesktop.io), then run this again.' -Err
|
||||
Say ' Or deploy in one click (Railway or Render): https://docs.litellm.ai/docs/proxy/docker_quick_start' -Err
|
||||
Say ' Only need to call models from Python? pip install litellm' -Err
|
||||
return 1
|
||||
}
|
||||
$docker = $engineCmd.Source
|
||||
$engine = $engineCmd.Name -replace '\.exe$', ''
|
||||
$engineName = switch ($engine) { 'podman' { 'Podman' } 'nerdctl' { 'nerdctl' } default { 'Docker' } }
|
||||
$compose = Invoke-Native $docker @('compose', 'version', '--short')
|
||||
if ($compose.Code -ne 0) { Fail "Compose v2 ('$engine compose') is required."; return 1 }
|
||||
if ((Invoke-Native $docker @('info')).Code -ne 0) {
|
||||
$startHint = switch ($engine) {
|
||||
'podman' { 'Start it (podman machine start) and run this again.' }
|
||||
'nerdctl' { 'Start Rancher Desktop and run this again.' }
|
||||
default { 'Start Docker Desktop and run this again.' }
|
||||
}
|
||||
Fail "$engineName is installed but not running." $startHint
|
||||
return 1
|
||||
}
|
||||
$server = Invoke-Native $docker @('version', '--format', '{{.Server.Version}}')
|
||||
$engineVersion = if ($server.Code -eq 0 -and $server.Out.Count) { "$($server.Out[0]) " } else { '' }
|
||||
$composeVersion = "$($compose.Out[0])".TrimStart('v')
|
||||
Step "$engineName ${engineVersion}with Compose $composeVersion is running"
|
||||
|
||||
$homeDir = Join-Path $HOME 'litellm-gateway'
|
||||
$hereDir = Join-Path (Get-Location).ProviderPath 'litellm-gateway'
|
||||
$found = $false
|
||||
if ($env:LITELLM_DIR) {
|
||||
$dir = $env:LITELLM_DIR
|
||||
} elseif (Test-Path -LiteralPath (Join-Path $hereDir '.env')) {
|
||||
$dir = $hereDir; $found = $true # installed in this folder before
|
||||
} elseif (Test-Path -LiteralPath (Join-Path $homeDir '.env')) {
|
||||
$dir = $homeDir; $found = $true # installed in the home folder before
|
||||
} elseif ($hereDir -eq $homeDir) {
|
||||
$dir = $homeDir
|
||||
} else {
|
||||
$pick = Menu 'Where should LiteLLM keep its files (.env with your keys, and the compose file)?' 1 @(
|
||||
, @((Tildify $homeDir), 'recommended, reruns always find it')
|
||||
, @((Tildify $hereDir), 'this folder'))
|
||||
$dir = if ($pick -eq 2) { $hereDir } else { $homeDir }
|
||||
}
|
||||
$created = -not (Test-Path -LiteralPath $dir)
|
||||
$null = New-Item -ItemType Directory -Force -Path $dir
|
||||
$dir = (Resolve-Path -LiteralPath $dir).ProviderPath
|
||||
Set-Location -LiteralPath $dir
|
||||
if ($found) { Step ('Found your install in {b:' + (Lit (Tildify $dir)) + '}') } else { Step ('Files go in {b:' + (Lit (Tildify $dir)) + '}') }
|
||||
|
||||
$git = Get-Command git -CommandType Application -ErrorAction SilentlyContinue | Select-Object -First 1
|
||||
$inRepo = $git -and (Invoke-Native $git.Source @('rev-parse', '--is-inside-work-tree')).Code -eq 0
|
||||
if ($inRepo -and -not (Test-Path -LiteralPath (Join-Path $dir '.env')) -and
|
||||
(Invoke-Native $git.Source @('ls-files', '--error-unmatch', '.env')).Code -eq 0) {
|
||||
# An ignore rule does not cover a tracked file, so new keys written here
|
||||
# would show up as a change to commit.
|
||||
Fail 'Git tracks a .env file in this folder,' 'so your keys could be committed.'
|
||||
Say '' -Err
|
||||
Say ' Install into another folder (set LITELLM_DIR), or stop tracking the file' -Err
|
||||
Say ' first with: git rm --cached .env' -Err
|
||||
return 1
|
||||
}
|
||||
if ($created) {
|
||||
# A folder this script made holds only its own files, so keep all of it out of git.
|
||||
Write-PlainText (Join-Path $dir '.gitignore') "*`n"
|
||||
} elseif ($inRepo -and (Invoke-Native $git.Source @('check-ignore', '-q', '.env')).Code -ne 0) {
|
||||
# In a folder that already existed, such as a repository root, leave the
|
||||
# tracked .gitignore alone and add only .env to this clone's local exclude
|
||||
# list, so the generated keys cannot be committed.
|
||||
$exclude = "$((Invoke-Native $git.Source @('rev-parse', '--git-path', 'info/exclude')).Out[0])"
|
||||
$exclude = [IO.Path]::GetFullPath([IO.Path]::Combine($dir, $exclude))
|
||||
$null = New-Item -ItemType Directory -Force -Path (Split-Path $exclude)
|
||||
$prefix = "$((Invoke-Native $git.Source @('rev-parse', '--show-prefix')).Out | Select-Object -First 1)"
|
||||
Write-PlainText $exclude "/$prefix.env`n" -Append
|
||||
Step ("Kept .env out of git {d:(added it to this repository's local exclude list, " + (Lit (Tildify $exclude)) + ')}')
|
||||
} elseif (-not $inRepo) {
|
||||
# An existing folder outside git: ignore only .env, so it stays out of
|
||||
# commits if the folder becomes a repository later.
|
||||
$ignore = Join-Path $dir '.gitignore'
|
||||
$current = if (Test-Path -LiteralPath $ignore) { [IO.File]::ReadAllText($ignore) } else { '' }
|
||||
if (($current -split "`r?`n") -notcontains '.env') {
|
||||
# Start on a new line if the file does not end with one.
|
||||
$lead = if ($current.Length -gt 0 -and -not $current.EndsWith("`n")) { "`n" } else { '' }
|
||||
Write-PlainText $ignore "$lead.env`n" -Append
|
||||
}
|
||||
}
|
||||
|
||||
# The engine names containers and the database volume after the project, so
|
||||
# an install outside the home folder gets its own name and never shares a
|
||||
# database with another litellm-gateway folder. Windows paths ignore case,
|
||||
# so there the name does too.
|
||||
$envFile = Join-Path $dir '.env'
|
||||
$project = 'litellm-gateway'
|
||||
if ($dir -ne $homeDir) {
|
||||
$key = if ($onWindows) { $dir.ToLowerInvariant() } else { $dir }
|
||||
$project = "litellm-gateway-$(Get-Cksum $key)"
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $envFile)) {
|
||||
# Postgres keeps the password it was created with, so a new password over an
|
||||
# old database volume would lock the gateway out. Stop and explain instead.
|
||||
if ((Invoke-Native $docker @('volume', 'inspect', "${project}_postgres_data")).Code -eq 0) {
|
||||
Fail 'Found a database from an earlier install' "($engineName volume ${project}_postgres_data)"
|
||||
Say ('but no ' + (Lit (Join-Path (Tildify $dir) '.env')) + ' with its password.') -Err
|
||||
Say '' -Err
|
||||
Say ' Restore that .env file and run this again to keep your models and keys, or' -Err
|
||||
Say ' delete the old database and start fresh (this removes its models and keys):' -Err
|
||||
Say " $engine volume rm ${project}_postgres_data" -Err
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
Invoke-WebRequest -UseBasicParsing -Uri $composeUrl -OutFile (Join-Path $dir 'docker-compose.quickstart.yml')
|
||||
Step 'Downloaded docker-compose.quickstart.yml'
|
||||
|
||||
$saved = ''
|
||||
if (Test-Path -LiteralPath $envFile) {
|
||||
$m = Get-Content -LiteralPath $envFile | Where-Object { $_ -match '^LITELLM_PORT=(.*)$' } | Select-Object -Last 1
|
||||
if ($m) { $saved = ($m -split '=', 2)[1] }
|
||||
}
|
||||
if ($env:LITELLM_PORT) {
|
||||
$port = [int]$env:LITELLM_PORT
|
||||
} elseif ($saved) {
|
||||
$port = [int]$saved
|
||||
} elseif (Test-Path -LiteralPath $envFile) {
|
||||
$port = 4000 # an existing install without a saved port runs on the compose default
|
||||
} else {
|
||||
$port = 4000
|
||||
while (-not (Test-PortFree $port)) {
|
||||
$port++
|
||||
if ($port -gt 4099) {
|
||||
Fail 'Ports 4000 to 4099 are all in use.' 'Set LITELLM_PORT to a free port and run this again.'
|
||||
return 1
|
||||
}
|
||||
}
|
||||
if ($port -eq 4000) { Step 'Port {b:4000} is free' } else { Warn "Port 4000 is in use, so LiteLLM will use {b:$port}" }
|
||||
}
|
||||
|
||||
if (Test-Path -LiteralPath $envFile) {
|
||||
Step 'Reusing .env, so existing keys and data keep working'
|
||||
} else {
|
||||
$rng = [Security.Cryptography.RandomNumberGenerator]::Create()
|
||||
$hex = {
|
||||
param([int]$n)
|
||||
$bytes = New-Object byte[] $n
|
||||
$rng.GetBytes($bytes)
|
||||
-join ($bytes | ForEach-Object { $_.ToString('x2') })
|
||||
}
|
||||
# Lock a temporary file down before anything secret goes into it, then
|
||||
# rename it, so a failed write never leaves a partial .env that a rerun
|
||||
# would mistake for a finished install.
|
||||
$tmp = "$envFile.tmp"
|
||||
try {
|
||||
Write-PlainText $tmp ''
|
||||
if ($onWindows) {
|
||||
$acl = New-Object System.Security.AccessControl.FileSecurity
|
||||
$acl.SetAccessRuleProtection($true, $false)
|
||||
$me = [Security.Principal.WindowsIdentity]::GetCurrent().User
|
||||
$acl.AddAccessRule((New-Object System.Security.AccessControl.FileSystemAccessRule($me, 'FullControl', 'Allow')))
|
||||
Set-Acl -LiteralPath $tmp -AclObject $acl
|
||||
} else {
|
||||
& chmod 600 $tmp
|
||||
if ($LASTEXITCODE -ne 0) { throw "Could not restrict the permissions of $tmp." }
|
||||
}
|
||||
Write-PlainText $tmp (("LITELLM_MASTER_KEY=sk-{0}`nLITELLM_SALT_KEY=sk-{1}`nPOSTGRES_PASSWORD={2}`n" +
|
||||
"LITELLM_PORT={3}`nLITELLM_BIND=127.0.0.1:`nCOMPOSE_PROJECT_NAME={4}`n") -f (& $hex 32), (& $hex 32), (& $hex 24), $port, $project)
|
||||
[IO.File]::Move($tmp, $envFile)
|
||||
} finally {
|
||||
Remove-Item -LiteralPath $tmp -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
Step 'Generated .env with your master key, salt key, and database password {d:(keep this file; only you can read it)}'
|
||||
}
|
||||
|
||||
# Compose prefers values already set in the environment over .env, so drop
|
||||
# inherited ones: .env stays the only source for keys and the project name.
|
||||
foreach ($name in 'LITELLM_MASTER_KEY', 'LITELLM_SALT_KEY', 'POSTGRES_PASSWORD', 'COMPOSE_PROJECT_NAME') {
|
||||
Remove-Item "Env:$name" -ErrorAction SilentlyContinue
|
||||
}
|
||||
# The bind address follows .env when .env sets it. For an older .env without
|
||||
# it, a value already in the environment is kept.
|
||||
if (Get-Content -LiteralPath $envFile | Where-Object { $_ -match '^LITELLM_BIND=' }) { Remove-Item Env:LITELLM_BIND -ErrorAction SilentlyContinue }
|
||||
$env:LITELLM_PORT = "$port"
|
||||
|
||||
$started = Get-Date
|
||||
$detail = ''
|
||||
foreach ($image in (Invoke-Native $docker @('compose', '-f', 'docker-compose.quickstart.yml', 'config', '--images')).Out) {
|
||||
if ($image -and (Invoke-Native $docker @('image', 'inspect', "$image")).Code -ne 0) { $detail = 'downloading images, first run only' }
|
||||
}
|
||||
if (-not $style) { Say 'Starting LiteLLM and Postgres (the first run downloads the images)...' }
|
||||
$code = Invoke-WithSpinner 'Starting LiteLLM and Postgres' $detail $docker @('compose', '-f', 'docker-compose.quickstart.yml', 'up', '-d')
|
||||
if ($code -ne 0) {
|
||||
Fail "$engineName could not start LiteLLM and Postgres." 'Its output is above.'
|
||||
return 1
|
||||
}
|
||||
if (-not (Wait-Ready $port)) {
|
||||
Fail 'The gateway did not become ready in 3 minutes.' 'See what it logged:'
|
||||
Say (' cd ' + (Lit (Tildify $dir)) + "; $engine compose -f docker-compose.quickstart.yml logs litellm") -Err
|
||||
return 1
|
||||
}
|
||||
Step ("LiteLLM and Postgres are up {d:$($sym.Dot) $(Elapsed $started)}")
|
||||
|
||||
$url = "http://localhost:$port/ui"
|
||||
$where = Tildify $dir
|
||||
if ($where -match ' ') { $where = "`"$where`"" }
|
||||
Write-Host ''
|
||||
# Over SSH the clipboard would be the server's, not yours. On Linux,
|
||||
# Set-Clipboard without xclip quietly keeps the text inside PowerShell, so
|
||||
# only Windows and macOS, which always have a clipboard, get the option.
|
||||
$canCopy = $interactive -and (-not $env:SSH_CONNECTION) -and
|
||||
($onWindows -or ((Test-Path variable:IsMacOS) -and $IsMacOS))
|
||||
$password = "the LITELLM_MASTER_KEY value in $(Join-Path (Tildify $dir) '.env')"
|
||||
if ($canCopy) { $password = "copy it below, or LITELLM_MASTER_KEY in $(Join-Path (Tildify $dir) '.env')" }
|
||||
Show-Box 'LiteLLM is running' @(
|
||||
, @('Admin UI', $url)
|
||||
, @('Username', 'admin')
|
||||
, @('Password', $password)
|
||||
, @('Next', 'in the UI, open Models + Endpoints > Add Model and paste a provider API key')
|
||||
, @('Stop it', "cd $where; $engine compose -f docker-compose.quickstart.yml down"))
|
||||
|
||||
if ($canCopy) {
|
||||
# Come back to the menu after each choice, with the next step as the default.
|
||||
$pick = 1
|
||||
while ($true) {
|
||||
$pick = Menu 'What next?' $pick @(
|
||||
, @('Copy the admin password', 'to the clipboard, ready to paste')
|
||||
, @('Open the admin UI in your browser')
|
||||
, @('Done'))
|
||||
if ($pick -eq 1) { Copy-Password $dir; $pick = 2 }
|
||||
elseif ($pick -eq 2) { Open-Url $url; $pick = 3 }
|
||||
else { break }
|
||||
}
|
||||
} elseif ($interactive) {
|
||||
if ((Menu 'Open the admin UI in your browser?' 1 @(, @('Yes'), @('No'))) -eq 1) { Open-Url $url }
|
||||
}
|
||||
return 0
|
||||
} finally {
|
||||
if ($savedEncoding) { [Console]::OutputEncoding = $savedEncoding }
|
||||
}
|
||||
}
|
||||
|
||||
$code = Invoke-LiteLLMQuickstart -Yes:$Yes | Select-Object -Last 1
|
||||
$global:LASTEXITCODE = $code
|
||||
# Run from a file, or as `powershell -c "irm ... | iex"`, the exit code should
|
||||
# reach the caller. Pasted into an open window, `exit` would close that window,
|
||||
# so there it only sets $LASTEXITCODE.
|
||||
$hostArgs = [Environment]::GetCommandLineArgs()
|
||||
$oneShot = ($hostArgs -match '^-(c|command)$') -and -not ($hostArgs -match '^-noexit$')
|
||||
if ($code -ne 0 -and ($PSCommandPath -or $oneShot)) { exit $code }
|
||||
|
|
@ -1,36 +1,183 @@
|
|||
#!/bin/sh
|
||||
# LiteLLM Gateway quickstart: the gateway, Postgres, and the admin UI in one command.
|
||||
# curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/main/scripts/quickstart.sh | sh
|
||||
# On Windows, scripts/quickstart.ps1 does the same in PowerShell.
|
||||
#
|
||||
# To read it before running it:
|
||||
# curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/main/scripts/quickstart.sh -o quickstart.sh
|
||||
# less quickstart.sh
|
||||
# sh quickstart.sh
|
||||
#
|
||||
# Asks at most two questions (where to keep the files, and whether to open the
|
||||
# admin UI), each with a default you accept by pressing Enter. It asks nothing
|
||||
# when there is no terminal, under CI or Claude Code, or when run with --yes.
|
||||
# Asks where to keep the files, then offers to copy the admin password and
|
||||
# open the admin UI; every question has a default you accept by pressing Enter.
|
||||
# It asks nothing when there is no terminal, under CI or Claude Code, or when
|
||||
# run with --yes.
|
||||
#
|
||||
# --yes, -y no questions: install to ~/litellm-gateway, don't open a browser
|
||||
# LITELLM_DIR folder to install into (skips the folder question)
|
||||
# LITELLM_PORT port for the gateway (default 4000, or the next free one)
|
||||
# NO_COLOR plain output, which agents, CI, and log files always get
|
||||
#
|
||||
# New installs listen on this machine only (127.0.0.1). To reach the gateway
|
||||
# from other machines, remove LITELLM_BIND from .env and put it behind TLS.
|
||||
#
|
||||
# Keys and the database password are random (openssl rand), written only to
|
||||
# .env with permissions 600, and never printed. Needs Docker with Compose v2.
|
||||
# .env with permissions 600, and never printed. When you ask, the admin
|
||||
# password goes straight to your clipboard. Needs Docker, Podman, or Rancher
|
||||
# Desktop, each with Compose v2.
|
||||
# Everything runs inside main(), so a partial download runs nothing.
|
||||
set -eu
|
||||
|
||||
COMPOSE_URL="${LITELLM_COMPOSE_URL:-https://raw.githubusercontent.com/BerriAI/litellm/main/docker/docker-compose.quickstart.yml}"
|
||||
|
||||
# ---------------------------------------------------------------- output
|
||||
|
||||
# A person watching a terminal gets colors, step marks, and a spinner. Agents,
|
||||
# CI, log files, and NO_COLOR get the same lines as plain text.
|
||||
STYLE=0
|
||||
ERR_STYLE=0 # stderr is styled only when it is a terminal too
|
||||
C_ACC='' C_OK='' C_WARN='' C_ERR='' C_DIM='' C_BOLD='' C_OFF=''
|
||||
POINTER='>' S_OK='+' S_WARN='!' S_ERR='x' S_HEAD='*' S_ASK='?'
|
||||
# shellcheck disable=SC1003 # the last frame is a backslash
|
||||
SPIN_FRAMES='| / - \'
|
||||
HINT='Up/Down to move, Enter to choose'
|
||||
BOX_TL='+' BOX_TR='+' BOX_BL='+' BOX_BR='+' BOX_H='-' BOX_V='|'
|
||||
SPIN_PID='' SPIN_LOG='' ENV_TMP=''
|
||||
|
||||
setup_output() {
|
||||
case "${LC_ALL:-${LC_CTYPE:-${LANG:-}}}" in
|
||||
*UTF-8* | *utf-8* | *UTF8* | *utf8*)
|
||||
POINTER='❯' S_OK='✓' S_WARN='!' S_ERR='✗' S_HEAD='◆' S_ASK='?'
|
||||
SPIN_FRAMES='⠋ ⠙ ⠹ ⠸ ⠼ ⠴ ⠦ ⠧ ⠇ ⠏'
|
||||
HINT='↑/↓ to move, Enter to choose'
|
||||
BOX_TL='╭' BOX_TR='╮' BOX_BL='╰' BOX_BR='╯' BOX_H='─' BOX_V='│'
|
||||
;;
|
||||
esac
|
||||
if [ -t 1 ] && [ -z "${NO_COLOR:-}" ] && [ "${TERM:-dumb}" != "dumb" ] &&
|
||||
[ -z "${CI:-}" ] && [ -z "${CLAUDECODE:-}" ]; then
|
||||
STYLE=1
|
||||
e="$(printf '\033')"
|
||||
# The accent is the LiteLLM blue, lightened so it reads on dark and light themes.
|
||||
case "${COLORTERM:-}" in
|
||||
truecolor | 24bit) C_ACC="${e}[38;2;91;108;255m" ;;
|
||||
*) C_ACC="${e}[94m" ;;
|
||||
esac
|
||||
C_OK="${e}[32m" C_WARN="${e}[33m" C_ERR="${e}[31m" C_DIM="${e}[90m" C_BOLD="${e}[1m" C_OFF="${e}[0m"
|
||||
if [ -t 2 ]; then ERR_STYLE=1; fi
|
||||
fi
|
||||
}
|
||||
|
||||
step() {
|
||||
if [ "$STYLE" = 1 ]; then
|
||||
printf '%s%s%s %s%s\n' "${C_OK}" "$S_OK" "${C_OFF}" "$1" "${C_OFF}"
|
||||
else
|
||||
printf '%s\n' "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
warn() {
|
||||
if [ "$STYLE" = 1 ]; then
|
||||
printf '%s%s%s %s%s\n' "${C_WARN}" "$S_WARN" "${C_OFF}" "$1" "${C_OFF}"
|
||||
else
|
||||
printf '%s\n' "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
fail() {
|
||||
if [ "$ERR_STYLE" = 1 ]; then
|
||||
printf '%s%s %s%s%s\n' "${C_ERR}${C_BOLD}" "$S_ERR" "$1" "${C_OFF}" "${2:+ $2}" >&2
|
||||
else
|
||||
printf '%s%s\n' "$1" "${2:+ $2}" >&2
|
||||
fi
|
||||
}
|
||||
|
||||
tildify() {
|
||||
case "$1" in
|
||||
"$HOME") printf '~' ;;
|
||||
"$HOME"/*) printf '~%s' "${1#"$HOME"}" ;;
|
||||
*) printf '%s' "$1" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
elapsed_since() {
|
||||
s=$(($(date +%s) - $1))
|
||||
if [ "$s" -lt 60 ]; then printf '%ss' "$s"; else printf '%sm %ss' "$((s / 60))" "$((s % 60))"; fi
|
||||
}
|
||||
|
||||
# spin "label" "detail" command...: run the command behind a spinner with the
|
||||
# elapsed time, keeping its output to show if it fails. Without styling the
|
||||
# command runs in the open, as before.
|
||||
spin() {
|
||||
label="$1" detail="$2"
|
||||
shift 2
|
||||
if [ "$STYLE" != 1 ]; then
|
||||
"$@"
|
||||
return
|
||||
fi
|
||||
SPIN_LOG="$(mktemp)"
|
||||
"$@" >"$SPIN_LOG" 2>&1 &
|
||||
SPIN_PID=$!
|
||||
start="$(date +%s)" s=0 n=0
|
||||
frames="$SPIN_FRAMES "
|
||||
printf '\033[?25l'
|
||||
while kill -0 "$SPIN_PID" 2>/dev/null; do
|
||||
# Rotate the frames in the shell and read the clock once a second, so a
|
||||
# redraw starts no process besides sleep.
|
||||
frame="${frames%% *}"
|
||||
frames="${frames#* }$frame "
|
||||
if [ $((n % 10)) = 0 ]; then s=$(($(date +%s) - start)); fi
|
||||
printf '\r\033[2K%s%s%s %s %s· %s%d:%02d%s' "${C_ACC}" "$frame" "${C_OFF}" "$label" "${C_DIM}" \
|
||||
"${detail:+$detail · }" "$((s / 60))" "$((s % 60))" "${C_OFF}"
|
||||
n=$((n + 1))
|
||||
sleep 0.1
|
||||
done
|
||||
rc=0
|
||||
wait "$SPIN_PID" || rc=$?
|
||||
SPIN_PID=''
|
||||
printf '\r\033[2K\033[?25h'
|
||||
if [ "$rc" != 0 ]; then cat "$SPIN_LOG" >&2; fi
|
||||
rm -f "$SPIN_LOG"
|
||||
SPIN_LOG=''
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
# box "Title" "Label|value"...: the closing summary. Plain output, and a
|
||||
# terminal too narrow for the frame, get the same lines without it.
|
||||
box() {
|
||||
title="$1"
|
||||
shift
|
||||
cols=''
|
||||
if [ "$STYLE" = 1 ]; then cols="$( (stty size </dev/tty) 2>/dev/null | cut -d ' ' -f 2)" || cols=''; fi
|
||||
width=${#title}
|
||||
for row in "$@"; do
|
||||
value="${row#*|}"
|
||||
if [ $((11 + ${#value})) -gt "$width" ]; then width=$((11 + ${#value})); fi
|
||||
done
|
||||
if [ "$STYLE" != 1 ] || [ -z "$cols" ] || [ $((width + 4)) -gt "$cols" ]; then
|
||||
if [ "$STYLE" = 1 ]; then printf '%s%s%s\n' "${C_ACC}${C_BOLD}" "$title" "${C_OFF}"; else printf '%s.\n' "$title"; fi
|
||||
for row in "$@"; do
|
||||
printf ' %-12s%s\n' "${row%%|*}:" "${row#*|}"
|
||||
done
|
||||
return 0
|
||||
fi
|
||||
rule="$(printf "%$((width + 2))s" '' | sed "s/ /$BOX_H/g")"
|
||||
printf '%s%s%s%s%s\n' "${C_ACC}" "$BOX_TL" "$rule" "$BOX_TR" "${C_OFF}"
|
||||
printf '%s%s%s %s%-*s%s %s%s%s\n' "${C_ACC}" "$BOX_V" "${C_OFF}" "${C_ACC}${C_BOLD}" "$width" "$title" "${C_OFF}" \
|
||||
"${C_ACC}" "$BOX_V" "${C_OFF}"
|
||||
for row in "$@"; do
|
||||
label="${row%%|*}" value="${row#*|}" shade=''
|
||||
if [ "$label" = "Admin UI" ]; then shade="${C_ACC}${C_BOLD}"; fi
|
||||
printf '%s%s%s %s%-10s%s %s%-*s%s %s%s%s\n' "${C_ACC}" "$BOX_V" "${C_OFF}" "${C_DIM}" "$label" "${C_OFF}" \
|
||||
"$shade" "$((width - 11))" "$value" "${C_OFF}" "${C_ACC}" "$BOX_V" "${C_OFF}"
|
||||
done
|
||||
printf '%s%s%s%s%s\n' "${C_ACC}" "$BOX_BL" "$rule" "$BOX_BR" "${C_OFF}"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- terminal
|
||||
|
||||
INTERACTIVE=0 # a person is at a terminal we can ask
|
||||
ARROWS=0 # that terminal supports the arrow-key menu
|
||||
STTY_SAVED=""
|
||||
POINTER='>'
|
||||
|
||||
detect_terminal() {
|
||||
# Piped from curl, stdin is the script itself, so questions go to /dev/tty.
|
||||
|
|
@ -40,19 +187,22 @@ detect_terminal() {
|
|||
ARROWS=1
|
||||
fi
|
||||
fi
|
||||
case "${LC_ALL:-${LC_CTYPE:-${LANG:-}}}" in
|
||||
*UTF-8* | *utf-8* | *UTF8* | *utf8*) POINTER='❯' ;;
|
||||
esac
|
||||
}
|
||||
|
||||
restore_terminal() {
|
||||
if [ -n "$STTY_SAVED" ]; then
|
||||
stty "$STTY_SAVED" </dev/tty 2>/dev/null || true
|
||||
fi
|
||||
if [ "$STYLE" = 1 ] || [ -n "$STTY_SAVED" ]; then
|
||||
printf '\033[?25h' >/dev/tty 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
on_interrupt() {
|
||||
if [ -n "$SPIN_PID" ]; then kill "$SPIN_PID" 2>/dev/null || true; fi
|
||||
if [ -n "$SPIN_LOG" ]; then rm -f "$SPIN_LOG"; fi
|
||||
if [ -n "$ENV_TMP" ]; then rm -f "$ENV_TMP"; fi
|
||||
if [ "$STYLE" = 1 ]; then printf '\r\033[2K'; fi
|
||||
restore_terminal
|
||||
printf '\nCancelled.\n' >&2
|
||||
exit 130
|
||||
|
|
@ -74,6 +224,7 @@ read_key() {
|
|||
}
|
||||
|
||||
# menu "Question" DEFAULT OPTION... -> sets CHOICE to the 1-based pick.
|
||||
# An option may carry a note after a tab, shown dimmed and lined up.
|
||||
menu() {
|
||||
question="$1"
|
||||
CHOICE="$2"
|
||||
|
|
@ -81,24 +232,38 @@ menu() {
|
|||
count=$#
|
||||
if [ "$INTERACTIVE" != 1 ]; then return 0; fi
|
||||
|
||||
printf '\n%s\n' "$question" >/dev/tty
|
||||
tab="$(printf '\t')"
|
||||
pad=0
|
||||
for opt in "$@"; do
|
||||
label="${opt%%"$tab"*}"
|
||||
if [ ${#label} -gt "$pad" ]; then pad=${#label}; fi
|
||||
done
|
||||
|
||||
if [ "$STYLE" = 1 ]; then
|
||||
printf '\n%s%s%s %s%s%s\n' "${C_ACC}" "$S_ASK" "${C_OFF}" "${C_BOLD}" "$question" "${C_OFF}" >/dev/tty
|
||||
else
|
||||
printf '\n%s\n' "$question" >/dev/tty
|
||||
fi
|
||||
if [ "$ARROWS" = 1 ]; then
|
||||
trap on_interrupt INT TERM
|
||||
stty -icanon -echo min 1 time 0 </dev/tty
|
||||
printf '\033[?25l' >/dev/tty
|
||||
first=1
|
||||
while :; do
|
||||
[ "$first" = 1 ] || printf '\033[%sA' "$count" >/dev/tty
|
||||
[ "$first" = 1 ] || printf '\033[%sA' "$((count + 1))" >/dev/tty
|
||||
first=0
|
||||
i=1
|
||||
for opt in "$@"; do
|
||||
label="${opt%%"$tab"*}" note=''
|
||||
[ "$label" = "$opt" ] || note="${opt#*"$tab"}"
|
||||
if [ "$i" = "$CHOICE" ]; then
|
||||
printf '\033[2K \033[1;36m%s %s\033[0m\n' "$POINTER" "$opt" >/dev/tty
|
||||
printf '\033[2K %s%s %-*s%s %s%s%s\n' "${C_ACC}${C_BOLD}" "$POINTER" "$pad" "$label" "${C_OFF}" \
|
||||
"${C_DIM}" "$note" "${C_OFF}" >/dev/tty
|
||||
else
|
||||
printf '\033[2K %s\n' "$opt" >/dev/tty
|
||||
printf '\033[2K %-*s %s%s%s\n' "$pad" "$label" "${C_DIM}" "$note" "${C_OFF}" >/dev/tty
|
||||
fi
|
||||
i=$((i + 1))
|
||||
done
|
||||
printf '\033[2K %s%s%s\n' "${C_DIM}" "$HINT" "${C_OFF}" >/dev/tty
|
||||
key="$(read_key)"
|
||||
case "$key" in
|
||||
up | k) [ "$CHOICE" -gt 1 ] && CHOICE=$((CHOICE - 1)) ;;
|
||||
|
|
@ -107,12 +272,15 @@ menu() {
|
|||
'' | "$(printf '\r')") break ;;
|
||||
esac
|
||||
done
|
||||
restore_terminal
|
||||
trap - INT TERM
|
||||
printf '\033[1A\033[2K' >/dev/tty
|
||||
stty "$STTY_SAVED" </dev/tty 2>/dev/null || true
|
||||
printf '\033[?25h' >/dev/tty
|
||||
else
|
||||
i=1
|
||||
for opt in "$@"; do
|
||||
printf ' %s) %s\n' "$i" "$opt" >/dev/tty
|
||||
label="${opt%%"$tab"*}" note=''
|
||||
[ "$label" = "$opt" ] || note=" ${opt#*"$tab"}"
|
||||
printf ' %s) %s%s\n' "$i" "$label" "$note" >/dev/tty
|
||||
i=$((i + 1))
|
||||
done
|
||||
printf 'Choose [%s]: ' "$CHOICE" >/dev/tty
|
||||
|
|
@ -138,18 +306,20 @@ port_free() {
|
|||
pick_folder() {
|
||||
home_dir="$HOME/litellm-gateway"
|
||||
here_dir="$(pwd)/litellm-gateway"
|
||||
found=0
|
||||
if [ -n "${LITELLM_DIR:-}" ]; then
|
||||
DIR="$LITELLM_DIR"
|
||||
elif [ -f "$here_dir/.env" ]; then
|
||||
DIR="$here_dir" # installed in this folder before
|
||||
DIR="$here_dir" found=1 # installed in this folder before
|
||||
elif [ -f "$home_dir/.env" ]; then
|
||||
DIR="$home_dir" # installed in the home folder before
|
||||
DIR="$home_dir" found=1 # installed in the home folder before
|
||||
elif [ "$here_dir" = "$home_dir" ]; then
|
||||
DIR="$home_dir"
|
||||
else
|
||||
tab="$(printf '\t')"
|
||||
menu "Where should LiteLLM keep its files (.env with your keys, and the compose file)?" 1 \
|
||||
"$home_dir recommended, reruns always find it" \
|
||||
"$here_dir this folder"
|
||||
"$(tildify "$home_dir")${tab}recommended, reruns always find it" \
|
||||
"$(tildify "$here_dir")${tab}this folder"
|
||||
if [ "$CHOICE" = 2 ]; then DIR="$here_dir"; else DIR="$home_dir"; fi
|
||||
fi
|
||||
created=0
|
||||
|
|
@ -157,6 +327,23 @@ pick_folder() {
|
|||
mkdir -p "$DIR"
|
||||
cd "$DIR"
|
||||
DIR="$(pwd)"
|
||||
if [ "$found" = 1 ]; then
|
||||
step "Found your install in ${C_BOLD}$(tildify "$DIR")${C_OFF}"
|
||||
else
|
||||
step "Files go in ${C_BOLD}$(tildify "$DIR")${C_OFF}"
|
||||
fi
|
||||
if [ ! -f .env ] && command -v git >/dev/null 2>&1 &&
|
||||
git ls-files --error-unmatch .env >/dev/null 2>&1; then
|
||||
# An ignore rule does not cover a tracked file, so new keys written here
|
||||
# would show up as a change to commit.
|
||||
fail "Git tracks a .env file in this folder," "so your keys could be committed."
|
||||
cat >&2 <<'EOF'
|
||||
|
||||
Install into another folder (set LITELLM_DIR), or stop tracking the file
|
||||
first with: git rm --cached .env
|
||||
EOF
|
||||
exit 1
|
||||
fi
|
||||
if [ "$created" = 1 ]; then
|
||||
# A folder this script made holds only its own files, so keep all of it out of git.
|
||||
printf '*\n' >.gitignore
|
||||
|
|
@ -169,7 +356,7 @@ pick_folder() {
|
|||
mkdir -p "$(dirname "$exclude")"
|
||||
exclude="$(cd "$(dirname "$exclude")" && pwd)/exclude"
|
||||
printf '/%s.env\n' "$(git rev-parse --show-prefix)" >>"$exclude"
|
||||
echo "Added .env to this repository's local git exclude list ($exclude), so your keys stay out of commits."
|
||||
step "Kept .env out of git ${C_DIM}(added it to this repository's local exclude list, $(tildify "$exclude"))"
|
||||
elif ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
|
||||
# An existing folder outside git: ignore only .env, so it stays out of
|
||||
# commits if the folder becomes a repository later.
|
||||
|
|
@ -196,11 +383,15 @@ pick_port() {
|
|||
while ! port_free "$PORT"; do
|
||||
PORT=$((PORT + 1))
|
||||
if [ "$PORT" -gt 4099 ]; then
|
||||
echo "Ports 4000 to 4099 are all in use. Set LITELLM_PORT to a free port and run this again." >&2
|
||||
fail "Ports 4000 to 4099 are all in use." "Set LITELLM_PORT to a free port and run this again."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
[ "$PORT" = 4000 ] || echo "Port 4000 is in use, so LiteLLM will use $PORT."
|
||||
if [ "$PORT" = 4000 ]; then
|
||||
step "Port ${C_BOLD}4000${C_OFF} is free"
|
||||
else
|
||||
warn "Port 4000 is in use, so LiteLLM will use ${C_BOLD}$PORT${C_OFF}"
|
||||
fi
|
||||
fi
|
||||
export LITELLM_PORT="$PORT"
|
||||
}
|
||||
|
|
@ -213,30 +404,96 @@ check_new_install() {
|
|||
[ "$DIR" = "$HOME/litellm-gateway" ] || project="litellm-gateway-$(printf '%s' "$DIR" | cksum | cut -d ' ' -f 1)"
|
||||
# Postgres keeps the password it was created with, so a new password over an
|
||||
# old database volume would lock the gateway out. Stop and explain instead.
|
||||
if docker volume inspect "${project}_postgres_data" >/dev/null 2>&1; then
|
||||
if "$ENGINE" volume inspect "${project}_postgres_data" >/dev/null 2>&1; then
|
||||
fail "Found a database from an earlier install" "($ENGINE_NAME volume ${project}_postgres_data)"
|
||||
cat >&2 <<EOF
|
||||
Found a database from an earlier install (Docker volume ${project}_postgres_data)
|
||||
but no $DIR/.env with its password.
|
||||
but no $(tildify "$DIR")/.env with its password.
|
||||
|
||||
Restore that .env file and run this again to keep your models and keys, or
|
||||
delete the old database and start fresh (this removes its models and keys):
|
||||
docker volume rm ${project}_postgres_data
|
||||
$ENGINE volume rm ${project}_postgres_data
|
||||
EOF
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
open_browser() {
|
||||
url="$1"
|
||||
menu "Open the admin UI in your browser?" 1 "Yes" "No"
|
||||
[ "$INTERACTIVE" = 1 ] && [ "$CHOICE" = 1 ] || return 0
|
||||
if command -v open >/dev/null 2>&1; then
|
||||
open "$url" >/dev/null 2>&1 || true
|
||||
elif command -v xdg-open >/dev/null 2>&1; then
|
||||
xdg-open "$url" >/dev/null 2>&1 || true
|
||||
start_stack() {
|
||||
"$ENGINE" compose -f docker-compose.quickstart.yml up -d
|
||||
}
|
||||
|
||||
wait_ready() {
|
||||
i=0
|
||||
until curl -fsS "http://127.0.0.1:$PORT/health/readiness" >/dev/null 2>&1; do
|
||||
i=$((i + 1))
|
||||
if [ "$i" -gt 90 ]; then return 1; fi
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
open_url() {
|
||||
for opener in open xdg-open; do
|
||||
if command -v "$opener" >/dev/null 2>&1 && "$opener" "$1" >/dev/null 2>&1; then
|
||||
step "Opened ${C_ACC}$1${C_OFF} in your browser"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
printf ' %sOpen%s %s%s%s %swhen you are ready.%s\n' "${C_DIM}" "${C_OFF}" "${C_ACC}" "$1" "${C_OFF}" "${C_DIM}" "${C_OFF}"
|
||||
}
|
||||
|
||||
# A clipboard to copy the admin password to, or nothing. Over SSH the
|
||||
# clipboard would be the server's, not yours.
|
||||
find_clipboard() {
|
||||
[ -z "${SSH_CONNECTION:-}${SSH_TTY:-}" ] || return 0
|
||||
if command -v pbcopy >/dev/null 2>&1; then CLIP=pbcopy
|
||||
elif [ -n "${WAYLAND_DISPLAY:-}" ] && command -v wl-copy >/dev/null 2>&1; then CLIP=wl-copy
|
||||
elif [ -n "${DISPLAY:-}" ] && command -v xclip >/dev/null 2>&1; then CLIP="xclip -selection clipboard"
|
||||
elif [ -n "${DISPLAY:-}" ] && command -v xsel >/dev/null 2>&1; then CLIP="xsel --clipboard --input"
|
||||
elif command -v clip.exe >/dev/null 2>&1; then CLIP=clip.exe
|
||||
fi
|
||||
}
|
||||
|
||||
# The admin password is the master key. It goes through a pipe, so it is
|
||||
# never on screen or in the process list.
|
||||
copy_password() {
|
||||
pw="$(sed -n 's/^LITELLM_MASTER_KEY=//p' .env | head -n 1)"
|
||||
if [ -z "$pw" ]; then
|
||||
warn "No LITELLM_MASTER_KEY in $(tildify "$DIR")/.env."
|
||||
return 0
|
||||
fi
|
||||
# shellcheck disable=SC2086 # CLIP is a command and its flags
|
||||
if printf '%s' "$pw" | $CLIP >/dev/null 2>&1; then
|
||||
step "Copied the admin password. Paste it into the sign-in page."
|
||||
else
|
||||
warn "Could not copy it. The password is LITELLM_MASTER_KEY in $(tildify "$DIR")/.env."
|
||||
fi
|
||||
pw=''
|
||||
}
|
||||
|
||||
# After the summary: copy the password, open the admin UI, or finish. The
|
||||
# menu comes back after each choice with the next step as its default.
|
||||
next_steps() {
|
||||
url="$1"
|
||||
[ "$INTERACTIVE" = 1 ] || return 0
|
||||
if [ -z "$CLIP" ]; then
|
||||
menu "Open the admin UI in your browser?" 1 "Yes" "No"
|
||||
if [ "$CHOICE" = 1 ]; then open_url "$url"; fi
|
||||
return 0
|
||||
fi
|
||||
tab="$(printf '\t')"
|
||||
pick=1
|
||||
while :; do
|
||||
menu "What next?" "$pick" \
|
||||
"Copy the admin password${tab}to the clipboard, ready to paste" \
|
||||
"Open the admin UI in your browser" \
|
||||
"Done"
|
||||
case "$CHOICE" in
|
||||
1) copy_password; pick=2 ;;
|
||||
2) open_url "$url"; pick=3 ;;
|
||||
*) return 0 ;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
main() {
|
||||
NO_QUESTIONS=0
|
||||
for arg in "$@"; do
|
||||
|
|
@ -246,37 +503,84 @@ main() {
|
|||
esac
|
||||
done
|
||||
|
||||
setup_output
|
||||
detect_terminal
|
||||
# Agents and CI get the defaults even inside a terminal, so nothing waits on a keypress.
|
||||
if [ "$NO_QUESTIONS" = 1 ] || [ -n "${CI:-}" ] || [ -n "${CLAUDECODE:-}" ]; then INTERACTIVE=0; fi
|
||||
trap restore_terminal EXIT
|
||||
trap on_interrupt INT TERM
|
||||
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
if [ "$STYLE" = 1 ]; then
|
||||
printf '\n%s%s LiteLLM quickstart%s\n' "${C_ACC}${C_BOLD}" "$S_HEAD" "${C_OFF}"
|
||||
printf '%s The gateway, Postgres, and the admin UI in one command%s\n\n' "${C_DIM}" "${C_OFF}"
|
||||
else
|
||||
echo "LiteLLM quickstart"
|
||||
fi
|
||||
|
||||
# Podman and Rancher Desktop (nerdctl in containerd mode; its dockerd mode
|
||||
# already provides a docker CLI) work the same way through Compose v2.
|
||||
ENGINE=''
|
||||
for cand in docker podman nerdctl; do
|
||||
if command -v "$cand" >/dev/null 2>&1; then
|
||||
ENGINE="$cand"
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ -z "$ENGINE" ]; then
|
||||
fail "No container engine found (docker, podman, or nerdctl)." "The LiteLLM Gateway runs in a container alongside a Postgres database."
|
||||
cat >&2 <<'EOF'
|
||||
Docker is not installed. The LiteLLM Gateway runs in Docker alongside a Postgres database.
|
||||
|
||||
Install Docker, then run this again: https://docs.docker.com/get-docker/
|
||||
Install Docker (https://docs.docker.com/get-docker/), Podman (https://podman.io),
|
||||
or Rancher Desktop (https://rancherdesktop.io), then run this again.
|
||||
Or deploy in one click (Railway or Render): https://docs.litellm.ai/docs/proxy/docker_quick_start
|
||||
Only need to call models from Python? pip install litellm
|
||||
EOF
|
||||
exit 1
|
||||
fi
|
||||
docker compose version >/dev/null 2>&1 || { echo "Docker Compose v2 ('docker compose') is required." >&2; exit 1; }
|
||||
docker info >/dev/null 2>&1 || { echo "Docker is installed but not running. Start it and run this again." >&2; exit 1; }
|
||||
command -v openssl >/dev/null 2>&1 || { echo "openssl is required to generate keys." >&2; exit 1; }
|
||||
case "$ENGINE" in
|
||||
podman) ENGINE_NAME=Podman ;;
|
||||
nerdctl) ENGINE_NAME=nerdctl ;;
|
||||
*) ENGINE_NAME=Docker ;;
|
||||
esac
|
||||
compose_version="$("$ENGINE" compose version --short 2>/dev/null)" ||
|
||||
{ fail "Compose v2 ('$ENGINE compose') is required."; exit 1; }
|
||||
if ! "$ENGINE" info >/dev/null 2>&1; then
|
||||
if [ "$ENGINE" = podman ]; then
|
||||
fail "Podman is installed but not running." "Start it (podman machine start) and run this again."
|
||||
else
|
||||
fail "$ENGINE_NAME is installed but not running." "Start it and run this again."
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
command -v openssl >/dev/null 2>&1 || { fail "openssl is required to generate keys."; exit 1; }
|
||||
engine_version="$("$ENGINE" version --format '{{.Server.Version}}' 2>/dev/null)" || engine_version=''
|
||||
step "$ENGINE_NAME ${engine_version:+$engine_version }with Compose ${compose_version#v} is running"
|
||||
|
||||
echo "LiteLLM quickstart"
|
||||
pick_folder
|
||||
[ -f .env ] || check_new_install
|
||||
curl -fsSL -o docker-compose.quickstart.yml "$COMPOSE_URL"
|
||||
step "Downloaded docker-compose.quickstart.yml"
|
||||
pick_port
|
||||
|
||||
if [ -f .env ]; then
|
||||
echo "Reusing $DIR/.env, so existing keys and data keep working."
|
||||
step "Reusing .env, so existing keys and data keep working"
|
||||
else
|
||||
(umask 077 && printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\nPOSTGRES_PASSWORD=%s\nLITELLM_PORT=%s\nLITELLM_BIND=127.0.0.1:\nCOMPOSE_PROJECT_NAME=%s\n' \
|
||||
"$(openssl rand -hex 32)" "$(openssl rand -hex 32)" "$(openssl rand -hex 24)" "$PORT" "$project" >.env)
|
||||
echo "Generated $DIR/.env with your master key, salt key, and database password. Keep this file."
|
||||
master="$(openssl rand -hex 32)"
|
||||
salt="$(openssl rand -hex 32)"
|
||||
db_password="$(openssl rand -hex 24)"
|
||||
# Write a fresh mktemp file (mode 600, never a reused one) and rename it, so a
|
||||
# failed write never leaves a partial .env that a rerun would mistake for a
|
||||
# finished install.
|
||||
if ! ENV_TMP="$(mktemp .env.XXXXXX)" ||
|
||||
! printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\nPOSTGRES_PASSWORD=%s\nLITELLM_PORT=%s\nLITELLM_BIND=127.0.0.1:\nCOMPOSE_PROJECT_NAME=%s\n' \
|
||||
"$master" "$salt" "$db_password" "$PORT" "$project" >"$ENV_TMP" || ! mv -f "$ENV_TMP" .env; then
|
||||
if [ -n "$ENV_TMP" ]; then rm -f "$ENV_TMP"; fi
|
||||
fail "Could not write $(tildify "$DIR")/.env."
|
||||
exit 1
|
||||
fi
|
||||
ENV_TMP=''
|
||||
unset master salt db_password
|
||||
step "Generated .env with your master key, salt key, and database password ${C_DIM}(keep this file; only you can read it)"
|
||||
fi
|
||||
|
||||
# Compose prefers values already set in the shell over .env, so drop any
|
||||
|
|
@ -287,28 +591,39 @@ EOF
|
|||
# is kept, so an intentional LITELLM_BIND=127.0.0.1: is not dropped.
|
||||
if grep -q '^LITELLM_BIND=' .env; then unset LITELLM_BIND; fi
|
||||
|
||||
echo "Starting LiteLLM and Postgres (the first run downloads the images)..."
|
||||
docker compose -f docker-compose.quickstart.yml up -d
|
||||
|
||||
i=0
|
||||
until curl -fsS "http://127.0.0.1:$PORT/health/readiness" >/dev/null 2>&1; do
|
||||
i=$((i + 1))
|
||||
if [ "$i" -gt 90 ]; then
|
||||
echo "The gateway did not become ready in 3 minutes. Check: cd $DIR && docker compose -f docker-compose.quickstart.yml logs litellm" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
started="$(date +%s)"
|
||||
detail=''
|
||||
for image in $("$ENGINE" compose -f docker-compose.quickstart.yml config --images 2>/dev/null); do
|
||||
"$ENGINE" image inspect "$image" >/dev/null 2>&1 || detail="downloading images, first run only"
|
||||
done
|
||||
[ "$STYLE" = 1 ] || echo "Starting LiteLLM and Postgres (the first run downloads the images)..."
|
||||
if ! spin "Starting LiteLLM and Postgres" "$detail" start_stack; then
|
||||
fail "$ENGINE_NAME could not start LiteLLM and Postgres." "Its output is above."
|
||||
exit 1
|
||||
fi
|
||||
if ! spin "Waiting for the gateway to be ready" "" wait_ready; then
|
||||
fail "The gateway did not become ready in 3 minutes." "See what it logged:"
|
||||
echo " cd $(tildify "$DIR") && $ENGINE compose -f docker-compose.quickstart.yml logs litellm" >&2
|
||||
exit 1
|
||||
fi
|
||||
step "LiteLLM and Postgres are up ${C_DIM}· $(elapsed_since "$started")"
|
||||
|
||||
url="http://localhost:$PORT/ui"
|
||||
password="the LITELLM_MASTER_KEY value in $(tildify "$DIR")/.env"
|
||||
CLIP=''
|
||||
if [ "$INTERACTIVE" = 1 ]; then find_clipboard; fi
|
||||
if [ -n "$CLIP" ]; then
|
||||
password="copy it below, or LITELLM_MASTER_KEY in $(tildify "$DIR")/.env"
|
||||
fi
|
||||
echo
|
||||
echo "LiteLLM is running."
|
||||
echo " Admin UI: http://localhost:$PORT/ui"
|
||||
echo " Username: admin"
|
||||
echo " Password: the LITELLM_MASTER_KEY value in $DIR/.env"
|
||||
echo " Next: in the UI, open Models + Endpoints > Add Model and paste a provider API key"
|
||||
echo " Stop it: cd $DIR && docker compose -f docker-compose.quickstart.yml down"
|
||||
box "LiteLLM is running" \
|
||||
"Admin UI|$url" \
|
||||
"Username|admin" \
|
||||
"Password|$password" \
|
||||
"Next|in the UI, open Models + Endpoints > Add Model and paste a provider API key" \
|
||||
"Stop it|cd $(tildify "$DIR") && $ENGINE compose -f docker-compose.quickstart.yml down"
|
||||
|
||||
open_browser "http://localhost:$PORT/ui"
|
||||
next_steps "$url"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
|
|
|||
|
|
@ -54,13 +54,9 @@ def get_all_functions_called_in_tests(base_dir):
|
|||
if file == "test_router_validate_fallbacks.py":
|
||||
print(f"tree: {tree}")
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Call) and isinstance(
|
||||
node.func, ast.Name
|
||||
):
|
||||
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
|
||||
called_functions.add(node.func.id)
|
||||
elif isinstance(node, ast.Call) and isinstance(
|
||||
node.func, ast.Attribute
|
||||
):
|
||||
elif isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
|
||||
called_functions.add(node.func.attr)
|
||||
|
||||
return called_functions
|
||||
|
|
@ -91,6 +87,8 @@ ignored_function_names = [
|
|||
"_get_claude_code_session_router_binding", # Tested through the two-worker session routing test in test_router.py
|
||||
"_apply_updated_routing_strategy_args", # Tested via update_settings in test_lowest_latency.py (file lacks "router" in name)
|
||||
"arm_routing_read_prefetch", # Tested in tests/unit/caching/test_request_redis_batch_pre_call.py (file lacks "router" in name)
|
||||
"_configured_model_info", # Tested through get_configured_service_tiers in test_router.py
|
||||
"_routable_deployments", # Tested through get_configured_service_tiers and get_routable_upstream_model in test_router.py
|
||||
"_async_get_available_deployment", # Body of the `route {model}` phase wrapper, exercised through async_get_available_deployment in test_router.py
|
||||
"_async_get_available_deployment_for_pass_through", # Same, through async_get_available_deployment_for_pass_through in test_router.py
|
||||
"_embedding",
|
||||
|
|
@ -109,9 +107,7 @@ def main():
|
|||
# "../../litellm/router_utils/pattern_match_deployments.py",
|
||||
# "../../litellm/router_utils/batch_utils.py",
|
||||
# ] ## LOCAL TESTING
|
||||
tests_dir = (
|
||||
"./tests/" # Update this path if your tests directory is located elsewhere
|
||||
)
|
||||
tests_dir = "./tests/" # Update this path if your tests directory is located elsewhere
|
||||
# tests_dir = "../../tests/" # LOCAL TESTING
|
||||
|
||||
router_functions = []
|
||||
|
|
@ -119,9 +115,7 @@ def main():
|
|||
router_functions.extend(get_functions_from_router(file))
|
||||
print("router_functions: ", router_functions)
|
||||
called_functions_in_tests = get_all_functions_called_in_tests(tests_dir)
|
||||
untested_functions = [
|
||||
fn for fn in router_functions if fn not in called_functions_in_tests
|
||||
]
|
||||
untested_functions = [fn for fn in router_functions if fn not in called_functions_in_tests]
|
||||
|
||||
if untested_functions:
|
||||
all_untested_functions = []
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ These tests exercise a running gateway, PostgreSQL and Redis with an owned local
|
|||
|
||||
The `cost` group is driven by `cost_tracking_cases.json`, which contains the cost map, literal requests, literal provider responses and expected accounting values. Each case has a name, contract ID, cost-map model, optional deployment overrides, request body, tagged response and exact or recount expectations. Request bodies use `$MODEL` for the registered proxy model, while responses use `$REQUEST_ID` for the per-run scenario ID. To add a case, add a cost-map entry when the model is new, add the request body and exact provider response data, and add hand-computed expected values. The upstream serves each stored response for any path under `/<scenario_id>`, while the test-owned cost map is served over loopback through `LITELLM_MODEL_COST_MAP_URL`
|
||||
|
||||
Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions`, `mcp`, `sdk` or `cost` to run a selected group. The group to directory mapping is the `GROUPS` literal at the top of `run.py`; a new directory needs a `GROUPS` entry and an `OWNED_DIRECTORIES` entry in `_support/manifest.py`. Set `INTEGRATION_WORKERS` above 1 to run a group under pytest-xdist; the `mcp` job does this in CI, so MCP tests must own their resources per scenario. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate
|
||||
Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions`, `mcp`, `sdk` or `cost` to run a selected group. The group to directory mapping is the `GROUPS` literal at the top of `run.py`; a new directory needs a `GROUPS` entry and an `OWNED_DIRECTORIES` entry in `_support/manifest.py`. Set `INTEGRATION_WORKERS` above 1 to run a group under pytest-xdist; the `mcp` job does this in CI, so MCP tests must own their resources per scenario. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. When that deployment runs more than one proxy worker, set `INTEGRATION_PROXY_WORKERS` to the count so a test that writes a model and then calls it waits out the config reload interval, the only cross-worker convergence bound the wire exposes. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate
|
||||
|
||||
Management also requires `INTEGRATION_PEER_URL`, `REDIS_HOST` and `REDIS_PORT`. CircleCI starts two directly addressed proxy processes sharing only that job's stores. The test-only CLI wrapper supplies enterprise route entitlement, following the existing behavior suite's convention. It does not qualify license validation; run it with one worker and no reload
|
||||
|
||||
|
|
|
|||
384
tests/integration/authorization/test_audit_any_sweep_auth.py
Normal file
384
tests/integration/authorization/test_audit_any_sweep_auth.py
Normal file
|
|
@ -0,0 +1,384 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from typing import Final, Literal
|
||||
|
||||
import anthropic
|
||||
import httpx
|
||||
import openai
|
||||
import pytest
|
||||
import websockets
|
||||
from integration._support.client import Gateway, eventually
|
||||
from integration._support.database import read_rows
|
||||
from pydantic import JsonValue
|
||||
|
||||
|
||||
def _marker() -> str:
|
||||
return uuid.uuid4().hex
|
||||
|
||||
|
||||
def _register_scenario(gateway: Gateway, scenario_id: str, response: dict[str, JsonValue]) -> None:
|
||||
with httpx.Client(timeout=5, trust_env=False) as client:
|
||||
result: Final = client.post(
|
||||
f"{gateway.upstream_url}/__scenarios",
|
||||
json={"scenario_id": scenario_id, "response": response},
|
||||
)
|
||||
assert result.status_code == 200, result.text
|
||||
|
||||
|
||||
_ANTHROPIC_MESSAGE: Final = {
|
||||
"id": "msg_$UNIQUE_ID",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-3-5-sonnet-20241022",
|
||||
"content": [{"type": "text", "text": "scripted reply"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 9, "output_tokens": 5},
|
||||
}
|
||||
|
||||
_ANTHROPIC_EVENTS: Final = (
|
||||
'event: message_start\ndata: {"type":"message_start","message":{"id":"msg_$UNIQUE_ID","type":"message","role":"assistant","model":"claude-3-5-sonnet-20241022","content":[],"usage":{"input_tokens":9,"output_tokens":0}}}',
|
||||
'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}',
|
||||
'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"scripted reply"}}',
|
||||
'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}',
|
||||
'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":5}}',
|
||||
'event: message_stop\ndata: {"type":"message_stop"}',
|
||||
)
|
||||
|
||||
_RESPONSES_BODY: Final = {
|
||||
"id": "resp_$UNIQUE_ID",
|
||||
"object": "response",
|
||||
"created_at": 1700000000,
|
||||
"status": "completed",
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"output": [{"type": "message", "content": [{"type": "output_text", "text": "done"}]}],
|
||||
"usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7},
|
||||
}
|
||||
|
||||
_RESPONSES_EVENTS: Final = (
|
||||
'event: response.created\ndata: {"type":"response.created","response":{"id":"$UNIQUE_ID","object":"response","status":"in_progress"}}',
|
||||
'event: response.completed\ndata: {"type":"response.completed","response":{"id":"$UNIQUE_ID","object":"response","status":"completed","output":[{"type":"message","content":[{"type":"output_text","text":"done"}]}],"usage":{"input_tokens":3,"output_tokens":4,"total_tokens":7}}}',
|
||||
)
|
||||
|
||||
|
||||
def _anthropic_model(gateway: Gateway, *, stream: bool) -> str:
|
||||
scenario_id: Final = "audit-anthropic-stream" if stream else "audit-anthropic"
|
||||
if stream:
|
||||
_register_scenario(gateway, scenario_id, {"content_type": "text/event-stream", "frames": _ANTHROPIC_EVENTS})
|
||||
else:
|
||||
_register_scenario(
|
||||
gateway,
|
||||
scenario_id,
|
||||
{
|
||||
"content_type": "application/x-routed",
|
||||
"routes": {"POST /v1/messages": {"content_type": "application/json", "body": _ANTHROPIC_MESSAGE}},
|
||||
},
|
||||
)
|
||||
name: Final = f"integration-{uuid.uuid4().hex}"
|
||||
created: Final = gateway.post(
|
||||
"/model/new",
|
||||
{
|
||||
"model_name": name,
|
||||
"litellm_params": {
|
||||
"model": "anthropic/claude-3-5-sonnet-20241022",
|
||||
"api_key": "integration-provider-key",
|
||||
"api_base": f"{gateway.upstream_url}/{scenario_id}",
|
||||
},
|
||||
},
|
||||
)
|
||||
assert isinstance(created["model_info"], dict) and created["model_info"]["id"], created
|
||||
return name
|
||||
|
||||
|
||||
def _responses_model(gateway: Gateway, *, stream: bool) -> str:
|
||||
scenario_id: Final = "audit-resp-stream" if stream else "audit-resp"
|
||||
if stream:
|
||||
_register_scenario(gateway, scenario_id, {"content_type": "text/event-stream", "frames": _RESPONSES_EVENTS})
|
||||
else:
|
||||
_register_scenario(
|
||||
gateway,
|
||||
scenario_id,
|
||||
{
|
||||
"content_type": "application/x-routed",
|
||||
"routes": {"POST /responses": {"content_type": "application/json", "body": _RESPONSES_BODY}},
|
||||
},
|
||||
)
|
||||
name: Final = f"integration-{uuid.uuid4().hex}"
|
||||
created: Final = gateway.post(
|
||||
"/model/new",
|
||||
{
|
||||
"model_name": name,
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_key": "integration-provider-key",
|
||||
"api_base": f"{gateway.upstream_url}/{scenario_id}",
|
||||
},
|
||||
},
|
||||
)
|
||||
assert isinstance(created["model_info"], dict) and created["model_info"]["id"], created
|
||||
return name
|
||||
|
||||
|
||||
def _openai_client(gateway: Gateway, key: str) -> openai.OpenAI:
|
||||
return openai.OpenAI(
|
||||
base_url=f"{gateway.client.base_url}/v1",
|
||||
api_key=key,
|
||||
http_client=httpx.Client(trust_env=False),
|
||||
)
|
||||
|
||||
|
||||
def _openai_async_client(gateway: Gateway, key: str) -> openai.AsyncOpenAI:
|
||||
return openai.AsyncOpenAI(
|
||||
base_url=f"{gateway.client.base_url}/v1",
|
||||
api_key=key,
|
||||
http_client=httpx.AsyncClient(trust_env=False),
|
||||
)
|
||||
|
||||
|
||||
def _anthropic_client(gateway: Gateway, key: str) -> anthropic.Anthropic:
|
||||
return anthropic.Anthropic(
|
||||
base_url=str(gateway.client.base_url).rstrip("/"),
|
||||
api_key=key,
|
||||
http_client=httpx.Client(trust_env=False),
|
||||
)
|
||||
|
||||
|
||||
def _anthropic_async_client(gateway: Gateway, key: str) -> anthropic.AsyncAnthropic:
|
||||
return anthropic.AsyncAnthropic(
|
||||
base_url=str(gateway.client.base_url).rstrip("/"),
|
||||
api_key=key,
|
||||
http_client=httpx.AsyncClient(trust_env=False),
|
||||
)
|
||||
|
||||
|
||||
def _sse_chunk_models(text: str) -> tuple[str, ...]:
|
||||
models: list[str] = []
|
||||
for line in text.splitlines():
|
||||
if not line.startswith("data: ") or line == "data: [DONE]":
|
||||
continue
|
||||
value = json.loads(line.removeprefix("data: "))
|
||||
if isinstance(value, dict) and isinstance(value.get("model"), str):
|
||||
models.append(value["model"])
|
||||
return tuple(models)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("client_kind", ("httpx", "openai", "openai_async"))
|
||||
@pytest.mark.parametrize("stream", (False, True))
|
||||
def test_allowed_key_chat_completions_restamps_alias(
|
||||
gateway: Gateway, client_kind: Literal["httpx", "openai", "openai_async"], stream: bool
|
||||
) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
key: Final = scenario.key(models=[model])
|
||||
body: Final = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": _marker()}],
|
||||
"stream": stream,
|
||||
**({"stream_options": {"include_usage": True}} if stream else {}),
|
||||
}
|
||||
if client_kind == "httpx":
|
||||
response: Final = gateway.request("POST", "/v1/chat/completions", body, key=key)
|
||||
assert response.status_code == 200, response.text
|
||||
if stream:
|
||||
models: Final = _sse_chunk_models(response.text)
|
||||
assert models and all(chunk_model == model for chunk_model in models), response.text
|
||||
else:
|
||||
assert response.json()["model"] == model, response.text
|
||||
elif client_kind == "openai":
|
||||
client: Final = _openai_client(gateway, key)
|
||||
completion: Final = client.chat.completions.create(**body)
|
||||
if stream:
|
||||
chunk_models: list[str] = []
|
||||
for chunk in completion:
|
||||
if chunk.model:
|
||||
chunk_models.append(chunk.model)
|
||||
assert chunk_models and all(chunk_model == model for chunk_model in chunk_models)
|
||||
else:
|
||||
assert completion.model == model
|
||||
else:
|
||||
|
||||
async def _run() -> None:
|
||||
client_async: Final = _openai_async_client(gateway, key)
|
||||
completion_async: Final = await client_async.chat.completions.create(**body)
|
||||
if stream:
|
||||
seen: list[str] = []
|
||||
async for chunk in completion_async:
|
||||
if chunk.model:
|
||||
seen.append(chunk.model)
|
||||
assert seen and all(chunk_model == model for chunk_model in seen)
|
||||
else:
|
||||
assert completion_async.model == model
|
||||
|
||||
import asyncio
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("client_kind", ("httpx", "anthropic", "anthropic_async"))
|
||||
@pytest.mark.parametrize("stream", (False, True))
|
||||
def test_allowed_key_messages(gateway: Gateway, client_kind: str, stream: bool) -> None:
|
||||
model: Final = _anthropic_model(gateway, stream=stream)
|
||||
with gateway.scenario() as scenario:
|
||||
key: Final = scenario.key(models=[model])
|
||||
body: Final = {
|
||||
"model": model,
|
||||
"max_tokens": 64,
|
||||
"messages": [{"role": "user", "content": _marker()}],
|
||||
"stream": stream,
|
||||
}
|
||||
if client_kind == "httpx":
|
||||
response: Final = gateway.request("POST", "/v1/messages", body, key=key)
|
||||
assert response.status_code == 200, response.text
|
||||
if stream:
|
||||
assert "message_start" in response.text and "message_stop" in response.text, response.text
|
||||
else:
|
||||
assert response.json()["type"] == "message", response.text
|
||||
elif client_kind == "anthropic":
|
||||
client: Final = _anthropic_client(gateway, key)
|
||||
if stream:
|
||||
with client.messages.stream(**{k: v for k, v in body.items() if k != "stream"}) as event_stream:
|
||||
message: Final = event_stream.get_final_message()
|
||||
assert message.type == "message" and message.content, message
|
||||
else:
|
||||
message = client.messages.create(**{k: v for k, v in body.items() if k != "stream"})
|
||||
assert message.type == "message" and message.content, message
|
||||
else:
|
||||
|
||||
async def _run() -> None:
|
||||
client_async: Final = _anthropic_async_client(gateway, key)
|
||||
params: Final = {k: v for k, v in body.items() if k != "stream"}
|
||||
if stream:
|
||||
async with client_async.messages.stream(**params) as event_stream:
|
||||
message_async: Final = await event_stream.get_final_message()
|
||||
else:
|
||||
message_async = await client_async.messages.create(**params)
|
||||
assert message_async.type == "message" and message_async.content, message_async
|
||||
|
||||
import asyncio
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("client_kind", ("httpx", "openai", "openai_async"))
|
||||
@pytest.mark.parametrize("stream", (False, True))
|
||||
def test_allowed_key_responses(gateway: Gateway, client_kind: str, stream: bool) -> None:
|
||||
model: Final = _responses_model(gateway, stream=stream)
|
||||
with gateway.scenario() as scenario:
|
||||
key: Final = scenario.key(models=[model])
|
||||
body: Final = {
|
||||
"model": model,
|
||||
"input": [{"role": "user", "content": [{"type": "input_text", "text": _marker()}]}],
|
||||
"stream": stream,
|
||||
}
|
||||
if client_kind == "httpx":
|
||||
response: Final = gateway.request("POST", "/v1/responses", body, key=key)
|
||||
assert response.status_code == 200, response.text
|
||||
if stream:
|
||||
assert "response.completed" in response.text, response.text
|
||||
else:
|
||||
assert response.json()["object"] == "response", response.text
|
||||
elif client_kind == "openai":
|
||||
client: Final = _openai_client(gateway, key)
|
||||
result: Final = client.responses.create(**body)
|
||||
if stream:
|
||||
events: list[str] = []
|
||||
for event in result:
|
||||
events.append(event.type)
|
||||
assert "response.completed" in events, events
|
||||
else:
|
||||
assert result.object == "response", result
|
||||
else:
|
||||
|
||||
async def _run() -> None:
|
||||
client_async: Final = _openai_async_client(gateway, key)
|
||||
result_async: Final = await client_async.responses.create(**body)
|
||||
if stream:
|
||||
events_async: list[str] = []
|
||||
async for event in result_async:
|
||||
events_async.append(event.type)
|
||||
assert "response.completed" in events_async, events_async
|
||||
else:
|
||||
assert result_async.object == "response", result_async
|
||||
|
||||
import asyncio
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
|
||||
def test_denied_key_rejected(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
other_model: Final = scenario.model()
|
||||
key: Final = scenario.key(models=[other_model])
|
||||
for path, body in (
|
||||
("/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": "x"}]}),
|
||||
("/v1/messages", {"model": model, "max_tokens": 4, "messages": [{"role": "user", "content": "x"}]}),
|
||||
("/v1/responses", {"model": model, "input": "x"}),
|
||||
):
|
||||
response: Final = gateway.request("POST", path, body, key=key)
|
||||
assert response.status_code in (401, 403), f"{path}: {response.status_code} {response.text}"
|
||||
|
||||
|
||||
async def test_realtime_websocket_allowed_and_denied(gateway: Gateway) -> None:
|
||||
rt_scenario: Final = f"auditrt{uuid.uuid4().hex[:8]}"
|
||||
_register_scenario(
|
||||
gateway,
|
||||
rt_scenario,
|
||||
{
|
||||
"content_type": "application/x-realtime",
|
||||
"session_model": "gpt-4o-realtime-preview",
|
||||
"events": (
|
||||
{"type": "response.done", "response": {"id": "resp_$UNIQUE_ID", "model": "gpt-4o-realtime-preview"}},
|
||||
),
|
||||
},
|
||||
)
|
||||
name: Final = f"integration-{uuid.uuid4().hex}"
|
||||
created: Final = gateway.post(
|
||||
"/model/new",
|
||||
{
|
||||
"model_name": name,
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-realtime-preview",
|
||||
"api_key": rt_scenario,
|
||||
"api_base": gateway.upstream_url,
|
||||
},
|
||||
},
|
||||
)
|
||||
assert isinstance(created["model_info"], dict) and created["model_info"]["id"], created
|
||||
with gateway.scenario() as scenario:
|
||||
key: Final = scenario.key(models=[name])
|
||||
other_model: Final = scenario.model()
|
||||
denied_key: Final = scenario.key(models=[other_model])
|
||||
url: Final = f"{str(gateway.client.base_url).replace('http://', 'ws://')}/v1/realtime?model={name}"
|
||||
async with websockets.connect(url, additional_headers={"Authorization": f"Bearer {key}"}) as websocket:
|
||||
session: Final = json.loads(await websocket.recv())
|
||||
assert session["type"] == "session.created", session
|
||||
assert session["session"]["model"] == "gpt-4o-realtime-preview", session
|
||||
await websocket.send(json.dumps({"type": "response.create"}))
|
||||
event: Final = json.loads(await websocket.recv())
|
||||
assert event["type"] == "response.done", event
|
||||
with pytest.raises(websockets.exceptions.InvalidHandshake):
|
||||
async with websockets.connect(
|
||||
url, additional_headers={"Authorization": f"Bearer {denied_key}"}
|
||||
) as denied_socket:
|
||||
await denied_socket.recv()
|
||||
|
||||
|
||||
def test_max_budget_allows_then_denies_and_records_spend(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
key: Final = scenario.key(models=[model], max_budget=0.0000001)
|
||||
body: Final = {"model": model, "messages": [{"role": "user", "content": _marker()}]}
|
||||
first: Final = gateway.request("POST", "/v1/chat/completions", body, key=key)
|
||||
assert first.status_code == 200, first.text
|
||||
request_id: Final = first.json()["id"]
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows('SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id = %s', (request_id,)),
|
||||
lambda values: len(values) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
assert rows, first.text
|
||||
second: Final = gateway.request("POST", "/v1/chat/completions", body, key=key)
|
||||
assert second.status_code == 422, second.text
|
||||
assert "budget_exceeded" in second.text, second.text
|
||||
826
tests/integration/compatibility/test_codex_model_catalog.py
Normal file
826
tests/integration/compatibility/test_codex_model_catalog.py
Normal file
|
|
@ -0,0 +1,826 @@
|
|||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from hashlib import sha256
|
||||
from itertools import chain
|
||||
from pathlib import Path
|
||||
from typing import Final, TypeVar
|
||||
|
||||
import anthropic
|
||||
import httpx
|
||||
import openai
|
||||
from pydantic import JsonValue
|
||||
|
||||
from litellm.constants import PROXY_CONFIG_RELOAD_INTERVAL_SECONDS
|
||||
from tests.integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, object_value, string_value
|
||||
from tests.integration._support.database import read_rows
|
||||
from tests.integration._support.wire import Reply, Request, Wire, wire_server
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
CLIENT_VERSION: Final = "0.159.3"
|
||||
CONSISTENT_READS: Final = 6
|
||||
CONVERGENCE_SECONDS: Final = 60
|
||||
RELOAD_MARGIN_SECONDS: Final = 5.0
|
||||
PROXY_WORKERS: Final = int(os.environ.get("INTEGRATION_PROXY_WORKERS", "1"))
|
||||
WORKER_SYNC_SECONDS: Final = 0.0 if PROXY_WORKERS == 1 else PROXY_CONFIG_RELOAD_INTERVAL_SECONDS + RELOAD_MARGIN_SECONDS
|
||||
ROOT: Final = Path(os.environ.get("INTEGRATION_PROXY_ROOT") or Path(__file__).resolve().parents[3])
|
||||
BUNDLED_STOCK_PATH: Final = ROOT / "litellm" / "proxy" / "common_utils" / "codex_bundled_models_0.159.3.json"
|
||||
BASE_INSTRUCTIONS_PATH: Final = ROOT / "litellm" / "proxy" / "common_utils" / "codex_base_instructions.md"
|
||||
FRESH_CONNECTION: Final = {"Connection": "close"}
|
||||
ANTHROPIC_HEADERS: Final = {"anthropic-version": "2023-06-01"}
|
||||
USAGE: Final = {"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15}
|
||||
|
||||
|
||||
def _generic_tier(identity: str) -> dict[str, JsonValue]:
|
||||
return {"id": identity, "name": identity.capitalize(), "description": f"Sends service_tier={identity} upstream"}
|
||||
|
||||
|
||||
def _bundled_stock() -> dict[str, dict[str, JsonValue]]:
|
||||
catalog: Final = JSON_OBJECT.validate_json(BUNDLED_STOCK_PATH.read_bytes())
|
||||
models: Final = catalog["models"]
|
||||
assert isinstance(models, list), catalog
|
||||
return {string_value(object_value(model)["slug"]): object_value(model) for model in models}
|
||||
|
||||
|
||||
def _catalog_response(
|
||||
gateway: Gateway,
|
||||
*,
|
||||
key: str | None = None,
|
||||
params: Mapping[str, str] | None = None,
|
||||
headers: Mapping[str, str] | None = None,
|
||||
path: str = "/v1/models",
|
||||
) -> httpx.Response:
|
||||
return gateway.request(
|
||||
"GET",
|
||||
path,
|
||||
key=key,
|
||||
params={"client_version": CLIENT_VERSION, **(params or {})},
|
||||
headers={**FRESH_CONNECTION, **(headers or {})},
|
||||
)
|
||||
|
||||
|
||||
def _entries(response: httpx.Response) -> list[dict[str, JsonValue]]:
|
||||
assert response.status_code == 200, response.text
|
||||
body: Final = JSON_OBJECT.validate_json(response.content)
|
||||
assert set(body) == {"models"}, response.text
|
||||
models: Final = body["models"]
|
||||
assert isinstance(models, list), response.text
|
||||
return [object_value(entry) for entry in models]
|
||||
|
||||
|
||||
def _catalog(
|
||||
gateway: Gateway, *, key: str | None = None, params: Mapping[str, str] | None = None
|
||||
) -> list[dict[str, JsonValue]]:
|
||||
return _entries(_catalog_response(gateway, key=key, params=params))
|
||||
|
||||
|
||||
def _by_slug(entries: Sequence[Mapping[str, JsonValue]]) -> dict[str, dict[str, JsonValue]]:
|
||||
return {string_value(entry["slug"]): dict(entry) for entry in entries}
|
||||
|
||||
|
||||
def _tier_ids(entry: Mapping[str, JsonValue]) -> tuple[str, ...]:
|
||||
tiers: Final = entry["service_tiers"]
|
||||
assert isinstance(tiers, list), entry
|
||||
return tuple(string_value(object_value(tier)["id"]) for tier in tiers)
|
||||
|
||||
|
||||
def _tiers_in(entries: Sequence[Mapping[str, JsonValue]]) -> dict[str, tuple[str, ...]]:
|
||||
return {slug: _tier_ids(entry) for slug, entry in _by_slug(entries).items()}
|
||||
|
||||
|
||||
def _tiers_by_slug(
|
||||
gateway: Gateway, *, key: str | None = None, params: Mapping[str, str] | None = None
|
||||
) -> dict[str, tuple[str, ...]]:
|
||||
return _tiers_in(_catalog(gateway, key=key, params=params))
|
||||
|
||||
|
||||
def _converged_catalog(
|
||||
gateway: Gateway,
|
||||
satisfied: Callable[[dict[str, dict[str, JsonValue]]], bool],
|
||||
*,
|
||||
key: str | None = None,
|
||||
params: Mapping[str, str] | None = None,
|
||||
) -> dict[str, dict[str, JsonValue]]:
|
||||
return eventually(
|
||||
lambda: _by_slug(_catalog(gateway, key=key, params=params)), satisfied, seconds=CONVERGENCE_SECONDS
|
||||
)
|
||||
|
||||
|
||||
def _settled(read: Callable[[], T], satisfied: Callable[[T], bool], *, seconds: float = CONVERGENCE_SECONDS) -> T:
|
||||
reads: Final = eventually(
|
||||
lambda: tuple(read() for _ in range(CONSISTENT_READS)),
|
||||
lambda values: all(satisfied(value) for value in values),
|
||||
seconds=seconds,
|
||||
)
|
||||
return reads[-1]
|
||||
|
||||
|
||||
def _settled_on_every_worker(read: Callable[[], T], satisfied: Callable[[T], bool], *, written_at: float) -> T:
|
||||
eventually(
|
||||
lambda: (time.monotonic() - written_at, read()),
|
||||
lambda stamped: stamped[0] >= WORKER_SYNC_SECONDS and satisfied(stamped[1]),
|
||||
seconds=WORKER_SYNC_SECONDS + CONVERGENCE_SECONDS,
|
||||
)
|
||||
return _settled(read, satisfied)
|
||||
|
||||
|
||||
def _plain_ids(gateway: Gateway, *, key: str | None = None, params: Mapping[str, str] | None = None) -> list[str]:
|
||||
response: Final = gateway.request("GET", "/v1/models", key=key, params=params, headers=FRESH_CONNECTION)
|
||||
assert response.status_code == 200, response.text
|
||||
body: Final = JSON_OBJECT.validate_json(response.content)
|
||||
assert set(body) == {"data", "object"} and body["object"] == "list", response.text
|
||||
data: Final = body["data"]
|
||||
assert isinstance(data, list), response.text
|
||||
return [string_value(object_value(entry)["id"]) for entry in data]
|
||||
|
||||
|
||||
def _model_info_rows(gateway: Gateway, model_name: str) -> list[dict[str, JsonValue]]:
|
||||
entries: Final = gateway.get("/model/info")["data"]
|
||||
assert isinstance(entries, list), entries
|
||||
return [object_value(entry) for entry in entries if object_value(entry)["model_name"] == model_name]
|
||||
|
||||
|
||||
def _delete_model_if_present(gateway: Gateway, identity: str) -> None:
|
||||
if read_rows('SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s', (identity,)):
|
||||
gateway.post("/model/delete", {"id": identity})
|
||||
assert read_rows('SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s', (identity,)) == []
|
||||
|
||||
|
||||
def _new_model(
|
||||
scenario: Scenario,
|
||||
name: str,
|
||||
*,
|
||||
model: str = "openai/gpt-4o-mini",
|
||||
model_info: Mapping[str, JsonValue] | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> str:
|
||||
created: Final = scenario.gateway.post(
|
||||
"/model/new",
|
||||
{
|
||||
"model_name": name,
|
||||
"litellm_params": {
|
||||
"model": model,
|
||||
"api_key": "integration-provider-key",
|
||||
"api_base": api_base or f"{scenario.gateway.upstream_url}/v1",
|
||||
},
|
||||
"model_info": dict(model_info) if model_info is not None else {},
|
||||
},
|
||||
)
|
||||
identity: Final = string_value(object_value(created["model_info"])["id"])
|
||||
scenario.cleanups.callback(_delete_model_if_present, scenario.gateway, identity)
|
||||
return identity
|
||||
|
||||
|
||||
def _model_name(prefix: str) -> str:
|
||||
return f"{prefix}-{uuid.uuid4().hex}"
|
||||
|
||||
|
||||
def test_client_version_answers_codex_catalog_shape(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model_info={"max_input_tokens": 2048})
|
||||
catalog: Final = _converged_catalog(
|
||||
gateway, lambda value: model in value and value[model]["context_window"] == 2048
|
||||
)
|
||||
slugs: Final = list(catalog)
|
||||
entry: Final = catalog[model]
|
||||
assert entry["display_name"] == model, entry
|
||||
assert entry["priority"] == slugs.index(model), entry
|
||||
assert entry["visibility"] == "list" and entry["supported_in_api"] is True, entry
|
||||
assert entry["service_tiers"] == [], entry
|
||||
assert entry["context_window"] == 2048, entry
|
||||
assert entry["base_instructions"] == BASE_INSTRUCTIONS_PATH.read_text(encoding="utf-8"), entry["slug"]
|
||||
|
||||
|
||||
def test_models_route_matches_v1_models(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
key: Final = scenario.key(models=[model])
|
||||
routes: Final = eventually(
|
||||
lambda: (_entries(_catalog_response(gateway, key=key, path="/models")), _catalog(gateway, key=key)),
|
||||
lambda pair: pair[0] == pair[1],
|
||||
seconds=CONVERGENCE_SECONDS,
|
||||
)
|
||||
assert [entry["slug"] for entry in routes[0]] == [model], routes
|
||||
|
||||
|
||||
def test_plain_listing_keeps_openai_shape(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model_info={"service_tiers": ["priority"]})
|
||||
key: Final = scenario.key(models=[model])
|
||||
ids: Final = eventually(
|
||||
lambda: _plain_ids(gateway, key=key), lambda value: value == [model], seconds=CONVERGENCE_SECONDS
|
||||
)
|
||||
assert ids == [model], ids
|
||||
response: Final = gateway.request("GET", "/v1/models", key=key, headers=FRESH_CONNECTION)
|
||||
assert "service_tiers" not in response.text and "base_instructions" not in response.text, response.text
|
||||
|
||||
|
||||
def test_client_version_wins_over_anthropic_headers(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
key: Final = scenario.key(models=[model])
|
||||
codex: Final = eventually(
|
||||
lambda: _catalog_response(gateway, key=key, headers={**ANTHROPIC_HEADERS, "x-api-key": key}),
|
||||
lambda response: [entry["slug"] for entry in _entries(response)] == [model],
|
||||
seconds=CONVERGENCE_SECONDS,
|
||||
)
|
||||
assert [entry["slug"] for entry in _entries(codex)] == [model], codex.text
|
||||
anthropic_listing: Final = gateway.request(
|
||||
"GET", "/v1/models", key=key, headers={**ANTHROPIC_HEADERS, "x-api-key": key, **FRESH_CONNECTION}
|
||||
)
|
||||
assert anthropic_listing.status_code == 200, anthropic_listing.text
|
||||
body: Final = JSON_OBJECT.validate_json(anthropic_listing.content)
|
||||
assert set(body) == {"data", "first_id", "has_more", "last_id"}, anthropic_listing.text
|
||||
|
||||
|
||||
def test_cursor_models_route_ignores_client_version(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
key: Final = scenario.key(models=[model])
|
||||
response: Final = eventually(
|
||||
lambda: _catalog_response(gateway, key=key, path="/cursor/v1/models"),
|
||||
lambda value: value.status_code == 200 and model in value.text,
|
||||
seconds=CONVERGENCE_SECONDS,
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body: Final = JSON_OBJECT.validate_json(response.content)
|
||||
assert set(body) == {"data", "object"}, response.text
|
||||
data: Final = body["data"]
|
||||
assert isinstance(data, list) and [object_value(entry)["id"] for entry in data] == [model], response.text
|
||||
|
||||
|
||||
def test_client_version_value_shapes(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model_info={"service_tiers": ["priority"]})
|
||||
key: Final = scenario.key(models=[model])
|
||||
for value in ("", "1", "a" * 5000):
|
||||
response: Final = gateway.request(
|
||||
"GET", "/v1/models", key=key, params={"client_version": value}, headers=FRESH_CONNECTION
|
||||
)
|
||||
assert [entry["slug"] for entry in _entries(response)] == [model], (value[:20], response.text)
|
||||
repeated: Final = gateway.client.get(
|
||||
"/v1/models?client_version=1&client_version=2",
|
||||
headers={"Authorization": f"Bearer {key}", **FRESH_CONNECTION},
|
||||
)
|
||||
assert [entry["slug"] for entry in _entries(repeated)] == [model], repeated.text
|
||||
eventually(
|
||||
lambda: (_catalog(gateway, key=key), _catalog(gateway, key=key)),
|
||||
lambda pair: pair[0] == pair[1] and _tiers_in(pair[0]) == {model: ("priority",)},
|
||||
seconds=CONVERGENCE_SECONDS,
|
||||
)
|
||||
unauthenticated: Final = gateway.client.get(
|
||||
"/v1/models", params={"client_version": CLIENT_VERSION}, headers=FRESH_CONNECTION
|
||||
)
|
||||
assert unauthenticated.status_code == 401, unauthenticated.text
|
||||
|
||||
|
||||
def test_configured_tiers_on_unknown_model(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
ultrafast: Final = {"id": "ultrafast", "name": "Ultra", "description": "fast lane"}
|
||||
model: Final = scenario.model(
|
||||
model_info={
|
||||
"service_tiers": ["priority", ultrafast],
|
||||
"display_name": "Probe Unknown",
|
||||
"max_input_tokens": 1234,
|
||||
}
|
||||
)
|
||||
entry: Final = _converged_catalog(
|
||||
gateway, lambda value: model in value and value[model]["context_window"] == 1234
|
||||
)[model]
|
||||
assert entry["service_tiers"] == [_generic_tier("priority"), ultrafast], entry
|
||||
assert entry["display_name"] == "Probe Unknown", entry
|
||||
assert entry["context_window"] == 1234, entry
|
||||
assert entry["upgrade"] is None, entry
|
||||
|
||||
|
||||
def test_stock_model_keeps_codex_entry(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
name: Final = _model_name("codex-stock")
|
||||
_new_model(scenario, name, model="openai/gpt-5.5")
|
||||
catalog: Final = _converged_catalog(gateway, lambda value: name in value)
|
||||
slugs: Final = list(catalog)
|
||||
stock: Final = _bundled_stock()["gpt-5.5"]
|
||||
assert stock["upgrade"] is not None and stock["service_tiers"] != [], stock
|
||||
expected: Final = {
|
||||
**stock,
|
||||
"slug": name,
|
||||
"display_name": name,
|
||||
"priority": slugs.index(name),
|
||||
"visibility": "list",
|
||||
"supported_in_api": True,
|
||||
"upgrade": None,
|
||||
}
|
||||
assert catalog[name] == expected, catalog[name]
|
||||
|
||||
|
||||
def test_stock_model_configured_tiers_override(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
stock: Final = _bundled_stock()["gpt-6-sol"]
|
||||
stock_tiers: Final = stock["service_tiers"]
|
||||
assert isinstance(stock_tiers, list) and len(stock_tiers) == 1, stock
|
||||
stock_tier: Final = object_value(stock_tiers[0])
|
||||
assert stock["default_service_tier"] == stock_tier["id"], stock
|
||||
unknown_tier: Final = _model_name("codex-unknown-tier")
|
||||
known_tier: Final = _model_name("codex-known-tier")
|
||||
_new_model(scenario, unknown_tier, model="openai/gpt-6-sol", model_info={"service_tiers": ["ultrafast"]})
|
||||
_new_model(scenario, known_tier, model="openai/gpt-6-sol", model_info={"service_tiers": [stock_tier["id"]]})
|
||||
catalog: Final = _converged_catalog(gateway, lambda value: unknown_tier in value and known_tier in value)
|
||||
assert catalog[unknown_tier]["service_tiers"] == [_generic_tier("ultrafast")], catalog[unknown_tier]
|
||||
assert catalog[unknown_tier]["default_service_tier"] is None, catalog[unknown_tier]
|
||||
assert catalog[known_tier]["service_tiers"] == [stock_tier], catalog[known_tier]
|
||||
assert catalog[known_tier]["default_service_tier"] == stock_tier["id"], catalog[known_tier]
|
||||
assert catalog[known_tier]["display_name"] == known_tier, catalog[known_tier]
|
||||
|
||||
|
||||
def test_stock_model_empty_tiers_disable(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
name: Final = _model_name("codex-no-tiers")
|
||||
_new_model(scenario, name, model="openai/gpt-6-sol", model_info={"service_tiers": []})
|
||||
entry: Final = _converged_catalog(gateway, lambda value: name in value)[name]
|
||||
assert entry["service_tiers"] == [] and entry["default_service_tier"] is None, entry
|
||||
|
||||
|
||||
def test_two_deployments_offer_only_shared_tiers(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
shared: Final = _model_name("codex-shared")
|
||||
half_set: Final = _model_name("codex-half-set")
|
||||
_new_model(scenario, shared, model_info={"service_tiers": ["a", "b"]})
|
||||
_new_model(scenario, shared, model_info={"service_tiers": ["b", "c"]})
|
||||
_new_model(scenario, half_set, model_info={"service_tiers": ["a"]})
|
||||
_new_model(scenario, half_set)
|
||||
tiers: Final = _settled(
|
||||
lambda: _tiers_by_slug(gateway),
|
||||
lambda value: value.get(shared) == ("b",) and value.get(half_set) == (),
|
||||
)
|
||||
assert tiers[shared] == ("b",) and tiers[half_set] == (), tiers
|
||||
|
||||
|
||||
def test_invalid_tiers_offer_nothing_and_keep_listing(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
as_string: Final = scenario.model(model_info={"service_tiers": "priority"})
|
||||
stray_key: Final = scenario.model(
|
||||
model_info={"service_tiers": [{"id": "x", "name": "X", "description": "d", "stray": 1}]}
|
||||
)
|
||||
empty_id: Final = scenario.model(model_info={"service_tiers": [{"id": "", "name": "", "description": ""}]})
|
||||
control: Final = scenario.model(model_info={"service_tiers": ["priority"]})
|
||||
written: Final = (as_string, stray_key, empty_id, control)
|
||||
catalog: Final = _converged_catalog(gateway, lambda value: all(slug in value for slug in written))
|
||||
assert catalog[as_string]["service_tiers"] == [], catalog[as_string]
|
||||
assert catalog[stray_key]["service_tiers"] == [], catalog[stray_key]
|
||||
assert catalog[empty_id]["service_tiers"] == [], catalog[empty_id]
|
||||
assert catalog[control]["service_tiers"] == [_generic_tier("priority")], catalog[control]
|
||||
|
||||
|
||||
def test_blocked_deployment_tiers_are_ignored(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
name: Final = _model_name("codex-blockable")
|
||||
_new_model(scenario, name, model_info={"service_tiers": ["a", "b"]})
|
||||
narrowing: Final = _new_model(scenario, name, model_info={"service_tiers": ["b", "c"]})
|
||||
_settled(lambda: _tiers_by_slug(gateway).get(name), lambda value: value == ("b",))
|
||||
blocked: Final = gateway.request("PATCH", f"/model/{narrowing}/update", {"blocked": True})
|
||||
assert blocked.status_code == 200, blocked.text
|
||||
_settled(lambda: _tiers_by_slug(gateway).get(name), lambda value: value == ("a", "b"))
|
||||
unblocked: Final = gateway.request("PATCH", f"/model/{narrowing}/update", {"blocked": False})
|
||||
assert unblocked.status_code == 200, unblocked.text
|
||||
_settled(lambda: _tiers_by_slug(gateway).get(name), lambda value: value == ("b",))
|
||||
|
||||
|
||||
def test_restricted_key_lists_only_its_models(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
tiered: Final = scenario.model(model_info={"service_tiers": ["priority"]})
|
||||
other: Final = scenario.model(model_info={"service_tiers": ["flex"]})
|
||||
key: Final = scenario.key(models=[tiered])
|
||||
tiers: Final = eventually(
|
||||
lambda: _tiers_by_slug(gateway, key=key),
|
||||
lambda value: value == {tiered: ("priority",)},
|
||||
seconds=CONVERGENCE_SECONDS,
|
||||
)
|
||||
assert tiers == {tiered: ("priority",)}, tiers
|
||||
admin_tiers: Final = eventually(
|
||||
lambda: _tiers_by_slug(gateway), lambda value: value.get(other) == ("flex",), seconds=CONVERGENCE_SECONDS
|
||||
)
|
||||
assert admin_tiers[other] == ("flex",), admin_tiers
|
||||
|
||||
|
||||
def test_team_scoped_model_tiers(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
owning_team: Final = scenario.team()
|
||||
other_team: Final = scenario.team()
|
||||
public_name: Final = _model_name("codex-team-model")
|
||||
_new_model(scenario, public_name, model_info={"team_id": owning_team, "service_tiers": ["priority"]})
|
||||
owning_key: Final = scenario.key(team_id=owning_team)
|
||||
other_key: Final = scenario.key(team_id=other_team)
|
||||
owning_tiers: Final = _settled(
|
||||
lambda: _tiers_by_slug(gateway, key=owning_key), lambda value: value.get(public_name) == ("priority",)
|
||||
)
|
||||
assert owning_tiers[public_name] == ("priority",), owning_tiers
|
||||
admin_tiers: Final = _settled(
|
||||
lambda: _tiers_by_slug(gateway, params={"team_id": owning_team}),
|
||||
lambda value: value.get(public_name) == ("priority",),
|
||||
)
|
||||
assert admin_tiers[public_name] == ("priority",), admin_tiers
|
||||
assert public_name not in _tiers_by_slug(gateway, key=other_key), other_key[:8]
|
||||
assert public_name not in _tiers_by_slug(gateway)
|
||||
|
||||
|
||||
def test_wildcard_deployment_never_listed(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
_new_model(scenario, "openai/*", model="openai/*", model_info={"service_tiers": ["priority"]})
|
||||
flags: Final = {"return_wildcard_routes": "true"}
|
||||
plain: Final = eventually(
|
||||
lambda: _plain_ids(gateway, params=flags), lambda ids: "openai/*" in ids and model in ids
|
||||
)
|
||||
assert "openai/*" in plain, plain
|
||||
slugs: Final = eventually(
|
||||
lambda: [string_value(entry["slug"]) for entry in _catalog(gateway, params=flags)],
|
||||
lambda values: model in values,
|
||||
)
|
||||
assert all("*" not in slug for slug in slugs), slugs
|
||||
|
||||
|
||||
def test_a_tiered_model_listed_last_survives_the_byte_cut(gateway: Gateway) -> None:
|
||||
"""Codex 0.159.3 rejects a `model_catalog_url` body over `MAX_MODEL_CATALOG_BYTES` (1 MiB,
|
||||
`codex-rs/model-provider/src/models_endpoint.rs`, read on 2026-10-03), so the proxy keeps the
|
||||
entries offering a tier first and then listing order; fallback entries carry Codex's base
|
||||
prompt, so some fifty of them overrun the limit."""
|
||||
with gateway.scenario() as scenario:
|
||||
plain: Final = tuple(scenario.model() for _ in range(52))
|
||||
tiered: Final = scenario.model(model_info={"service_tiers": ["ultrafast"]})
|
||||
key: Final = scenario.key(models=[*plain, tiered])
|
||||
listing: Final = eventually(
|
||||
lambda: _plain_ids(gateway, key=key), lambda ids: set(ids) == {*plain, tiered}, seconds=CONVERGENCE_SECONDS
|
||||
)
|
||||
assert listing.index(tiered) == len(plain), listing
|
||||
response: Final = eventually(
|
||||
lambda: _catalog_response(gateway, key=key),
|
||||
lambda value: value.status_code == 200 and tiered in {entry["slug"] for entry in _entries(value)},
|
||||
seconds=CONVERGENCE_SECONDS,
|
||||
)
|
||||
entries: Final = _entries(response)
|
||||
slugs: Final = [string_value(entry["slug"]) for entry in entries]
|
||||
assert len(response.content) <= 1024 * 1024, len(response.content)
|
||||
assert 0 < len(slugs) < len(listing), (len(slugs), len(listing))
|
||||
assert slugs == [slug for slug in listing if slug in slugs], slugs
|
||||
assert slugs[-1] == tiered and _tier_ids(entries[-1]) == ("ultrafast",), entries[-1]
|
||||
assert slugs[:-1] == list(plain[: len(slugs) - 1]), slugs
|
||||
assert [entry["priority"] for entry in entries] == [listing.index(slug) for slug in slugs], slugs
|
||||
|
||||
|
||||
def test_an_oversized_tiered_model_is_left_out_alone(gateway: Gateway) -> None:
|
||||
"""A tier description longer than Codex's whole byte limit makes an entry that can never fit;
|
||||
it is taken first as a tiered entry, so the cut passes over it and still keeps the models after it."""
|
||||
with gateway.scenario() as scenario:
|
||||
plain: Final = scenario.model()
|
||||
oversized_tier: Final = {"id": "huge", "name": "Huge", "description": "x" * (1024 * 1024)}
|
||||
oversized: Final = scenario.model(model_info={"service_tiers": [oversized_tier]})
|
||||
tiered: Final = scenario.model(model_info={"service_tiers": ["ultrafast"]})
|
||||
key: Final = scenario.key(models=[plain, oversized, tiered])
|
||||
listing: Final = eventually(
|
||||
lambda: _plain_ids(gateway, key=key),
|
||||
lambda ids: set(ids) == {plain, oversized, tiered},
|
||||
seconds=CONVERGENCE_SECONDS,
|
||||
)
|
||||
assert listing == [plain, oversized, tiered], listing
|
||||
|
||||
def kept() -> tuple[int, tuple[tuple[JsonValue, JsonValue, tuple[str, ...]], ...]]:
|
||||
response: Final = _catalog_response(gateway, key=key)
|
||||
return len(response.content), tuple(
|
||||
(entry["slug"], entry["priority"], _tier_ids(entry)) for entry in _entries(response)
|
||||
)
|
||||
|
||||
expected: Final = ((plain, 0, ()), (tiered, 2, ("ultrafast",)))
|
||||
size, entries = _settled(kept, lambda value: value[1] == expected)
|
||||
assert entries == expected, entries
|
||||
assert size <= 1024 * 1024, size
|
||||
|
||||
|
||||
def test_model_info_echoes_configured_tiers(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
configured: Final = ["priority", {"id": "ultrafast", "name": "Ultra", "description": "fast lane"}]
|
||||
model: Final = scenario.model(model_info={"service_tiers": configured})
|
||||
matching: Final = eventually(
|
||||
lambda: _model_info_rows(gateway, model), lambda rows: len(rows) == 1, seconds=CONVERGENCE_SECONDS
|
||||
)
|
||||
assert len(matching) == 1, model
|
||||
assert object_value(matching[0]["model_info"])["service_tiers"] == configured, matching[0]
|
||||
|
||||
|
||||
def test_tier_update_propagates(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
name: Final = _model_name("codex-updatable")
|
||||
identity: Final = _new_model(scenario, name, model_info={"service_tiers": ["priority"]})
|
||||
_settled(lambda: _tiers_by_slug(gateway).get(name), lambda value: value == ("priority",))
|
||||
updated: Final = gateway.request(
|
||||
"PATCH", f"/model/{identity}/update", {"model_info": {"service_tiers": ["flex", "ultrafast"]}}
|
||||
)
|
||||
assert updated.status_code == 200, updated.text
|
||||
_settled(lambda: _tiers_by_slug(gateway).get(name), lambda value: value == ("flex", "ultrafast"))
|
||||
|
||||
|
||||
def test_model_delete_drops_the_slug(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
name: Final = _model_name("codex-deletable")
|
||||
identity: Final = _new_model(scenario, name, model_info={"service_tiers": ["priority"]})
|
||||
_settled(lambda: _tiers_by_slug(gateway).get(name), lambda value: value == ("priority",))
|
||||
gateway.post("/model/delete", {"id": identity})
|
||||
_settled(lambda: _tiers_by_slug(gateway).get(name), lambda value: value is None)
|
||||
|
||||
|
||||
def _sse(events: Sequence[object]) -> tuple[bytes, ...]:
|
||||
return tuple(b"data: " + json.dumps(event).encode() + b"\n\n" for event in events) + (b"data: [DONE]\n\n",)
|
||||
|
||||
|
||||
def _chat_reply(stream: bool) -> Reply:
|
||||
identity: Final = "chatcmpl-" + uuid.uuid4().hex
|
||||
if not stream:
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"id": identity,
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [
|
||||
{"index": 0, "message": {"role": "assistant", "content": "tier probe"}, "finish_reason": "stop"}
|
||||
],
|
||||
"usage": USAGE,
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
head: Final = {"id": identity, "object": "chat.completion.chunk", "created": 1, "model": "gpt-4o-mini"}
|
||||
return Reply(
|
||||
content_type="text/event-stream",
|
||||
chunks=_sse(
|
||||
(
|
||||
{**head, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "tier "}}]},
|
||||
{**head, "choices": [{"index": 0, "delta": {"content": "probe"}}]},
|
||||
{**head, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},
|
||||
{**head, "choices": [], "usage": USAGE},
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _responses_reply() -> Reply:
|
||||
identity: Final = uuid.uuid4().hex
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"id": "resp_" + identity,
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "completed",
|
||||
"model": "gpt-4o-mini",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_" + identity,
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "tier probe", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"parallel_tool_calls": False,
|
||||
"tool_choice": "auto",
|
||||
"tools": [],
|
||||
"usage": {"input_tokens": 11, "output_tokens": 4, "total_tokens": 15},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
|
||||
|
||||
def _upstream(request: Request) -> Reply:
|
||||
if request.method == "GET" and request.target.endswith("/models"):
|
||||
return Reply(body=json.dumps({"object": "list", "data": []}).encode())
|
||||
if request.target.endswith("/responses"):
|
||||
return _responses_reply()
|
||||
return _chat_reply(json.loads(request.body).get("stream") is True)
|
||||
|
||||
|
||||
def _v1(gateway: Gateway) -> str:
|
||||
return str(gateway.client.base_url).rstrip("/") + "/v1"
|
||||
|
||||
|
||||
def _llm_requests(wire: Wire) -> tuple[Request, ...]:
|
||||
return tuple(request for request in wire.drain() if request.method == "POST")
|
||||
|
||||
|
||||
def _request_bodies(requests: Sequence[Request]) -> list[dict[str, JsonValue]]:
|
||||
return [JSON_OBJECT.validate_json(request.body) for request in requests]
|
||||
|
||||
|
||||
def _texts(value: JsonValue) -> tuple[str, ...]:
|
||||
if isinstance(value, str):
|
||||
return (value,)
|
||||
if isinstance(value, list):
|
||||
return tuple(chain.from_iterable(_texts(item) for item in value))
|
||||
if isinstance(value, dict):
|
||||
return tuple(chain.from_iterable(_texts(value.get(key)) for key in ("content", "text", "input", "messages")))
|
||||
return ()
|
||||
|
||||
|
||||
def _prompt_of(body: Mapping[str, JsonValue]) -> str:
|
||||
return " ".join(_texts(body.get("input", body.get("messages"))))
|
||||
|
||||
|
||||
def _sent_prompt(body: Mapping[str, JsonValue], sent: Sequence[str]) -> str:
|
||||
text: Final = _prompt_of(body)
|
||||
matches: Final = [prompt for prompt in sent if prompt in text]
|
||||
assert len(matches) == 1, (matches, text)
|
||||
return matches[0]
|
||||
|
||||
|
||||
def _spend_ids_for_key(key: str) -> list[str]:
|
||||
rows: Final = read_rows(
|
||||
'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE api_key=%s', (sha256(key.encode()).hexdigest(),)
|
||||
)
|
||||
return [string_value(row["request_id"]) for row in rows]
|
||||
|
||||
|
||||
def test_advertised_tier_reaches_the_upstream(gateway: Gateway) -> None:
|
||||
with wire_server(_upstream) as wire, gateway.scenario() as scenario:
|
||||
written_at: Final = time.monotonic()
|
||||
model: Final = scenario.model(api_base=f"{wire.url}/v1", model_info={"service_tiers": ["priority"]})
|
||||
key: Final = scenario.key(models=[model])
|
||||
marker: Final = uuid.uuid4().hex
|
||||
advertised: Final = _settled_on_every_worker(
|
||||
lambda: _tiers_by_slug(gateway, key=key),
|
||||
lambda tiers: tiers == {model: ("priority",)},
|
||||
written_at=written_at,
|
||||
)
|
||||
assert advertised == {model: ("priority",)}, advertised
|
||||
sync_client: Final = openai.OpenAI(
|
||||
api_key=key, base_url=_v1(gateway), max_retries=0, http_client=httpx.Client(timeout=15, trust_env=False)
|
||||
)
|
||||
responses_id: Final = sync_client.responses.create(
|
||||
model=model, input=f"{marker} responses probe", service_tier="priority"
|
||||
).id
|
||||
chat_id: Final = sync_client.chat.completions.create(
|
||||
model=model, messages=[{"role": "user", "content": f"{marker} chat probe"}], service_tier="priority"
|
||||
).id
|
||||
|
||||
async def streamed() -> str:
|
||||
client: Final = openai.AsyncOpenAI(
|
||||
api_key=key,
|
||||
base_url=_v1(gateway),
|
||||
max_retries=0,
|
||||
http_client=httpx.AsyncClient(timeout=15, trust_env=False),
|
||||
)
|
||||
stream: Final = await client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": f"{marker} stream probe"}],
|
||||
service_tier="priority",
|
||||
stream=True,
|
||||
)
|
||||
chunks: Final = [chunk async for chunk in stream]
|
||||
assert chunks, "no chunks streamed"
|
||||
return chunks[0].id
|
||||
|
||||
stream_id: Final = asyncio.run(streamed())
|
||||
messages_id: Final = (
|
||||
anthropic.Anthropic(
|
||||
api_key=key,
|
||||
base_url=str(gateway.client.base_url).rstrip("/"),
|
||||
max_retries=0,
|
||||
http_client=httpx.Client(timeout=15, trust_env=False),
|
||||
)
|
||||
.messages.create(
|
||||
model=model, max_tokens=64, messages=[{"role": "user", "content": f"{marker} messages probe"}]
|
||||
)
|
||||
.id
|
||||
)
|
||||
received: Final = _llm_requests(wire)
|
||||
assert len(received) == 4, [request.target for request in received]
|
||||
targets: Final = sorted(request.target for request in received)
|
||||
assert targets == ["/v1/chat/completions"] * 2 + ["/v1/responses"] * 2, targets
|
||||
openai_shaped: Final = [
|
||||
body for body in _request_bodies(received) if f"{marker} messages probe" not in _prompt_of(body)
|
||||
]
|
||||
assert len(openai_shaped) == 3, _request_bodies(received)
|
||||
assert all(body["service_tier"] == "priority" for body in openai_shaped), openai_shaped
|
||||
ids: Final = sorted((responses_id, chat_id, stream_id, messages_id))
|
||||
assert len(set(ids)) == 4, ids
|
||||
landed: Final = eventually(lambda: _spend_ids_for_key(key), lambda values: len(values) == 4, seconds=70)
|
||||
assert sorted(landed) == ids, (landed, ids)
|
||||
|
||||
|
||||
BURST_CATALOG: Final = 12
|
||||
BURST_CHAT: Final = 8
|
||||
BURST_STREAM: Final = 8
|
||||
BURST_RESPONSES: Final = 4
|
||||
BURST_MESSAGES: Final = 4
|
||||
BURST_SIZES: Final = (
|
||||
("chat", BURST_CHAT),
|
||||
("stream", BURST_STREAM),
|
||||
("responses", BURST_RESPONSES),
|
||||
("messages", BURST_MESSAGES),
|
||||
)
|
||||
|
||||
|
||||
def _llm_tasks() -> tuple[tuple[str, int], ...]:
|
||||
return tuple(
|
||||
(kind, index) for kind, size in BURST_SIZES for index in range(size)
|
||||
) # comprehension-ok: a flat task list of two small axes
|
||||
|
||||
|
||||
def _stream_id(response: httpx.Response) -> str:
|
||||
assert response.status_code == 200, response.text
|
||||
events: Final = [
|
||||
object_value(json.loads(line.removeprefix("data:")))
|
||||
for line in response.iter_lines()
|
||||
if line.startswith("data:") and line.removeprefix("data:").strip() != "[DONE]"
|
||||
]
|
||||
assert events, response.text
|
||||
return string_value(events[0]["id"])
|
||||
|
||||
|
||||
def _json_id(response: httpx.Response) -> str:
|
||||
assert response.status_code == 200, response.text
|
||||
return string_value(JSON_OBJECT.validate_json(response.content)["id"])
|
||||
|
||||
|
||||
def test_catalog_burst_with_model_churn(gateway: Gateway) -> None:
|
||||
with wire_server(_upstream) as wire, gateway.scenario() as scenario:
|
||||
written_at: Final = time.monotonic()
|
||||
model: Final = scenario.model(api_base=f"{wire.url}/v1", model_info={"service_tiers": ["priority"]})
|
||||
key: Final = scenario.key(models=[model])
|
||||
marker: Final = uuid.uuid4().hex
|
||||
_settled_on_every_worker(
|
||||
lambda: _tiers_by_slug(gateway, key=key),
|
||||
lambda tiers: tiers == {model: ("priority",)},
|
||||
written_at=written_at,
|
||||
)
|
||||
|
||||
def prompt(kind: str, index: int) -> str:
|
||||
return f"{marker} {kind} {index}"
|
||||
|
||||
def catalog(index: int) -> tuple[str, str]:
|
||||
tiers: Final = _tiers_by_slug(gateway, key=key)
|
||||
assert tiers == {model: ("priority",)}, (index, tiers)
|
||||
return ("catalog", "")
|
||||
|
||||
def chat(index: int) -> tuple[str, str]:
|
||||
body: Final = {"model": model, "messages": [{"role": "user", "content": prompt("chat", index)}]}
|
||||
return ("chat", _json_id(gateway.request("POST", "/v1/chat/completions", body, key=key)))
|
||||
|
||||
def stream(index: int) -> tuple[str, str]:
|
||||
body: Final = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": prompt("stream", index)}],
|
||||
"stream": True,
|
||||
}
|
||||
return ("stream", _stream_id(gateway.request("POST", "/v1/chat/completions", body, key=key)))
|
||||
|
||||
def responses(index: int) -> tuple[str, str]:
|
||||
body: Final = {"model": model, "input": prompt("responses", index), "service_tier": "priority"}
|
||||
return ("responses", _json_id(gateway.request("POST", "/v1/responses", body, key=key)))
|
||||
|
||||
def messages(index: int) -> tuple[str, str]:
|
||||
body: Final = {
|
||||
"model": model,
|
||||
"max_tokens": 64,
|
||||
"messages": [{"role": "user", "content": prompt("messages", index)}],
|
||||
}
|
||||
return ("messages", _json_id(gateway.request("POST", "/v1/messages", body, key=key)))
|
||||
|
||||
def churn(index: int) -> tuple[str, str]:
|
||||
churned: Final = _new_model(
|
||||
scenario, _model_name(f"codex-churn-{index}"), model_info={"service_tiers": ["flex"]}
|
||||
)
|
||||
gateway.post("/model/delete", {"id": churned})
|
||||
return ("churn", churned)
|
||||
|
||||
tasks: Final[tuple[tuple[Callable[[int], tuple[str, str]], int], ...]] = (
|
||||
*((catalog, index) for index in range(BURST_CATALOG)),
|
||||
*((chat, index) for index in range(BURST_CHAT)),
|
||||
*((stream, index) for index in range(BURST_STREAM)),
|
||||
*((responses, index) for index in range(BURST_RESPONSES)),
|
||||
*((messages, index) for index in range(BURST_MESSAGES)),
|
||||
(churn, 0),
|
||||
(churn, 1),
|
||||
)
|
||||
with ThreadPoolExecutor(max_workers=len(tasks)) as pool:
|
||||
outcomes: Final = tuple(pool.map(lambda task: task[0](task[1]), tasks))
|
||||
llm_kinds: Final = frozenset({"chat", "stream", "responses", "messages"})
|
||||
llm_ids: Final = sorted(identity for kind, identity in outcomes if kind in llm_kinds)
|
||||
assert len(llm_ids) == BURST_CHAT + BURST_STREAM + BURST_RESPONSES + BURST_MESSAGES, outcomes
|
||||
assert len(set(llm_ids)) == len(llm_ids), llm_ids
|
||||
received: Final = _request_bodies(_llm_requests(wire))
|
||||
assert len(received) == len(llm_ids), len(received)
|
||||
prompts_sent: Final = sorted(prompt(kind, index) for kind, index in _llm_tasks())
|
||||
prompts_seen: Final = sorted(_sent_prompt(body, prompts_sent) for body in received)
|
||||
assert prompts_seen == prompts_sent, (prompts_seen, prompts_sent)
|
||||
landed: Final = eventually(
|
||||
lambda: _spend_ids_for_key(key), lambda values: len(values) == len(llm_ids), seconds=70
|
||||
)
|
||||
assert sorted(landed) == llm_ids, (landed, llm_ids)
|
||||
final_tiers: Final = _settled(
|
||||
lambda: _tiers_by_slug(gateway, key=key), lambda value: value == {model: ("priority",)}
|
||||
)
|
||||
assert final_tiers == {model: ("priority",)}, final_tiers
|
||||
430
tests/integration/configuration/test_codex_catalog_boot.py
Normal file
430
tests/integration/configuration/test_codex_catalog_boot.py
Normal file
|
|
@ -0,0 +1,430 @@
|
|||
import json
|
||||
import os
|
||||
import signal
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
from collections.abc import Mapping, Sequence
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import psutil
|
||||
import pytest
|
||||
from pydantic import JsonValue
|
||||
|
||||
from tests.integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, object_value, string_value
|
||||
from tests.integration._support.process import group_members, owned_proxy, owned_proxy_process
|
||||
|
||||
CLIENT_VERSION: Final = "0.159.3"
|
||||
FRESH_CONNECTION: Final = {"Connection": "close"}
|
||||
CONSISTENT_READS: Final = 8
|
||||
CONVERGENCE_SECONDS: Final = 60
|
||||
BURST: Final = 24
|
||||
CATALOG_FILENAME: Final = "litellm-models.json"
|
||||
SHIM_STOCK_ENV: Final = "CODEX_SHIM_STOCK"
|
||||
SHIM_LAUNCH_ENV: Final = "CODEX_SHIM_LAUNCH"
|
||||
SHIM_SCRIPT: Final = """\
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
argv = sys.argv[1:]
|
||||
if argv[-2:] == ["debug", "models"]:
|
||||
override = next((arg for arg in argv if arg.startswith("model_catalog_json=")), None)
|
||||
if override is None:
|
||||
sys.stdout.write(open(os.environ[%(stock)r], encoding="utf-8").read())
|
||||
else:
|
||||
catalog = json.load(open(json.loads(override.partition("=")[2]), encoding="utf-8"))
|
||||
if not catalog.get("models"):
|
||||
sys.stderr.write("empty model catalog\\n")
|
||||
sys.exit(2)
|
||||
sys.stdout.write(json.dumps(catalog))
|
||||
sys.exit(0)
|
||||
recorded = {name: os.environ.get(name) for name in ("OPENAI_BASE_URL", "OPENAI_API_KEY", "LITELLM_PROXY_API_KEY")}
|
||||
with open(os.environ[%(launch)r], "w", encoding="utf-8") as handle:
|
||||
json.dump({"argv": argv, "env": recorded}, handle)
|
||||
""" % {"stock": SHIM_STOCK_ENV, "launch": SHIM_LAUNCH_ENV}
|
||||
|
||||
|
||||
def _generic_tier(identity: str) -> dict[str, JsonValue]:
|
||||
return {"id": identity, "name": identity.capitalize(), "description": f"Sends service_tier={identity} upstream"}
|
||||
|
||||
|
||||
def _catalog_response(proxy: Gateway, *, key: str | None = None) -> httpx.Response:
|
||||
return proxy.request(
|
||||
"GET", "/v1/models", key=key, params={"client_version": CLIENT_VERSION}, headers=FRESH_CONNECTION
|
||||
)
|
||||
|
||||
|
||||
def _entries(response: httpx.Response) -> list[dict[str, JsonValue]]:
|
||||
assert response.status_code == 200, response.text
|
||||
body: Final = JSON_OBJECT.validate_json(response.content)
|
||||
assert set(body) == {"models"}, response.text
|
||||
models: Final = body["models"]
|
||||
assert isinstance(models, list), response.text
|
||||
return [object_value(entry) for entry in models]
|
||||
|
||||
|
||||
def _by_slug(entries: Sequence[Mapping[str, JsonValue]]) -> dict[str, dict[str, JsonValue]]:
|
||||
return {string_value(entry["slug"]): dict(entry) for entry in entries}
|
||||
|
||||
|
||||
def _write_config(directory: Path, upstream_url: str, tiered: str, plain: str, alias: str) -> Path:
|
||||
config: Final = directory / f"codex_catalog_{uuid.uuid4().hex}.yaml"
|
||||
params: Final = {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_base": f"{upstream_url}/v1",
|
||||
"api_key": "integration-provider-key",
|
||||
}
|
||||
config.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"model_list": [
|
||||
{
|
||||
"model_name": tiered,
|
||||
"litellm_params": params,
|
||||
"model_info": {
|
||||
"service_tiers": ["priority", {"id": "flex", "name": "Flex", "description": "cheap lane"}],
|
||||
"display_name": "YAML Tiered",
|
||||
"max_input_tokens": 4321,
|
||||
},
|
||||
},
|
||||
{"model_name": plain, "litellm_params": params},
|
||||
],
|
||||
"router_settings": {"model_group_alias": {alias: tiered}},
|
||||
"general_settings": {
|
||||
"master_key": "os.environ/LITELLM_MASTER_KEY",
|
||||
"database_url": "os.environ/DATABASE_URL",
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
@pytest.mark.timeout(240)
|
||||
def test_yaml_tiers_and_alias_in_owned_proxy(gateway: Gateway, tmp_path: Path) -> None:
|
||||
tiered: Final = f"codex-yaml-tiered-{uuid.uuid4().hex}"
|
||||
plain: Final = f"codex-yaml-plain-{uuid.uuid4().hex}"
|
||||
alias: Final = f"codex-yaml-alias-{uuid.uuid4().hex}"
|
||||
config: Final = _write_config(tmp_path, gateway.upstream_url, tiered, plain, alias)
|
||||
with owned_proxy(gateway, tmp_path, {"STORE_MODEL_IN_DB": "False"}, config=config, workers=2) as candidate:
|
||||
readings: Final = tuple(_entries(_catalog_response(candidate)) for _ in range(CONSISTENT_READS))
|
||||
assert all(reading == readings[0] for reading in readings), readings
|
||||
catalog: Final = _by_slug(readings[0])
|
||||
assert sorted(catalog) == sorted((tiered, plain, alias)), sorted(catalog)
|
||||
assert catalog[tiered]["service_tiers"] == [
|
||||
_generic_tier("priority"),
|
||||
{"id": "flex", "name": "Flex", "description": "cheap lane"},
|
||||
], catalog[tiered]
|
||||
assert catalog[tiered]["display_name"] == "YAML Tiered", catalog[tiered]
|
||||
assert catalog[tiered]["context_window"] == 4321, catalog[tiered]
|
||||
assert catalog[plain]["service_tiers"] == [] and catalog[plain]["display_name"] == plain, catalog[plain]
|
||||
assert catalog[alias]["service_tiers"] == catalog[tiered]["service_tiers"], catalog[alias]
|
||||
assert catalog[alias]["display_name"] == alias, catalog[alias]
|
||||
assert catalog[alias]["context_window"] == 4321, catalog[alias]
|
||||
assert string_value(catalog[alias]["base_instructions"]), alias
|
||||
|
||||
|
||||
ROOT: Final = Path(os.environ.get("INTEGRATION_PROXY_ROOT") or Path(__file__).resolve().parents[3])
|
||||
BUNDLED_STOCK_PATH: Final = ROOT / "litellm" / "proxy" / "common_utils" / "codex_bundled_models_0.159.3.json"
|
||||
STOCK_UPSTREAM: Final = "gpt-5.5"
|
||||
|
||||
|
||||
def _bundled_stock_entry(slug: str) -> dict[str, JsonValue]:
|
||||
models: Final = JSON_OBJECT.validate_json(BUNDLED_STOCK_PATH.read_bytes())["models"]
|
||||
assert isinstance(models, list), slug
|
||||
(entry,) = (object_value(model) for model in models if object_value(model)["slug"] == slug)
|
||||
return entry
|
||||
|
||||
|
||||
def _write_team_owned_config(
|
||||
directory: Path, upstream_url: str, stock_team: str, plain_team: str, owned: str, mixed: str, alias: str
|
||||
) -> Path:
|
||||
config: Final = directory / f"codex_catalog_teams_{uuid.uuid4().hex}.yaml"
|
||||
plain_params: Final = {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_base": f"{upstream_url}/v1",
|
||||
"api_key": "integration-provider-key",
|
||||
}
|
||||
stock_params: Final = {**plain_params, "model": f"openai/{STOCK_UPSTREAM}"}
|
||||
config.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"model_list": [
|
||||
{"model_name": owned, "litellm_params": stock_params, "model_info": {"team_id": stock_team}},
|
||||
{
|
||||
"model_name": owned,
|
||||
"litellm_params": plain_params,
|
||||
"model_info": {"team_id": plain_team, "service_tiers": ["flex"]},
|
||||
},
|
||||
{"model_name": mixed, "litellm_params": stock_params, "model_info": {"team_id": stock_team}},
|
||||
{"model_name": mixed, "litellm_params": plain_params, "model_info": {"service_tiers": ["flex"]}},
|
||||
],
|
||||
"router_settings": {"model_group_alias": {alias: owned}},
|
||||
"general_settings": {
|
||||
"master_key": "os.environ/LITELLM_MASTER_KEY",
|
||||
"database_url": "os.environ/DATABASE_URL",
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
@pytest.mark.timeout(240)
|
||||
def test_team_owned_deployments_of_one_name_serve_each_team_its_own_upstream(gateway: Gateway, tmp_path: Path) -> None:
|
||||
"""Two teams own a deployment of one model name (and of a `model_group_alias` of it): Codex's stock
|
||||
entry for the first deployment's upstream goes only to the team whose requests reach it, and a
|
||||
caller outside the owning team reads the deployment no team owns."""
|
||||
owned: Final = f"codex-team-owned-{uuid.uuid4().hex}"
|
||||
mixed: Final = f"codex-team-mixed-{uuid.uuid4().hex}"
|
||||
alias: Final = f"codex-team-alias-{uuid.uuid4().hex}"
|
||||
stock: Final = _bundled_stock_entry(STOCK_UPSTREAM)
|
||||
assert stock["supported_reasoning_levels"] != [] and stock["service_tiers"] != [], stock
|
||||
with gateway.scenario() as scenario:
|
||||
stock_team: Final = scenario.team()
|
||||
plain_team: Final = scenario.team()
|
||||
stock_key: Final = scenario.key(team_id=stock_team, models=[owned, alias, mixed])
|
||||
plain_key: Final = scenario.key(team_id=plain_team, models=[owned, alias, mixed])
|
||||
teamless_key: Final = scenario.key(models=[mixed])
|
||||
config: Final = _write_team_owned_config(
|
||||
tmp_path, gateway.upstream_url, stock_team, plain_team, owned, mixed, alias
|
||||
)
|
||||
with owned_proxy(gateway, tmp_path, {"STORE_MODEL_IN_DB": "False"}, config=config, workers=2) as candidate:
|
||||
|
||||
def stable_catalog(key: str) -> dict[str, dict[str, JsonValue]]:
|
||||
readings: Final = tuple(
|
||||
_entries(_catalog_response(candidate, key=key)) for _ in range(CONSISTENT_READS)
|
||||
)
|
||||
assert all(reading == readings[0] for reading in readings), readings
|
||||
return _by_slug(readings[0])
|
||||
|
||||
for_stock_team: Final = stable_catalog(stock_key)
|
||||
for_plain_team: Final = stable_catalog(plain_key)
|
||||
for_teamless: Final = stable_catalog(teamless_key)
|
||||
assert sorted(for_stock_team) == sorted(for_plain_team) == sorted((owned, alias, mixed)), sorted(for_stock_team)
|
||||
assert sorted(for_teamless) == [mixed], sorted(for_teamless)
|
||||
for slug in (owned, alias):
|
||||
assert for_stock_team[slug]["supported_reasoning_levels"] == stock["supported_reasoning_levels"], slug
|
||||
assert for_stock_team[slug]["model_messages"] == stock["model_messages"], slug
|
||||
assert for_stock_team[slug]["service_tiers"] == stock["service_tiers"], slug
|
||||
assert for_plain_team[slug]["supported_reasoning_levels"] == [], for_plain_team[slug]
|
||||
assert "model_messages" not in for_plain_team[slug], slug
|
||||
assert string_value(for_plain_team[slug]["base_instructions"]), slug
|
||||
assert for_plain_team[slug]["service_tiers"] == [_generic_tier("flex")], for_plain_team[slug]
|
||||
assert for_stock_team[mixed]["supported_reasoning_levels"] == stock["supported_reasoning_levels"], mixed
|
||||
assert for_stock_team[mixed]["service_tiers"] == [], for_stock_team[mixed]
|
||||
for outsider in (for_plain_team, for_teamless):
|
||||
assert outsider[mixed]["supported_reasoning_levels"] == [], outsider[mixed]
|
||||
assert "model_messages" not in outsider[mixed], mixed
|
||||
assert outsider[mixed]["service_tiers"] == [_generic_tier("flex")], outsider[mixed]
|
||||
|
||||
|
||||
def _worker_pids(root_pid: int) -> frozenset[int]:
|
||||
def is_worker(process: psutil.Process) -> bool:
|
||||
try:
|
||||
return "spawn_main" in " ".join(process.cmdline())
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
|
||||
return False
|
||||
|
||||
return frozenset(process.pid for process in group_members(root_pid) if is_worker(process))
|
||||
|
||||
|
||||
def _catalog_outcome(proxy: Gateway, slugs: Sequence[JsonValue]) -> str:
|
||||
try:
|
||||
response: Final = _catalog_response(proxy)
|
||||
except httpx.TransportError as error:
|
||||
return f"transport:{type(error).__name__}"
|
||||
assert response.status_code == 200, response.text
|
||||
assert sorted(entry["slug"] for entry in _entries(response)) == list(slugs), response.text
|
||||
return "ok"
|
||||
|
||||
|
||||
@pytest.mark.timeout(300)
|
||||
def test_catalog_survives_a_worker_kill(gateway: Gateway, tmp_path: Path) -> None:
|
||||
tiered: Final = f"codex-yaml-tiered-{uuid.uuid4().hex}"
|
||||
plain: Final = f"codex-yaml-plain-{uuid.uuid4().hex}"
|
||||
alias: Final = f"codex-yaml-alias-{uuid.uuid4().hex}"
|
||||
config: Final = _write_config(tmp_path, gateway.upstream_url, tiered, plain, alias)
|
||||
with owned_proxy_process(gateway, tmp_path, {"STORE_MODEL_IN_DB": "False"}, config=config, workers=2) as owned:
|
||||
candidate: Final = owned.gateway
|
||||
workers: Final = eventually(lambda: _worker_pids(owned.process.pid), lambda pids: len(pids) == 2, seconds=30)
|
||||
victim: Final = min(workers)
|
||||
slugs: Final = sorted(entry["slug"] for entry in _entries(_catalog_response(candidate)))
|
||||
assert slugs == sorted((tiered, plain, alias)), slugs
|
||||
|
||||
def attempt(index: int) -> str:
|
||||
if index == 2:
|
||||
os.kill(victim, signal.SIGKILL)
|
||||
return _catalog_outcome(candidate, slugs)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=BURST) as pool:
|
||||
outcomes: Final = tuple(pool.map(attempt, range(BURST)))
|
||||
assert outcomes.count("ok") >= 1, outcomes
|
||||
assert all(outcome == "ok" or outcome.startswith("transport:") for outcome in outcomes), outcomes
|
||||
respawned: Final = eventually(
|
||||
lambda: _worker_pids(owned.process.pid),
|
||||
lambda pids: len(pids) == 2 and victim not in pids,
|
||||
seconds=60,
|
||||
)
|
||||
assert victim not in respawned, respawned
|
||||
|
||||
def settled_burst() -> tuple[str, ...]:
|
||||
with ThreadPoolExecutor(max_workers=BURST) as pool:
|
||||
return tuple(pool.map(lambda _: _catalog_outcome(candidate, slugs), range(BURST)))
|
||||
|
||||
final: Final = eventually(settled_burst, lambda values: all(value == "ok" for value in values), seconds=40)
|
||||
assert final == ("ok",) * BURST, final
|
||||
|
||||
|
||||
def _install_shim(directory: Path, stock: Mapping[str, JsonValue]) -> tuple[Path, dict[str, str]]:
|
||||
shim_dir: Final = directory / "bin"
|
||||
shim_dir.mkdir()
|
||||
script: Final = directory / "codex_shim.py"
|
||||
script.write_text(SHIM_SCRIPT, encoding="utf-8")
|
||||
shim: Final = shim_dir / "codex"
|
||||
shim.write_text(f'#!/bin/sh\nexec "{sys.executable}" "{script}" "$@"\n', encoding="utf-8")
|
||||
shim.chmod(shim.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
stock_path: Final = directory / "stock.json"
|
||||
stock_path.write_text(json.dumps(stock), encoding="utf-8")
|
||||
launch_path: Final = directory / "launch.json"
|
||||
home: Final = directory / "home"
|
||||
home.mkdir()
|
||||
codex_home: Final = directory / "codex-home"
|
||||
environment: Final = {
|
||||
**os.environ,
|
||||
"HOME": str(home),
|
||||
"CODEX_HOME": str(codex_home),
|
||||
"PATH": f"{shim_dir}{os.pathsep}{os.environ['PATH']}",
|
||||
SHIM_STOCK_ENV: str(stock_path),
|
||||
SHIM_LAUNCH_ENV: str(launch_path),
|
||||
}
|
||||
return launch_path, environment
|
||||
|
||||
|
||||
def _run_lite_codex(proxy: Gateway, key: str, environment: Mapping[str, str]) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
[sys.executable, "-I", "-c", "from litellm.proxy.client.cli import cli; cli()", "codex", "--shim-marker"],
|
||||
env={**environment, "LITELLM_PROXY_URL": str(proxy.client.base_url).rstrip("/"), "LITELLM_PROXY_API_KEY": key},
|
||||
stdin=subprocess.DEVNULL,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=90,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def _new_model(scenario: Scenario, name: str, model_info: Mapping[str, JsonValue]) -> None:
|
||||
created: Final = scenario.gateway.post(
|
||||
"/model/new",
|
||||
{
|
||||
"model_name": name,
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_key": "integration-provider-key",
|
||||
"api_base": f"{scenario.gateway.upstream_url}/v1",
|
||||
},
|
||||
"model_info": dict(model_info),
|
||||
},
|
||||
)
|
||||
scenario.cleanups.callback(scenario.delete_model, string_value(object_value(created["model_info"])["id"]))
|
||||
|
||||
|
||||
def _launch_record(path: Path) -> dict[str, JsonValue]:
|
||||
assert path.exists(), "the shim was never launched"
|
||||
return JSON_OBJECT.validate_json(path.read_bytes())
|
||||
|
||||
|
||||
def _catalog_override(argv: Sequence[JsonValue]) -> str | None:
|
||||
overrides: Final = [
|
||||
string_value(arg) for arg in argv if isinstance(arg, str) and arg.startswith("model_catalog_json=")
|
||||
]
|
||||
assert len(overrides) <= 1, argv
|
||||
return json.loads(overrides[0].partition("=")[2]) if overrides else None
|
||||
|
||||
|
||||
def test_lite_codex_writes_the_shared_catalog(gateway: Gateway, tmp_path: Path) -> None:
|
||||
"""`lite codex` writes the installed Codex's own entry for a proxy model whose name is a stock slug and
|
||||
the fallback entry for any other; the proxy's configured tiers never reach the file (Decision 17)."""
|
||||
stock_slug: Final = f"codex-stock-{uuid.uuid4().hex}"
|
||||
stock_tiers: Final = [{"id": "priority", "name": "Fast", "description": "1.5x speed"}]
|
||||
stock: Final = {
|
||||
"models": [
|
||||
{
|
||||
"slug": stock_slug,
|
||||
"display_name": "Stock Model",
|
||||
"priority": 7,
|
||||
"visibility": "hide",
|
||||
"supported_in_api": False,
|
||||
"service_tiers": stock_tiers,
|
||||
"default_service_tier": None,
|
||||
"supported_reasoning_levels": [{"effort": "high", "description": "thinks longer"}],
|
||||
}
|
||||
]
|
||||
}
|
||||
launch_path, environment = _install_shim(tmp_path, stock)
|
||||
with gateway.scenario() as scenario:
|
||||
_new_model(scenario, stock_slug, {"service_tiers": ["ultrafast"]})
|
||||
fallback: Final = scenario.model(model_info={"service_tiers": ["priority"]})
|
||||
key: Final = scenario.key(models=[stock_slug, fallback])
|
||||
completed: Final = _run_lite_codex(gateway, key, environment)
|
||||
assert completed.returncode == 0, (completed.stdout, completed.stderr)
|
||||
assert "not syncing" not in completed.stderr, completed.stderr
|
||||
record: Final = _launch_record(launch_path)
|
||||
argv: Final = record["argv"]
|
||||
assert isinstance(argv, list) and argv[-1] == "--shim-marker", argv
|
||||
catalog_path: Final = Path(environment["CODEX_HOME"]) / CATALOG_FILENAME
|
||||
assert _catalog_override(argv) == str(catalog_path), argv
|
||||
env: Final = object_value(record["env"])
|
||||
assert env["OPENAI_BASE_URL"] == str(gateway.client.base_url).rstrip("/") + "/v1", env
|
||||
assert env["OPENAI_API_KEY"] == key, env.keys()
|
||||
written: Final = JSON_OBJECT.validate_json(catalog_path.read_bytes())
|
||||
assert set(written) == {"models"}, written.keys()
|
||||
models: Final = written["models"]
|
||||
assert isinstance(models, list), written
|
||||
file_entries: Final = _by_slug([object_value(entry) for entry in models])
|
||||
served: Final = eventually(
|
||||
lambda: _by_slug(_entries(_catalog_response(gateway, key=key))),
|
||||
lambda value: (
|
||||
stock_slug in value
|
||||
and fallback in value
|
||||
and value[stock_slug]["service_tiers"] == [_generic_tier("ultrafast")]
|
||||
and value[fallback]["service_tiers"] == [_generic_tier("priority")]
|
||||
),
|
||||
seconds=CONVERGENCE_SECONDS,
|
||||
)
|
||||
assert list(file_entries) == list(served), (list(file_entries), list(served))
|
||||
assert file_entries[stock_slug]["service_tiers"] == stock_tiers, file_entries[stock_slug]
|
||||
assert file_entries[stock_slug]["visibility"] == "list", file_entries[stock_slug]
|
||||
assert file_entries[stock_slug]["supported_in_api"] is True, file_entries[stock_slug]
|
||||
assert file_entries[stock_slug]["priority"] == list(file_entries).index(stock_slug), file_entries[stock_slug]
|
||||
assert file_entries[stock_slug]["supported_reasoning_levels"] == [
|
||||
{"effort": "high", "description": "thinks longer"}
|
||||
]
|
||||
assert file_entries[fallback]["service_tiers"] == [], file_entries[fallback]
|
||||
assert file_entries[fallback]["display_name"] == fallback, file_entries[fallback]
|
||||
assert string_value(file_entries[fallback]["base_instructions"]), fallback
|
||||
assert served[stock_slug]["service_tiers"] == [_generic_tier("ultrafast")], served[stock_slug]
|
||||
assert served[fallback]["service_tiers"] == [_generic_tier("priority")], served[fallback]
|
||||
|
||||
|
||||
def test_lite_codex_skips_a_stock_dump_without_display_name(gateway: Gateway, tmp_path: Path) -> None:
|
||||
"""A `codex debug models` dump that predates `display_name` is reported, not written, and Codex still
|
||||
launches without a catalog override."""
|
||||
stock: Final = {"models": [{"slug": "gpt-5.5", "priority": 0, "visibility": "list"}]}
|
||||
launch_path, environment = _install_shim(tmp_path, stock)
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model_info={"service_tiers": ["priority"]})
|
||||
key: Final = scenario.key(models=[model])
|
||||
completed: Final = _run_lite_codex(gateway, key, environment)
|
||||
assert completed.returncode == 0, (completed.stdout, completed.stderr)
|
||||
assert "printed no model catalog" in completed.stderr, completed.stderr
|
||||
record: Final = _launch_record(launch_path)
|
||||
argv: Final = record["argv"]
|
||||
assert isinstance(argv, list) and argv[-1] == "--shim-marker", argv
|
||||
assert _catalog_override(argv) is None, argv
|
||||
assert not (Path(environment["CODEX_HOME"]) / CATALOG_FILENAME).exists(), environment["CODEX_HOME"]
|
||||
|
|
@ -0,0 +1,128 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from pydantic import JsonValue
|
||||
|
||||
from litellm.harness.context import GatewayTarget
|
||||
from litellm.harness.endpoint import ModelEndpoint
|
||||
from litellm.harness.types import Harness
|
||||
|
||||
|
||||
def _register_scenario(gateway: Gateway, scenario_id: str, response: dict[str, JsonValue]) -> None:
|
||||
with httpx.Client(timeout=5, trust_env=False) as client:
|
||||
result: Final = client.post(
|
||||
f"{gateway.upstream_url}/__scenarios",
|
||||
json={"scenario_id": scenario_id, "response": response},
|
||||
)
|
||||
assert result.status_code == 200, result.text
|
||||
|
||||
|
||||
def _client(endpoint: ModelEndpoint) -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(
|
||||
base_url=endpoint.url,
|
||||
headers={"Authorization": f"Bearer {endpoint.token}"},
|
||||
trust_env=False,
|
||||
timeout=15,
|
||||
)
|
||||
|
||||
|
||||
_CHAT_BODY: Final = {
|
||||
"id": "chatcmpl-$UNIQUE_ID",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "scripted-model",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 11, "completion_tokens": 7, "total_tokens": 18},
|
||||
}
|
||||
|
||||
_RESPONSES_BODY: Final = {
|
||||
"id": "resp_$UNIQUE_ID",
|
||||
"object": "response",
|
||||
"status": "completed",
|
||||
"model": "scripted-model",
|
||||
"output": [{"type": "message", "content": [{"type": "output_text", "text": "hi"}]}],
|
||||
"usage": {"input_tokens": 5, "output_tokens": 3, "total_tokens": 8},
|
||||
}
|
||||
|
||||
|
||||
async def test_endpoint_forwards_json_object_bodies(gateway: Gateway) -> None:
|
||||
scenario_id: Final = f"audithend{uuid.uuid4().hex[:8]}"
|
||||
_register_scenario(
|
||||
gateway,
|
||||
scenario_id,
|
||||
{
|
||||
"content_type": "application/x-routed",
|
||||
"routes": {
|
||||
"POST /v1/chat/completions": {"content_type": "application/json", "body": _CHAT_BODY},
|
||||
"POST /v1/responses": {"content_type": "application/json", "body": _RESPONSES_BODY},
|
||||
},
|
||||
},
|
||||
)
|
||||
target: Final = GatewayTarget(api_base=f"{gateway.upstream_url}/{scenario_id}", api_key="scripted")
|
||||
async with ModelEndpoint(Harness.CODEX, "audit-model", target, api_key="x") as endpoint:
|
||||
assert isinstance(endpoint.port, int) and endpoint.port > 0
|
||||
async with _client(endpoint) as client:
|
||||
chat: Final = await client.post("/v1/chat/completions", json={"messages": []})
|
||||
assert chat.status_code == 200, chat.text
|
||||
assert chat.json()["object"] == "chat.completion", chat.text
|
||||
responses: Final = await client.post("/v1/responses", json={"input": "x"})
|
||||
assert responses.status_code == 200, responses.text
|
||||
assert responses.json()["object"] == "response", responses.text
|
||||
usage: Final = endpoint.usage.snapshot()
|
||||
assert usage.input_tokens == 11 + 5 and usage.output_tokens == 7 + 3, usage
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", (b"[1, 2]", b'"a string"', b"42", b"null"))
|
||||
async def test_endpoint_rejects_valid_json_non_object(gateway: Gateway, raw: bytes) -> None:
|
||||
async with ModelEndpoint(Harness.CODEX, "audit-model", None, api_key="x") as endpoint:
|
||||
async with _client(endpoint) as client:
|
||||
response: Final = await client.post("/v1/chat/completions", content=raw)
|
||||
assert response.status_code == 400, response.text
|
||||
assert "must be a JSON object" in response.text, response.text
|
||||
|
||||
|
||||
async def test_endpoint_rejects_unparseable_body(gateway: Gateway) -> None:
|
||||
async with ModelEndpoint(Harness.CODEX, "audit-model", None, api_key="x") as endpoint:
|
||||
async with _client(endpoint) as client:
|
||||
response: Final = await client.post("/v1/chat/completions", content=b"this is not json{")
|
||||
assert response.status_code == 400, response.text
|
||||
|
||||
|
||||
async def test_endpoint_records_usage_from_sse_stream(gateway: Gateway) -> None:
|
||||
scenario_id: Final = f"audithend{uuid.uuid4().hex[:8]}"
|
||||
_register_scenario(
|
||||
gateway,
|
||||
scenario_id,
|
||||
{
|
||||
"content_type": "text/event-stream",
|
||||
"frames": (
|
||||
'data: {"id":"chatcmpl-$UNIQUE_ID","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":"hi"},"finish_reason":null}]}',
|
||||
"data: 42",
|
||||
'data: {"id":"chatcmpl-$UNIQUE_ID","object":"chat.completion.chunk","choices":[],"usage":{"prompt_tokens":13,"completion_tokens":4}}',
|
||||
"data: [DONE]",
|
||||
),
|
||||
},
|
||||
)
|
||||
target: Final = GatewayTarget(api_base=f"{gateway.upstream_url}/{scenario_id}", api_key="scripted")
|
||||
async with ModelEndpoint(Harness.CODEX, "audit-model", target, api_key="x") as endpoint:
|
||||
async with _client(endpoint) as client:
|
||||
response: Final = await client.post("/v1/chat/completions", json={"messages": []})
|
||||
assert response.status_code == 200, response.text
|
||||
assert "[DONE]" in response.text, response.text
|
||||
usage: Final = endpoint.usage.snapshot()
|
||||
assert usage.input_tokens == 13 and usage.output_tokens == 4, usage
|
||||
|
||||
|
||||
async def test_endpoint_binds_free_port() -> None:
|
||||
async with ModelEndpoint(Harness.CODEX, "audit-model", None) as endpoint:
|
||||
port: Final = endpoint.port
|
||||
assert isinstance(port, int) and port > 0
|
||||
async with _client(endpoint) as client:
|
||||
response: Final = await client.get("/v1/models")
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["object"] == "list", response.text
|
||||
346
tests/integration/providers/test_audit_any_sweep_vector_store.py
Normal file
346
tests/integration/providers/test_audit_any_sweep_vector_store.py
Normal file
|
|
@ -0,0 +1,346 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import openai
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import Gateway, gateway_from_environment, object_value
|
||||
from integration._support.process import owned_proxy
|
||||
from integration.authorization._guardrail_opt_out import upstream_observations
|
||||
from pydantic import JsonValue
|
||||
|
||||
CONFIG_STORE_ID: Final = "vs_integration_config_store"
|
||||
FAIL_STORE_ID: Final = "vs_audit_fail_store"
|
||||
PROXY_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml"
|
||||
|
||||
|
||||
def _marker() -> str:
|
||||
return uuid.uuid4().hex
|
||||
|
||||
|
||||
def _chat_body(model: str, marker: str, store_id: str, *, stream: bool = False) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": marker}],
|
||||
"vector_store_ids": [store_id],
|
||||
"stream": stream,
|
||||
**({"stream_options": {"include_usage": True}} if stream else {}),
|
||||
}
|
||||
|
||||
|
||||
def _sdk_chat_body(body: dict[str, JsonValue]) -> dict[str, JsonValue]:
|
||||
return {
|
||||
**{key: value for key, value in body.items() if key != "vector_store_ids"},
|
||||
"extra_body": {"vector_store_ids": body["vector_store_ids"]},
|
||||
}
|
||||
|
||||
|
||||
def _search_observations(gateway: Gateway, marker: str, store_id: str) -> tuple[dict[str, JsonValue], ...]:
|
||||
path: Final = f"/vector_stores/{store_id}/search"
|
||||
return tuple(
|
||||
observation
|
||||
for observation in upstream_observations(gateway)
|
||||
if observation["path"] == path and marker in json.dumps(observation["body"])
|
||||
)
|
||||
|
||||
|
||||
def _chat_observations(gateway: Gateway, marker: str) -> tuple[dict[str, JsonValue], ...]:
|
||||
return tuple(
|
||||
observation
|
||||
for observation in upstream_observations(gateway)
|
||||
if observation["path"] == "/v1/chat/completions" and marker in json.dumps(observation["body"])
|
||||
)
|
||||
|
||||
|
||||
def _message_provider_fields(payload: dict[str, JsonValue]) -> dict[str, JsonValue]:
|
||||
choices: Final = payload["choices"]
|
||||
assert isinstance(choices, list) and choices, payload
|
||||
message: Final = object_value(object_value(choices[0])["message"])
|
||||
fields: Final = message.get("provider_specific_fields")
|
||||
assert isinstance(fields, dict), payload
|
||||
return object_value(fields)
|
||||
|
||||
|
||||
def _sse_data_lines(text: str) -> tuple[dict[str, JsonValue], ...]:
|
||||
parsed: list[dict[str, JsonValue]] = []
|
||||
for line in text.splitlines():
|
||||
if not line.startswith("data: ") or line == "data: [DONE]":
|
||||
continue
|
||||
value = json.loads(line.removeprefix("data: "))
|
||||
if isinstance(value, dict):
|
||||
parsed.append(value)
|
||||
return tuple(parsed)
|
||||
|
||||
|
||||
def _chunk_annotated(chunk: dict[str, JsonValue]) -> bool:
|
||||
choices: Final = chunk.get("choices")
|
||||
if not isinstance(choices, list) or not choices:
|
||||
return False
|
||||
delta: Final = object_value(choices[0]).get("delta")
|
||||
if not isinstance(delta, dict):
|
||||
return False
|
||||
fields: Final = object_value(delta).get("provider_specific_fields")
|
||||
return "search_results" in json.dumps(fields)
|
||||
|
||||
|
||||
def _openai_client(gateway: Gateway, key: str) -> openai.OpenAI:
|
||||
return openai.OpenAI(
|
||||
base_url=f"{gateway.client.base_url}/v1",
|
||||
api_key=key,
|
||||
http_client=httpx.Client(trust_env=False),
|
||||
)
|
||||
|
||||
|
||||
def _openai_async_client(gateway: Gateway, key: str) -> openai.AsyncOpenAI:
|
||||
return openai.AsyncOpenAI(
|
||||
base_url=f"{gateway.client.base_url}/v1",
|
||||
api_key=key,
|
||||
http_client=httpx.AsyncClient(trust_env=False),
|
||||
)
|
||||
|
||||
|
||||
def _register_scenario(gateway: Gateway, scenario_id: str, response: dict[str, JsonValue]) -> None:
|
||||
with httpx.Client(timeout=5, trust_env=False) as client:
|
||||
result: Final = client.post(
|
||||
f"{gateway.upstream_url}/__scenarios",
|
||||
json={"scenario_id": scenario_id, "response": response},
|
||||
)
|
||||
assert result.status_code == 200, result.text
|
||||
|
||||
|
||||
def test_vector_store_hook_chat_completions_raw_httpx(gateway: Gateway) -> None:
|
||||
marker: Final = _marker()
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
key: Final = scenario.key(models=[model])
|
||||
response: Final = gateway.request(
|
||||
"POST", "/v1/chat/completions", _chat_body(model, marker, CONFIG_STORE_ID), key=key
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert _message_provider_fields(response.json())["search_results"]
|
||||
assert len(_search_observations(gateway, marker, CONFIG_STORE_ID)) == 1
|
||||
|
||||
|
||||
def test_vector_store_hook_chat_completions_raw_httpx_stream(gateway: Gateway) -> None:
|
||||
marker: Final = _marker()
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
key: Final = scenario.key(models=[model])
|
||||
response: Final = gateway.request(
|
||||
"POST", "/v1/chat/completions", _chat_body(model, marker, CONFIG_STORE_ID, stream=True), key=key
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
chunks: Final = _sse_data_lines(response.text)
|
||||
assert any(_chunk_annotated(chunk) for chunk in chunks), response.text
|
||||
assert len(_search_observations(gateway, marker, CONFIG_STORE_ID)) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", (False, True))
|
||||
def test_vector_store_hook_chat_completions_openai_sdk(gateway: Gateway, stream: bool) -> None:
|
||||
marker: Final = _marker()
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
key: Final = scenario.key(models=[model])
|
||||
client: Final = _openai_client(gateway, key)
|
||||
body: Final = _chat_body(model, marker, CONFIG_STORE_ID, stream=stream)
|
||||
if stream:
|
||||
with client.chat.completions.stream(
|
||||
**{k: v for k, v in _sdk_chat_body(body).items() if k != "stream"}
|
||||
) as event_stream:
|
||||
for _event in event_stream:
|
||||
pass
|
||||
snapshot: Final = event_stream.get_final_completion().model_dump(mode="json")
|
||||
assert _message_provider_fields(snapshot)["search_results"], snapshot
|
||||
else:
|
||||
completion: Final = client.chat.completions.create(**_sdk_chat_body(body))
|
||||
assert _message_provider_fields(completion.model_dump(mode="json"))["search_results"]
|
||||
assert len(_search_observations(gateway, marker, CONFIG_STORE_ID)) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", (False, True))
|
||||
async def test_vector_store_hook_chat_completions_openai_async_sdk(gateway: Gateway, stream: bool) -> None:
|
||||
marker: Final = _marker()
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
key: Final = scenario.key(models=[model])
|
||||
client: Final = _openai_async_client(gateway, key)
|
||||
body: Final = _chat_body(model, marker, CONFIG_STORE_ID, stream=stream)
|
||||
if stream:
|
||||
stream_response: Final = await client.chat.completions.create(**_sdk_chat_body(body))
|
||||
chunks: list[dict[str, JsonValue]] = []
|
||||
async for chunk in stream_response:
|
||||
chunks.append(chunk.model_dump(mode="json"))
|
||||
assert any(_chunk_annotated(chunk) for chunk in chunks), chunks
|
||||
else:
|
||||
completion: Final = await client.chat.completions.create(**_sdk_chat_body(body))
|
||||
assert _message_provider_fields(completion.model_dump(mode="json"))["search_results"]
|
||||
assert len(_search_observations(gateway, marker, CONFIG_STORE_ID)) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", (False, True))
|
||||
def test_vector_store_hook_responses_api(gateway: Gateway, stream: bool) -> None:
|
||||
marker: Final = _marker()
|
||||
scenario_id: Final = "auditresp" if not stream else "auditrespstream"
|
||||
if stream:
|
||||
_register_scenario(
|
||||
gateway,
|
||||
scenario_id,
|
||||
{
|
||||
"content_type": "text/event-stream",
|
||||
"frames": (
|
||||
'event: response.created\ndata: {"type":"response.created","response":{"id":"$UNIQUE_ID","object":"response","status":"in_progress"}}',
|
||||
'event: response.completed\ndata: {"type":"response.completed","response":{"id":"$UNIQUE_ID","object":"response","status":"completed","output":[{"type":"message","content":[{"type":"output_text","text":"done"}]}],"usage":{"input_tokens":3,"output_tokens":4,"total_tokens":7}}}',
|
||||
),
|
||||
},
|
||||
)
|
||||
else:
|
||||
_register_scenario(
|
||||
gateway,
|
||||
scenario_id,
|
||||
{
|
||||
"content_type": "application/x-routed",
|
||||
"routes": {
|
||||
"POST /responses": {
|
||||
"content_type": "application/json",
|
||||
"body": {
|
||||
"id": "resp_$UNIQUE_ID",
|
||||
"object": "response",
|
||||
"created_at": 1700000000,
|
||||
"status": "completed",
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"output": [{"type": "message", "content": [{"type": "output_text", "text": "done"}]}],
|
||||
"usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7},
|
||||
},
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(api_base=f"{gateway.upstream_url}/{scenario_id}")
|
||||
key: Final = scenario.key(models=[model])
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/responses",
|
||||
{
|
||||
"model": model,
|
||||
"input": marker,
|
||||
"vector_store_ids": [CONFIG_STORE_ID],
|
||||
"stream": stream,
|
||||
},
|
||||
key=key,
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
if stream:
|
||||
assert "response.completed" in response.text
|
||||
else:
|
||||
payload: Final = response.json()
|
||||
assert payload["object"] == "response" and payload["id"], payload
|
||||
assert len(_search_observations(gateway, marker, CONFIG_STORE_ID)) == 1
|
||||
|
||||
|
||||
def test_vector_store_annotation_survives_cache_hit(gateway: Gateway) -> None:
|
||||
marker: Final = _marker()
|
||||
with gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
key: Final = scenario.key(models=[model])
|
||||
body: Final = _chat_body(model, marker, CONFIG_STORE_ID)
|
||||
first: Final = gateway.request("POST", "/v1/chat/completions", body, key=key)
|
||||
assert first.status_code == 200, first.text
|
||||
second: Final = gateway.request("POST", "/v1/chat/completions", body, key=key)
|
||||
assert second.status_code == 200, second.text
|
||||
assert second.json()["id"] == first.json()["id"]
|
||||
assert _message_provider_fields(second.json())["search_results"]
|
||||
assert len(_chat_observations(gateway, marker)) == 1
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FailureModeGateways:
|
||||
annotate: Gateway
|
||||
error: Gateway
|
||||
upstream: Gateway
|
||||
|
||||
|
||||
def _failure_mode_config(directory: Path, mode: str, upstream_url: str) -> Path:
|
||||
config: Final = yaml.safe_load(PROXY_CONFIG.read_text())
|
||||
config["litellm_settings"]["vector_store_search_failure_mode"] = mode
|
||||
config["vector_store_registry"] = [
|
||||
{
|
||||
"vector_store_name": "audit-fail-store",
|
||||
"litellm_params": {
|
||||
"vector_store_id": FAIL_STORE_ID,
|
||||
"custom_llm_provider": "openai",
|
||||
"api_base": f"{upstream_url}/auditnostore",
|
||||
"api_key": "integration-provider-key",
|
||||
},
|
||||
}
|
||||
]
|
||||
path: Final = directory / f"proxy_vector_store_{mode}.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def failure_gateways(tmp_path_factory: pytest.TempPathFactory) -> Iterator[FailureModeGateways]:
|
||||
with gateway_from_environment() as upstream_gateway, ExitStack() as stack:
|
||||
gateways: dict[str, Gateway] = {}
|
||||
for mode in ("annotate", "error"):
|
||||
directory: Final = tmp_path_factory.mktemp(f"audit_vs_{mode}")
|
||||
config: Final = _failure_mode_config(directory, mode, upstream_gateway.upstream_url)
|
||||
owned: Final = stack.enter_context(owned_proxy(upstream_gateway, directory, {}, config=config, workers=2))
|
||||
gateways[mode] = owned
|
||||
yield FailureModeGateways(gateways["annotate"], gateways["error"], upstream_gateway)
|
||||
|
||||
|
||||
def _register_failing_store(gateway: Gateway) -> None:
|
||||
_register_scenario(
|
||||
gateway,
|
||||
"auditnostore",
|
||||
{
|
||||
"content_type": "application/x-routed",
|
||||
"routes": {
|
||||
f"POST /vector_stores/{FAIL_STORE_ID}/search": {
|
||||
"content_type": "application/json",
|
||||
"body": {"error": {"message": "scripted store failure"}},
|
||||
"status": 500,
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_vector_store_search_failure_annotate_mode(failure_gateways: FailureModeGateways) -> None:
|
||||
_register_failing_store(failure_gateways.upstream)
|
||||
marker: Final = _marker()
|
||||
with failure_gateways.annotate.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
key: Final = scenario.key(models=[model])
|
||||
response: Final = failure_gateways.annotate.request(
|
||||
"POST", "/v1/chat/completions", _chat_body(model, marker, FAIL_STORE_ID), key=key
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
fields: Final = _message_provider_fields(response.json())
|
||||
failures: Final = fields.get("vector_store_search_failures")
|
||||
assert isinstance(failures, list) and failures, response.text
|
||||
assert FAIL_STORE_ID in json.dumps(failures) and "scripted store failure" in json.dumps(failures), failures
|
||||
|
||||
|
||||
def test_vector_store_search_failure_error_mode(failure_gateways: FailureModeGateways) -> None:
|
||||
_register_failing_store(failure_gateways.upstream)
|
||||
marker: Final = _marker()
|
||||
with failure_gateways.error.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
key: Final = scenario.key(models=[model])
|
||||
response: Final = failure_gateways.error.request(
|
||||
"POST", "/v1/chat/completions", _chat_body(model, marker, FAIL_STORE_ID), key=key
|
||||
)
|
||||
assert response.status_code != 200, response.text
|
||||
assert "error" in response.json(), response.text
|
||||
assert "vector store" in response.text.lower() or FAIL_STORE_ID in response.text, response.text
|
||||
|
|
@ -0,0 +1,340 @@
|
|||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import signal
|
||||
import threading
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from queue import SimpleQueue
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import httpx
|
||||
import psutil
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import Gateway, eventually, string_value
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.process import owned_proxy_process
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_BACKEND: Final = "anthropic.claude-haiku-4-5"
|
||||
_API_KEY: Final = "synthetic-mantle-bearer"
|
||||
_CONFIG_MODEL: Final = "bedrock-mantle-claude-chat-chaos"
|
||||
_MESSAGES_PATH: Final = "/anthropic/v1/messages"
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_MARKER: Final = re.compile(r"marker-([0-9a-f]{32})")
|
||||
_STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]")
|
||||
_ROWS_BY_CALL: Final = (
|
||||
"SELECT litellm_call_id, status FROM \"LiteLLM_SpendLogs\" WHERE litellm_call_id = ANY(string_to_array(%s, ','))"
|
||||
)
|
||||
|
||||
Endpoint = Literal["chat", "messages", "responses"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Call:
|
||||
endpoint: Endpoint
|
||||
stream: bool
|
||||
marker: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Served:
|
||||
call: _Call
|
||||
status: int
|
||||
text: str
|
||||
call_id: str
|
||||
|
||||
|
||||
def _answer(marker: str) -> str:
|
||||
return f"answer marker-{marker}"
|
||||
|
||||
|
||||
def _path(endpoint: Endpoint) -> str:
|
||||
match endpoint:
|
||||
case "chat":
|
||||
return "/v1/chat/completions"
|
||||
case "messages":
|
||||
return "/v1/messages"
|
||||
case "responses":
|
||||
return "/v1/responses"
|
||||
|
||||
|
||||
def _body(model: str, call: _Call) -> dict[str, JsonValue]:
|
||||
question: Final = f"Question marker-{call.marker}"
|
||||
common: Final[dict[str, JsonValue]] = {"model": model, "stream": call.stream}
|
||||
match call.endpoint:
|
||||
case "chat":
|
||||
return {**common, "messages": [{"role": "user", "content": question}]}
|
||||
case "messages":
|
||||
return {**common, "max_tokens": 64, "messages": [{"role": "user", "content": question}]}
|
||||
case "responses":
|
||||
return {**common, "input": question}
|
||||
|
||||
|
||||
def _mantle_reply(marker: str, stream: bool, *, abort: bool = False, pause: float = 0) -> Reply:
|
||||
message: Final = {
|
||||
"id": f"msg_bdrk_{marker}",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": _BACKEND,
|
||||
"stop_sequence": None,
|
||||
}
|
||||
if not stream:
|
||||
payload: Final = json.dumps(
|
||||
{
|
||||
**message,
|
||||
"content": [{"type": "text", "text": _answer(marker)}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 23, "output_tokens": 7},
|
||||
}
|
||||
).encode()
|
||||
return Reply(chunks=(payload,), abort_after=0) if abort else Reply(body=payload)
|
||||
opening: Final = {**message, "content": [], "stop_reason": None, "usage": {"input_tokens": 23, "output_tokens": 1}}
|
||||
events: Final = (
|
||||
("message_start", {"message": opening}),
|
||||
("content_block_start", {"index": 0, "content_block": {"type": "text", "text": ""}}),
|
||||
("content_block_delta", {"index": 0, "delta": {"type": "text_delta", "text": "answer "}}),
|
||||
("content_block_delta", {"index": 0, "delta": {"type": "text_delta", "text": f"marker-{marker}"}}),
|
||||
("content_block_stop", {"index": 0}),
|
||||
("message_delta", {"delta": {"stop_reason": "end_turn", "stop_sequence": None}, "usage": {"output_tokens": 7}}),
|
||||
("message_stop", {}),
|
||||
)
|
||||
return Reply(
|
||||
content_type="text/event-stream",
|
||||
chunks=tuple(
|
||||
f"event: {kind}\ndata: {json.dumps({'type': kind, **payload})}\n\n".encode() for kind, payload in events
|
||||
),
|
||||
abort_after=0 if abort else None,
|
||||
pause_between_chunks=pause,
|
||||
)
|
||||
|
||||
|
||||
def _marker_of(request: Request) -> str:
|
||||
found: Final = _MARKER.search(request.body.decode())
|
||||
assert found is not None, request.body
|
||||
return found.group(1)
|
||||
|
||||
|
||||
def _native_peer(aborted: frozenset[str] = frozenset(), pause: float = 0) -> Callable[[Request], Reply]:
|
||||
def respond(request: Request) -> Reply:
|
||||
assert (request.method, request.target) == ("POST", _MESSAGES_PATH), request.target
|
||||
assert request.headers["authorization"] == f"Bearer {_API_KEY}", sorted(request.headers)
|
||||
marker: Final = _marker_of(request)
|
||||
stream: Final = _JSON_OBJECT.validate_json(request.body).get("stream") is True
|
||||
return _mantle_reply(marker, stream, abort=marker in aborted, pause=pause)
|
||||
|
||||
return respond
|
||||
|
||||
|
||||
def _assert_native_requests_for(received: tuple[Request, ...], calls: tuple[_Call, ...]) -> None:
|
||||
assert {(request.method, request.target) for request in received} == {("POST", _MESSAGES_PATH)}
|
||||
assert sorted(_marker_of(request) for request in received) == sorted(call.marker for call in calls)
|
||||
|
||||
|
||||
def _spend_status_by_call(served: tuple[_Served, ...]) -> dict[str, JsonValue]:
|
||||
wanted: Final = sorted(item.call_id for item in served)
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(_ROWS_BY_CALL, (",".join(wanted),)), lambda found: len(found) >= len(wanted), seconds=70
|
||||
)
|
||||
assert sorted(string_value(row["litellm_call_id"]) for row in rows) == wanted, rows
|
||||
return {string_value(row["litellm_call_id"]): row["status"] for row in rows}
|
||||
|
||||
|
||||
async def _send(client: httpx.AsyncClient, key: str, model: str, call: _Call) -> _Served:
|
||||
async with client.stream(
|
||||
"POST",
|
||||
_path(call.endpoint),
|
||||
json=_body(model, call),
|
||||
headers={"Authorization": f"Bearer {key}", "anthropic-version": "2023-06-01"},
|
||||
) as response:
|
||||
raw: Final = await response.aread()
|
||||
return _Served(
|
||||
call=call,
|
||||
status=response.status_code,
|
||||
text=raw.decode(),
|
||||
call_id=response.headers.get("x-litellm-call-id", ""),
|
||||
)
|
||||
|
||||
|
||||
async def _burst(
|
||||
base_url: str, key: str, model: str, calls: tuple[_Call, ...], *, tolerate_transport_errors: bool = False
|
||||
) -> tuple[_Served, ...]:
|
||||
async with httpx.AsyncClient(base_url=base_url, timeout=60, trust_env=False) as client:
|
||||
results: Final = await asyncio.gather(
|
||||
*(_send(client, key, model, call) for call in calls), return_exceptions=tolerate_transport_errors
|
||||
)
|
||||
for result in results:
|
||||
assert not isinstance(result, BaseException) or isinstance(result, httpx.TransportError), repr(result)
|
||||
return tuple(result for result in results if isinstance(result, _Served))
|
||||
|
||||
|
||||
def _calls(count: int, endpoints: tuple[Endpoint, ...], stream: Callable[[int], bool]) -> tuple[_Call, ...]:
|
||||
return tuple(
|
||||
_Call(endpoint=endpoints[index % len(endpoints)], stream=stream(index), marker=uuid.uuid4().hex)
|
||||
for index in range(count)
|
||||
)
|
||||
|
||||
|
||||
def _assert_answered_with_its_own_marker(served: _Served) -> None:
|
||||
assert served.status == 200, served.text
|
||||
assert set(_MARKER.findall(served.text)) == {served.call.marker}, served.text
|
||||
|
||||
|
||||
async def test_concurrent_burst_across_endpoints_is_answered_from_the_native_route_and_logged_once_per_call(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
calls: Final = _calls(30, ("chat", "messages", "responses"), lambda index: index % 2 == 0)
|
||||
with wire_server(_native_peer()) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"bedrock_mantle/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
|
||||
served: Final = await _burst(str(gateway.client.base_url), gateway.key, model, calls)
|
||||
assert len(served) == 30
|
||||
for item in served:
|
||||
_assert_answered_with_its_own_marker(item)
|
||||
_assert_native_requests_for(wire.drain(), calls)
|
||||
assert _spend_status_by_call(served) == {item.call_id: "success" for item in served}
|
||||
|
||||
|
||||
def _unhealthy_count(gateway: Gateway, model: str) -> JsonValue:
|
||||
response: Final = gateway.request("GET", "/health", params={"model": model})
|
||||
return _JSON_OBJECT.validate_json(response.content).get("unhealthy_count")
|
||||
|
||||
|
||||
async def test_upstream_aborts_then_an_outage_fail_each_call_once_and_the_native_route_recovers(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
calls: Final = _calls(21, ("chat", "messages", "responses"), lambda index: index % 2 == 0)
|
||||
aborted: Final = frozenset(call.marker for call in calls if call.endpoint == "chat")
|
||||
during_outage: Final = (
|
||||
_Call(endpoint="chat", stream=True, marker=uuid.uuid4().hex),
|
||||
_Call(endpoint="chat", stream=False, marker=uuid.uuid4().hex),
|
||||
_Call(endpoint="messages", stream=False, marker=uuid.uuid4().hex),
|
||||
_Call(endpoint="responses", stream=False, marker=uuid.uuid4().hex),
|
||||
)
|
||||
after_restart: Final = _calls(3, ("chat", "messages", "responses"), lambda index: index == 0)
|
||||
proxy: Final = str(gateway.client.base_url)
|
||||
with gateway.scenario() as scenario:
|
||||
with wire_server(_native_peer(aborted)) as wire:
|
||||
upstream: Final = wire.url
|
||||
model: Final = scenario.model(model=f"bedrock_mantle/{_BACKEND}", api_base=upstream, api_key=_API_KEY)
|
||||
burst: Final = await _burst(proxy, gateway.key, model, calls)
|
||||
assert len(burst) == 21
|
||||
for item in burst:
|
||||
if item.call.marker in aborted:
|
||||
assert item.status == (500 if item.call.stream else 503), item.text
|
||||
assert "Response payload is not completed" in item.text and "answer marker-" not in item.text
|
||||
else:
|
||||
_assert_answered_with_its_own_marker(item)
|
||||
_assert_native_requests_for(wire.drain(), calls)
|
||||
refused: Final = await _burst(proxy, gateway.key, model, during_outage)
|
||||
assert len(refused) == 4
|
||||
for item in refused:
|
||||
assert item.status == 503, item.text
|
||||
assert "Cannot connect to host" in item.text, item.text
|
||||
assert await asyncio.to_thread(
|
||||
eventually, lambda: _unhealthy_count(gateway, model), lambda count: count == 1, 30
|
||||
)
|
||||
with wire_server(_native_peer(), port=urlsplit(upstream).port or 0) as restarted:
|
||||
recovered: Final = await _burst(proxy, gateway.key, model, after_restart)
|
||||
assert len(recovered) == 3
|
||||
for item in recovered:
|
||||
_assert_answered_with_its_own_marker(item)
|
||||
_assert_native_requests_for(restarted.drain(), after_restart)
|
||||
failed: Final = frozenset(item.call_id for item in (*burst, *refused) if item.status != 200)
|
||||
assert len(failed) == len(aborted) + 4
|
||||
assert _spend_status_by_call((*burst, *refused, *recovered)) == {
|
||||
item.call_id: "failure" if item.call_id in failed else "success" for item in (*burst, *refused, *recovered)
|
||||
}
|
||||
|
||||
|
||||
async def test_slow_native_streams_reach_every_caller_whole_and_are_logged_once(gateway: Gateway) -> None:
|
||||
calls: Final = _calls(10, ("chat", "responses"), lambda _: True)
|
||||
with wire_server(_native_peer(pause=0.3)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"bedrock_mantle/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
|
||||
served: Final = await _burst(str(gateway.client.base_url), gateway.key, model, calls)
|
||||
assert len(served) == 10
|
||||
for item in served:
|
||||
_assert_answered_with_its_own_marker(item)
|
||||
assert item.text.rstrip().endswith("data: [DONE]"), item.text
|
||||
_assert_native_requests_for(wire.drain(), calls)
|
||||
assert _spend_status_by_call(served) == {item.call_id: "success" for item in served}
|
||||
|
||||
|
||||
def _chaos_config(wire: Wire, tmp_path: Path) -> Path:
|
||||
config: Final = {
|
||||
**yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()),
|
||||
"model_list": [
|
||||
{
|
||||
"model_name": _CONFIG_MODEL,
|
||||
"litellm_params": {"model": f"bedrock_mantle/{_BACKEND}", "api_base": wire.url, "api_key": _API_KEY},
|
||||
}
|
||||
],
|
||||
}
|
||||
target: Final = tmp_path / "bedrock-mantle-claude-chat-chaos.yaml"
|
||||
target.write_text(yaml.safe_dump(config))
|
||||
return target
|
||||
|
||||
|
||||
def _live_workers(log: Path) -> tuple[int, ...]:
|
||||
started: Final = (int(pid) for pid in _STARTED_WORKER.findall(log.read_text()))
|
||||
return tuple(pid for pid in started if psutil.pid_exists(pid))
|
||||
|
||||
|
||||
def _open_upstream_connections(pid: int, upstream: str) -> int:
|
||||
port: Final = urlsplit(upstream).port
|
||||
return sum(
|
||||
1
|
||||
for connection in psutil.Process(pid).net_connections(kind="tcp")
|
||||
if connection.status == psutil.CONN_ESTABLISHED and connection.raddr and connection.raddr.port == port
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.timeout(300)
|
||||
async def test_worker_sigkill_mid_burst_leaves_the_sibling_serving_the_native_route(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
calls: Final = _calls(20, ("chat",), lambda _: False)
|
||||
release: Final = threading.Event()
|
||||
held_markers: Final[SimpleQueue[str]] = SimpleQueue()
|
||||
answer: Final = _native_peer()
|
||||
|
||||
def held(request: Request) -> Reply:
|
||||
held_markers.put(_marker_of(request))
|
||||
assert release.wait(timeout=60), "The burst was never released"
|
||||
return answer(request)
|
||||
|
||||
with wire_server(held) as wire:
|
||||
config: Final = _chaos_config(wire, tmp_path)
|
||||
with owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2) as owned:
|
||||
candidate: Final = owned.gateway
|
||||
workers: Final = eventually(lambda: _live_workers(owned.log), lambda pids: len(pids) == 2, seconds=30)
|
||||
burst: Final = asyncio.create_task(
|
||||
_burst(
|
||||
str(candidate.client.base_url), candidate.key, _CONFIG_MODEL, calls, tolerate_transport_errors=True
|
||||
)
|
||||
)
|
||||
await asyncio.to_thread(eventually, held_markers.qsize, lambda size: size == 20, 60)
|
||||
held_by: Final = MappingProxyType({pid: _open_upstream_connections(pid, wire.url) for pid in workers})
|
||||
assert sum(held_by.values()) == 20, held_by
|
||||
victim_pid, survivor_pid = sorted(workers, key=held_by.__getitem__)
|
||||
victim: Final = psutil.Process(victim_pid)
|
||||
victim.suspend()
|
||||
victim.send_signal(signal.SIGKILL)
|
||||
release.set()
|
||||
served: Final = await burst
|
||||
assert held_by[survivor_pid] >= 10, held_by
|
||||
assert len(served) == held_by[survivor_pid], (held_by, len(served))
|
||||
for item in served:
|
||||
_assert_answered_with_its_own_marker(item)
|
||||
follow_up: Final = _Call(endpoint="chat", stream=False, marker=uuid.uuid4().hex)
|
||||
(answered,) = await _burst(str(candidate.client.base_url), candidate.key, _CONFIG_MODEL, (follow_up,))
|
||||
_assert_answered_with_its_own_marker(answered)
|
||||
_assert_native_requests_for(wire.drain(), (*calls, follow_up))
|
||||
assert _spend_status_by_call((*served, answered)) == {
|
||||
item.call_id: "success" for item in (*served, answered)
|
||||
}
|
||||
1200
tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py
Normal file
1200
tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py
Normal file
File diff suppressed because it is too large
Load diff
203
tests/integration/routing/test_audit_any_sweep_chaos.py
Normal file
203
tests/integration/routing/test_audit_any_sweep_chaos.py
Normal file
|
|
@ -0,0 +1,203 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import signal
|
||||
from collections.abc import Mapping
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import psutil
|
||||
from integration._support.client import Gateway, eventually, gateway_from_environment
|
||||
from integration._support.process import owned_proxy_process
|
||||
|
||||
_RESPONSES_JSON_SCENARIO: Final = "audit-chaos-responses"
|
||||
_RESPONSES_STREAM_SCENARIO: Final = "audit-chaos-responses-stream"
|
||||
_MESSAGES_JSON_SCENARIO: Final = "audit-chaos-messages"
|
||||
_MESSAGES_STREAM_SCENARIO: Final = "audit-chaos-messages-stream"
|
||||
|
||||
_RESPONSES_BODY: Final = {
|
||||
"id": "resp_$UNIQUE_ID",
|
||||
"object": "response",
|
||||
"created_at": 1700000000,
|
||||
"status": "completed",
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"output": [{"type": "message", "content": [{"type": "output_text", "text": "done"}]}],
|
||||
"usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7},
|
||||
}
|
||||
|
||||
_ANTHROPIC_MESSAGE: Final = {
|
||||
"id": "msg_$UNIQUE_ID",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-3-5-sonnet-20241022",
|
||||
"content": [{"type": "text", "text": "scripted reply"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 9, "output_tokens": 5},
|
||||
}
|
||||
|
||||
_ANTHROPIC_EVENTS: Final = (
|
||||
'event: message_start\ndata: {"type":"message_start","message":{"id":"msg_$UNIQUE_ID","type":"message","role":"assistant","model":"claude-3-5-sonnet-20241022","content":[],"usage":{"input_tokens":9,"output_tokens":0}}}',
|
||||
'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}',
|
||||
'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"scripted reply"}}',
|
||||
'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}',
|
||||
'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":5}}',
|
||||
'event: message_stop\ndata: {"type":"message_stop"}',
|
||||
)
|
||||
|
||||
_RESPONSES_EVENTS: Final = (
|
||||
'event: response.completed\ndata: {"type":"response.completed","response":{"id":"resp_$UNIQUE_ID","object":"response","created_at":1700000000,"status":"completed","model":"openai/gpt-4o-mini","output":[{"type":"message","content":[{"type":"output_text","text":"done"}]}],"usage":{"input_tokens":3,"output_tokens":4,"total_tokens":7}}}',
|
||||
)
|
||||
|
||||
|
||||
def _register_scenarios(upstream_url: str) -> None:
|
||||
with httpx.Client(timeout=5, trust_env=False) as client:
|
||||
for scenario_id, response in (
|
||||
(
|
||||
_RESPONSES_JSON_SCENARIO,
|
||||
{
|
||||
"content_type": "application/x-routed",
|
||||
"routes": {"POST /responses": {"content_type": "application/json", "body": _RESPONSES_BODY}},
|
||||
},
|
||||
),
|
||||
(_RESPONSES_STREAM_SCENARIO, {"content_type": "text/event-stream", "frames": _RESPONSES_EVENTS}),
|
||||
(
|
||||
_MESSAGES_JSON_SCENARIO,
|
||||
{
|
||||
"content_type": "application/x-routed",
|
||||
"routes": {"POST /v1/messages": {"content_type": "application/json", "body": _ANTHROPIC_MESSAGE}},
|
||||
},
|
||||
),
|
||||
(_MESSAGES_STREAM_SCENARIO, {"content_type": "text/event-stream", "frames": _ANTHROPIC_EVENTS}),
|
||||
):
|
||||
result: Final = client.post(
|
||||
f"{upstream_url}/__scenarios", json={"scenario_id": scenario_id, "response": response}
|
||||
)
|
||||
assert result.status_code == 200, result.text
|
||||
|
||||
|
||||
def _stream_response_id(text: str) -> str | None:
|
||||
for line in text.splitlines():
|
||||
if not line.startswith("data: ") or line == "data: [DONE]":
|
||||
continue
|
||||
value: Final = json.loads(line.removeprefix("data: "))
|
||||
if isinstance(value, dict):
|
||||
if isinstance(value.get("id"), str):
|
||||
return value["id"]
|
||||
for key_name in ("response", "message"):
|
||||
nested: Final = value.get(key_name)
|
||||
if isinstance(nested, dict) and isinstance(nested.get("id"), str):
|
||||
return nested["id"]
|
||||
return None
|
||||
|
||||
|
||||
_KINDS: Final = ("chat", "chat-stream", "messages", "messages-stream", "responses", "responses-stream")
|
||||
_PATHS: Final = MappingProxyType(
|
||||
{"chat": "/v1/chat/completions", "messages": "/v1/messages", "responses": "/v1/responses"}
|
||||
)
|
||||
|
||||
|
||||
def _burst_kind(index: int) -> str:
|
||||
return f"{_KINDS[index % 3 * 2].removesuffix('-stream')}{'-stream' if index % 2 == 0 else ''}"
|
||||
|
||||
|
||||
def _fire(
|
||||
owned: Gateway, models: Mapping[str, str], key: str, kind: str, marker: str
|
||||
) -> tuple[str, int | None, str | None]:
|
||||
stream: Final = kind.endswith("-stream")
|
||||
path: Final = _PATHS[kind.removesuffix("-stream")]
|
||||
model: Final = models[f"{path.rsplit('/', 1)[-1]}{'-stream' if stream else ''}"]
|
||||
if path == "/v1/messages":
|
||||
body: Final = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": f"burst-{kind}-{marker}"}],
|
||||
"max_tokens": 64,
|
||||
"stream": stream,
|
||||
}
|
||||
elif path == "/v1/responses":
|
||||
body = {"model": model, "input": f"burst-{kind}-{marker}", "stream": stream}
|
||||
else:
|
||||
body = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": f"burst-{kind}-{marker}"}],
|
||||
"stream": stream,
|
||||
**({"stream_options": {"include_usage": True}} if stream else {}),
|
||||
}
|
||||
try:
|
||||
response: Final = owned.request("POST", path, body, key=key)
|
||||
except Exception:
|
||||
return kind, None, None
|
||||
if response.status_code != 200:
|
||||
return kind, response.status_code, None
|
||||
if stream:
|
||||
return kind, 200, _stream_response_id(response.text)
|
||||
response_id: Final = response.json().get("id")
|
||||
return kind, 200, response_id if isinstance(response_id, str) else None
|
||||
|
||||
|
||||
_WORKER_PID: Final = re.compile(r"Started server process \[(\d+)\]")
|
||||
|
||||
|
||||
def _worker_pids(log: Path) -> tuple[int, ...]:
|
||||
return tuple(int(match) for match in _WORKER_PID.findall(log.read_text()))
|
||||
|
||||
|
||||
def test_proxy_survives_worker_kill_mid_burst(tmp_path: Path) -> None:
|
||||
with gateway_from_environment() as upstream_gateway:
|
||||
directory: Final = tmp_path
|
||||
_register_scenarios(upstream_gateway.upstream_url)
|
||||
with owned_proxy_process(upstream_gateway, directory, {}, workers=2) as owned:
|
||||
with owned.gateway.scenario() as scenario:
|
||||
model: Final = scenario.model()
|
||||
responses_model: Final = scenario.model(
|
||||
api_base=f"{owned.gateway.upstream_url}/{_RESPONSES_JSON_SCENARIO}"
|
||||
)
|
||||
responses_stream_model: Final = scenario.model(
|
||||
api_base=f"{owned.gateway.upstream_url}/{_RESPONSES_STREAM_SCENARIO}"
|
||||
)
|
||||
messages_model: Final = scenario.model(
|
||||
model="anthropic/claude-3-5-sonnet-20241022",
|
||||
api_base=f"{owned.gateway.upstream_url}/{_MESSAGES_JSON_SCENARIO}",
|
||||
)
|
||||
messages_stream_model: Final = scenario.model(
|
||||
model="anthropic/claude-3-5-sonnet-20241022",
|
||||
api_base=f"{owned.gateway.upstream_url}/{_MESSAGES_STREAM_SCENARIO}",
|
||||
)
|
||||
key: Final = scenario.key(
|
||||
models=[model, responses_model, responses_stream_model, messages_model, messages_stream_model]
|
||||
)
|
||||
models: Final = MappingProxyType(
|
||||
{
|
||||
"completions": model,
|
||||
"completions-stream": model,
|
||||
"messages": messages_model,
|
||||
"messages-stream": messages_stream_model,
|
||||
"responses": responses_model,
|
||||
"responses-stream": responses_stream_model,
|
||||
}
|
||||
)
|
||||
workers: Final = eventually(lambda: _worker_pids(owned.log), lambda pids: len(pids) == 2, seconds=30)
|
||||
records: dict[int, tuple[str, int | None, str | None]] = {}
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
futures: Final = [
|
||||
(index, pool.submit(_fire, owned.gateway, models, key, _burst_kind(index), str(index)))
|
||||
for index in range(30)
|
||||
]
|
||||
psutil.Process(workers[0]).send_signal(signal.SIGKILL)
|
||||
for index, future in futures:
|
||||
records[index] = future.result()
|
||||
proxy_stamped: Final = [
|
||||
record[2]
|
||||
for record in records.values()
|
||||
if record[2] is not None and not record[0].startswith("messages")
|
||||
]
|
||||
malformed: Final = {i: r for i, r in records.items() if r[1] == 200 and r[2] is None}
|
||||
assert not malformed, f"200 responses without a well-formed body: {malformed}"
|
||||
assert len(proxy_stamped) == len(set(proxy_stamped)), records
|
||||
assert proxy_stamped, "no burst request completed"
|
||||
for kind in _KINDS:
|
||||
_, status, response_id = _fire(owned.gateway, models, key, kind, "probe")
|
||||
assert status == 200 and response_id is not None, (kind, status, records)
|
||||
|
|
@ -9,10 +9,12 @@ import tempfile
|
|||
import threading
|
||||
import time
|
||||
import typing
|
||||
from collections.abc import Mapping
|
||||
from contextlib import asynccontextmanager, contextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import httpx2
|
||||
|
|
@ -21,9 +23,12 @@ import uvicorn
|
|||
import yaml
|
||||
from mcp import ClientSession
|
||||
from mcp.client.streamable_http import streamable_http_client
|
||||
from mcp.shared._httpx_utils import create_mcp_http_client
|
||||
from mcp.types import CallToolResult
|
||||
from starlette.requests import Request
|
||||
|
||||
from tests.integration._support.wire import Reply, Request as WireRequest, Wire, wire_server
|
||||
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.proxy._experimental.mcp_server.tool_search import handle_mcp_proxy_tool
|
||||
from litellm.proxy._types import LiteLLM_ObjectPermissionTable, ProxyException, UserAPIKeyAuth
|
||||
|
|
@ -45,6 +50,173 @@ PROXY_START_TIMEOUT = 30
|
|||
PROXY_AUTHORIZATION_HEADER = "Bearer sk-1234"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cold_concurrent_schema_validation_accepts_valid_arguments() -> None:
|
||||
from litellm.proxy._experimental.mcp_server.tool_search import _tool_argument_validation_error
|
||||
|
||||
schema: Final = {
|
||||
"type": "object",
|
||||
"$defs": {"amount": {"type": "number", "multipleOf": 0.25}},
|
||||
"properties": {"amount": {"$ref": "#/$defs/amount"}},
|
||||
}
|
||||
original_affinity: Final = os.sched_getaffinity(0) if sys.platform == "linux" else None
|
||||
try:
|
||||
if original_affinity is not None:
|
||||
os.sched_setaffinity(0, {min(original_affinity)})
|
||||
for _ in range(2):
|
||||
results: Final = await asyncio.gather(
|
||||
*(_tool_argument_validation_error(schema, {"amount": 0.75}) for _ in range(8))
|
||||
)
|
||||
assert results == [None] * 8, "Cold and warm workers must accept valid concurrent tool arguments"
|
||||
finally:
|
||||
if original_affinity is not None:
|
||||
os.sched_setaffinity(0, original_affinity)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("cancel", (False, True))
|
||||
@pytest.mark.parametrize("concurrency", (1, 8))
|
||||
async def test_schema_validation_stops_expensive_work_and_recovers(cancel: bool, concurrency: int) -> None:
|
||||
import psutil
|
||||
|
||||
from litellm.proxy._experimental.mcp_server.tool_search import _tool_argument_validation_error
|
||||
|
||||
existing_children: Final = frozenset(child.pid for child in psutil.Process().children())
|
||||
warm_count: Final = min(concurrency, 4)
|
||||
assert await asyncio.gather(
|
||||
*(_tool_argument_validation_error({"type": "object"}, {}) for _ in range(warm_count))
|
||||
) == [None] * warm_count
|
||||
started: Final = time.monotonic()
|
||||
tasks: Final = tuple(
|
||||
asyncio.create_task(
|
||||
_tool_argument_validation_error(
|
||||
{"type": "object", "properties": {"value": {"type": "string", "pattern": "^(a+)+$"}}},
|
||||
{"value": "a" * 80 + "!"},
|
||||
)
|
||||
)
|
||||
for _ in range(concurrency)
|
||||
)
|
||||
group: Final = asyncio.gather(*tasks, return_exceptions=True)
|
||||
try:
|
||||
with pytest.raises(TimeoutError):
|
||||
await asyncio.wait_for(asyncio.shield(group), timeout=0.2)
|
||||
workers: Final = tuple(
|
||||
child
|
||||
for child in psutil.Process().children()
|
||||
if child.pid not in existing_children and "anyio.to_process" in child.cmdline()
|
||||
)
|
||||
assert 1 <= len(workers) <= 4
|
||||
if cancel:
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
assert all(isinstance(result, asyncio.CancelledError) for result in await group)
|
||||
else:
|
||||
assert await group == ["Tool argument validation exceeded its time limit"] * concurrency
|
||||
assert time.monotonic() - started < 35
|
||||
assert all(not worker.is_running() for worker in workers), "Cancelled validation must terminate worker CPU work"
|
||||
assert await _tool_argument_validation_error({"type": "object"}, {}) is None
|
||||
finally:
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await group
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def schema_peer() -> typing.Iterator[Wire]:
|
||||
def respond(request: WireRequest) -> Reply:
|
||||
if request.method == "GET":
|
||||
return Reply(body=b'{"type":"number"}')
|
||||
body: Final = json.loads(request.body)
|
||||
if "id" not in body:
|
||||
return Reply(status=202)
|
||||
result: Final[Mapping[str, object]]
|
||||
if body["method"] == "initialize":
|
||||
result = {
|
||||
"protocolVersion": body["params"]["protocolVersion"],
|
||||
"capabilities": {"tools": {}},
|
||||
"serverInfo": {"name": "schema-peer", "version": "1"},
|
||||
}
|
||||
elif body["method"] == "tools/list":
|
||||
result = {
|
||||
"tools": [
|
||||
{
|
||||
"name": name,
|
||||
"description": "Schema validation",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"$defs": {"amount": {"type": "number", "minimum": 0.25, "multipleOf": 0.25}},
|
||||
"properties": {
|
||||
"value": {"$ref": peer.url + "/ref" if name == "external" else "#/$defs/amount"}
|
||||
},
|
||||
},
|
||||
}
|
||||
for name in ("external", "local")
|
||||
]
|
||||
}
|
||||
else:
|
||||
assert body["method"] == "tools/call"
|
||||
result = {"content": [{"type": "text", "text": "called"}], "isError": False}
|
||||
return Reply(body=json.dumps({"jsonrpc": "2.0", "id": body["id"], "result": result}).encode())
|
||||
|
||||
with wire_server(respond) as peer:
|
||||
yield peer
|
||||
|
||||
|
||||
@pytest.mark.parametrize("external", (True, False))
|
||||
def test_proxy_schema_validation_resolves_only_local_references(
|
||||
proxy_server_url: str, schema_peer: Wire, external: bool
|
||||
) -> None:
|
||||
with httpx.Client(
|
||||
base_url=proxy_server_url,
|
||||
headers={
|
||||
"Authorization": "Bearer sk-schema",
|
||||
"Accept": "application/json, text/event-stream",
|
||||
"x-mcp-servers": "schema",
|
||||
},
|
||||
timeout=30,
|
||||
) as client:
|
||||
search: Final = _rpc_result(
|
||||
client.post(
|
||||
"/mcp/proxy",
|
||||
json={
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "tools/call",
|
||||
"params": {"name": "search_tools", "arguments": {"query": "schema"}},
|
||||
},
|
||||
)
|
||||
)
|
||||
name: Final = "schema-external" if external else "schema-local"
|
||||
tool_id: Final = next(hit["tool_id"] for hit in json.loads(search["content"][0]["text"]) if hit["name"] == name)
|
||||
schema_peer.drain()
|
||||
called: Final = _rpc_result(
|
||||
client.post(
|
||||
"/mcp/proxy",
|
||||
json={
|
||||
"jsonrpc": "2.0",
|
||||
"id": 2,
|
||||
"method": "tools/call",
|
||||
"params": {
|
||||
"name": "call_tool",
|
||||
"arguments": {"tool_id": tool_id, "arguments": {"value": 0.75}},
|
||||
},
|
||||
},
|
||||
)
|
||||
)
|
||||
observed: Final = schema_peer.drain()
|
||||
assert not any(request.method == "GET" for request in observed), (
|
||||
"Schema validation must not retrieve external references"
|
||||
)
|
||||
calls: Final = tuple(
|
||||
request
|
||||
for request in observed
|
||||
if request.method == "POST" and json.loads(request.body).get("method") == "tools/call"
|
||||
)
|
||||
assert len(calls) == (0 if external else 1), "Only valid local schemas may reach upstream execution"
|
||||
assert called["isError"] is external
|
||||
if not external:
|
||||
assert called["content"][0]["text"] == "called"
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def _clear_proxy_database_env() -> typing.Iterator[None]:
|
||||
"""Ensure local proxy DB settings don't leak into tests."""
|
||||
|
|
@ -175,10 +347,12 @@ def _proxy_server(
|
|||
tmp_path_factory: pytest.TempPathFactory,
|
||||
math_streamable_http_server: str,
|
||||
math_restricted_server: str,
|
||||
schema_peer: Wire,
|
||||
):
|
||||
config_dir = tmp_path_factory.mktemp("mcp_e2e")
|
||||
config_path = config_dir / "config.yaml"
|
||||
config = yaml.safe_load(CONFIG_TEMPLATE_PATH.read_text())
|
||||
config["mcp_servers"]["schema"] = {"transport": "http", "url": schema_peer.url + "/mcp"}
|
||||
config["mcp_servers"]["math_stdio"]["command"] = MCP_PEER_PYTHON
|
||||
config["mcp_servers"]["math_streamable_http"]["url"] = f"{math_streamable_http_server}/mcp"
|
||||
config["mcp_servers"]["math_restricted"]["url"] = f"{math_restricted_server}/mcp"
|
||||
|
|
@ -208,7 +382,7 @@ def proxy_server_url(_proxy_server: ProxyRig, setup_and_teardown: None) -> str:
|
|||
|
||||
@asynccontextmanager
|
||||
async def _http_streams(url: str, headers: dict[str, str]):
|
||||
async with httpx2.AsyncClient(headers=headers) as http_client:
|
||||
async with create_mcp_http_client(headers=headers) as http_client:
|
||||
async with streamable_http_client(url, http_client=http_client) as streams:
|
||||
yield streams
|
||||
|
||||
|
|
@ -528,6 +702,7 @@ class TestProxyMcpSchemaDiscoveryMode:
|
|||
|
||||
async def authorize_proxy_key(request: Request, api_key: str) -> UserAPIKeyAuth:
|
||||
permissions = {
|
||||
"sk-schema": LiteLLM_ObjectPermissionTable(object_permission_id="schema", mcp_servers=["schema"]),
|
||||
"sk-1234": LiteLLM_ObjectPermissionTable(object_permission_id="open", mcp_servers=["math_stdio"]),
|
||||
"sk-restricted": LiteLLM_ObjectPermissionTable(
|
||||
object_permission_id="restricted", mcp_servers=["math_restricted"]
|
||||
|
|
|
|||
|
|
@ -3114,8 +3114,8 @@ async def test_modern_upstream_requests_are_self_contained_without_initializatio
|
|||
assert client._last_initialize_instructions == "modern instructions"
|
||||
assert tuple(methods.get_nowait() for _ in range(methods.qsize())) == (
|
||||
"server/discover",
|
||||
"tools/call",
|
||||
"tools/list",
|
||||
"tools/call",
|
||||
)
|
||||
else:
|
||||
with pytest.raises((MCPError, RuntimeError), match="protocol version"):
|
||||
|
|
@ -3123,6 +3123,297 @@ async def test_modern_upstream_requests_are_self_contained_without_initializatio
|
|||
assert tuple(methods.get_nowait() for _ in range(methods.qsize())) == ("server/discover",)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("paginated", (False, True))
|
||||
@pytest.mark.parametrize("valid_annotation", (True, False))
|
||||
async def test_modern_call_emits_listed_argument_headers(paginated: bool, valid_annotation: bool) -> None:
|
||||
from collections.abc import Mapping
|
||||
from queue import SimpleQueue
|
||||
|
||||
from mcp.types import DiscoverResult, ToolsCapability
|
||||
|
||||
calls: Final[SimpleQueue[str]] = SimpleQueue()
|
||||
|
||||
def response_result(request: httpx2.Request, payload: JSONRPCRequest) -> Mapping[str, object]:
|
||||
assert payload.params is not None
|
||||
caller: Final = request.headers["authorization"].removeprefix("Bearer ")
|
||||
if payload.method == "server/discover":
|
||||
return DiscoverResult(
|
||||
supported_versions=["2026-07-28"], capabilities=ServerCapabilities(tools=ToolsCapability())
|
||||
).model_dump(by_alias=True, exclude_none=True)
|
||||
if payload.method == "tools/list":
|
||||
if paginated and "cursor" not in payload.params:
|
||||
return {
|
||||
"resultType": "complete",
|
||||
"cacheScope": "private",
|
||||
"ttlMs": 0,
|
||||
"tools": [],
|
||||
"nextCursor": "second",
|
||||
}
|
||||
return {
|
||||
"resultType": "complete",
|
||||
"cacheScope": "private",
|
||||
"ttlMs": 0,
|
||||
"tools": [
|
||||
{
|
||||
"name": "quote",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"workspace": {
|
||||
"type": "string" if valid_annotation else "number",
|
||||
"x-mcp-header": caller,
|
||||
},
|
||||
"count": {"type": "integer", "x-mcp-header": "Count"},
|
||||
"preview": {"type": "boolean", "x-mcp-header": "Preview"},
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
assert payload.method == "tools/call"
|
||||
calls.put(caller)
|
||||
assert valid_annotation, "Invalid header definitions must prevent dispatch"
|
||||
assert request.headers.get(f"mcp-param-{caller.lower()}") == caller
|
||||
assert request.headers.get("mcp-param-count") == "3"
|
||||
assert request.headers.get("mcp-param-preview") == "true"
|
||||
assert set(key for key in request.headers if key.startswith("mcp-param-")) == {
|
||||
f"mcp-param-{caller.lower()}",
|
||||
"mcp-param-count",
|
||||
"mcp-param-preview",
|
||||
}
|
||||
assert payload.params["arguments"] == {"workspace": caller, "count": 3, "preview": True}
|
||||
return {"resultType": "complete", "content": [{"type": "text", "text": "quoted"}], "isError": False}
|
||||
|
||||
def respond(request: httpx2.Request) -> httpx2.Response:
|
||||
payload: Final = _JSONRPC_MESSAGE_ADAPTER.validate_json(request.content)
|
||||
assert isinstance(payload, JSONRPCRequest)
|
||||
return httpx2.Response(
|
||||
200, json={"jsonrpc": "2.0", "id": payload.id, "result": response_result(request, payload)}
|
||||
)
|
||||
|
||||
async def call_as(caller: str) -> None:
|
||||
client: Final = _MockTransportClient(
|
||||
respond,
|
||||
server_url="https://example.com/mcp",
|
||||
protocol_version="2026-07-28",
|
||||
auth_type=MCPAuth.bearer_token,
|
||||
auth_value=caller,
|
||||
)
|
||||
params: Final = CallToolRequestParams(
|
||||
name="quote", arguments={"workspace": caller, "count": 3, "preview": True}
|
||||
)
|
||||
if not valid_annotation:
|
||||
with pytest.raises(MCPError, match="schema is unavailable"):
|
||||
await client.call_tool(params, raise_on_error=True)
|
||||
return
|
||||
result: Final = await client.call_tool(params, raise_on_error=True)
|
||||
assert result.is_error is False
|
||||
assert result.content[0].text == "quoted"
|
||||
|
||||
await asyncio.gather(call_as("Engineering"), call_as("Finance"))
|
||||
assert calls.qsize() == (2 if valid_annotation else 0)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stop", ("page_cap", "repeated_cursor"))
|
||||
@pytest.mark.parametrize("schema_available", (False, True))
|
||||
async def test_modern_call_requires_schema_from_bounded_listing(
|
||||
stop: str, schema_available: bool, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from queue import SimpleQueue
|
||||
|
||||
from mcp.types import DiscoverResult, ToolsCapability
|
||||
|
||||
monkeypatch.setattr("litellm.experimental_mcp_client.tools.MCP_TOOL_LISTING_MAX_PAGES", 2)
|
||||
methods: Final[SimpleQueue[str]] = SimpleQueue()
|
||||
|
||||
def respond(request: httpx2.Request) -> httpx2.Response:
|
||||
payload: Final = _JSONRPC_MESSAGE_ADAPTER.validate_json(request.content)
|
||||
assert isinstance(payload, JSONRPCRequest)
|
||||
methods.put(payload.method)
|
||||
if payload.method == "server/discover":
|
||||
return httpx2.Response(
|
||||
200,
|
||||
json={
|
||||
"jsonrpc": "2.0",
|
||||
"id": payload.id,
|
||||
"result": DiscoverResult(
|
||||
supported_versions=["2026-07-28"], capabilities=ServerCapabilities(tools=ToolsCapability())
|
||||
).model_dump(by_alias=True, exclude_none=True),
|
||||
},
|
||||
)
|
||||
if payload.method == "tools/list":
|
||||
assert payload.params is not None
|
||||
return httpx2.Response(
|
||||
200,
|
||||
json={
|
||||
"jsonrpc": "2.0",
|
||||
"id": payload.id,
|
||||
"result": {
|
||||
"resultType": "complete",
|
||||
"cacheScope": "private",
|
||||
"ttlMs": 0,
|
||||
"nextCursor": "loop"
|
||||
if stop == "repeated_cursor"
|
||||
else str(int(payload.params.get("cursor", "0")) + 1),
|
||||
"tools": [
|
||||
{
|
||||
"name": "quote",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {"workspace": {"type": "string", "x-mcp-header": "Workspace"}},
|
||||
},
|
||||
}
|
||||
]
|
||||
if schema_available
|
||||
else [],
|
||||
},
|
||||
},
|
||||
)
|
||||
assert payload.method == "tools/call"
|
||||
assert schema_available, "An unavailable schema must not permit dispatch without required headers"
|
||||
assert request.headers["mcp-param-workspace"] == "engineering"
|
||||
return httpx2.Response(
|
||||
200,
|
||||
json={
|
||||
"jsonrpc": "2.0",
|
||||
"id": payload.id,
|
||||
"result": {"resultType": "complete", "content": [{"type": "text", "text": "quoted"}], "isError": False},
|
||||
},
|
||||
)
|
||||
|
||||
client: Final = _MockTransportClient(respond, server_url="https://example.com/mcp", protocol_version="2026-07-28")
|
||||
params: Final = CallToolRequestParams(name="quote", arguments={"workspace": "engineering"})
|
||||
if schema_available:
|
||||
result: Final = await client.call_tool(params, raise_on_error=True)
|
||||
assert not result.is_error
|
||||
assert result.content[0].text == "quoted"
|
||||
else:
|
||||
with pytest.raises(MCPError, match="schema is unavailable") as error:
|
||||
await client.call_tool(params, raise_on_error=True)
|
||||
assert error.value.code == -32603
|
||||
observed: Final = tuple(methods.get_nowait() for _ in range(methods.qsize()))
|
||||
assert observed.count("tools/list") == 2
|
||||
assert observed.count("tools/call") == int(schema_available)
|
||||
|
||||
|
||||
def test_modern_call_uses_discovery_deadline_for_multiple_pages() -> None:
|
||||
from mcp.types import DiscoverResult, ToolsCapability
|
||||
|
||||
loop: Final = _AutojumpClockLoop()
|
||||
|
||||
async def respond(request: httpx2.Request) -> httpx2.Response:
|
||||
payload: Final = _JSONRPC_MESSAGE_ADAPTER.validate_json(request.content)
|
||||
assert isinstance(payload, JSONRPCRequest)
|
||||
if payload.method == "server/discover":
|
||||
return httpx2.Response(
|
||||
200,
|
||||
json={
|
||||
"jsonrpc": "2.0",
|
||||
"id": payload.id,
|
||||
"result": DiscoverResult(
|
||||
supported_versions=["2026-07-28"], capabilities=ServerCapabilities(tools=ToolsCapability())
|
||||
).model_dump(by_alias=True, exclude_none=True),
|
||||
},
|
||||
)
|
||||
if payload.method == "tools/list":
|
||||
await asyncio.sleep(0.06)
|
||||
assert payload.params is not None
|
||||
page: Final = (
|
||||
{"tools": [], "nextCursor": "second"}
|
||||
if "cursor" not in payload.params
|
||||
else {
|
||||
"tools": [
|
||||
{
|
||||
"name": "quote",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {"workspace": {"type": "string", "x-mcp-header": "Workspace"}},
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
return httpx2.Response(
|
||||
200,
|
||||
json={
|
||||
"jsonrpc": "2.0",
|
||||
"id": payload.id,
|
||||
"result": {"resultType": "complete", "cacheScope": "private", "ttlMs": 0, **page},
|
||||
},
|
||||
)
|
||||
assert payload.method == "tools/call"
|
||||
assert request.headers["mcp-param-workspace"] == "engineering"
|
||||
return httpx2.Response(
|
||||
200,
|
||||
json={
|
||||
"jsonrpc": "2.0",
|
||||
"id": payload.id,
|
||||
"result": {"resultType": "complete", "content": [{"type": "text", "text": "quoted"}], "isError": False},
|
||||
},
|
||||
)
|
||||
|
||||
async def run() -> None:
|
||||
client: Final = _MockTransportClient(
|
||||
respond, server_url="https://example.com/mcp", protocol_version="2026-07-28", timeout=0.1
|
||||
)
|
||||
assert [tool.name for tool in await client.list_tools(raise_on_error=True)] == ["quote"]
|
||||
result: Final = await client.call_tool(
|
||||
CallToolRequestParams(name="quote", arguments={"workspace": "engineering"}), raise_on_error=True
|
||||
)
|
||||
assert result.is_error is False
|
||||
assert result.content[0].text == "quoted"
|
||||
|
||||
try:
|
||||
loop.run_until_complete(run())
|
||||
finally:
|
||||
loop.run_until_complete(loop.shutdown_asyncgens())
|
||||
loop.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancelled_modern_catalog_load_prevents_tool_execution() -> None:
|
||||
from queue import SimpleQueue
|
||||
|
||||
from mcp.types import DiscoverResult, ToolsCapability
|
||||
|
||||
listing_started: Final = asyncio.Event()
|
||||
hold_listing: Final = asyncio.Event()
|
||||
methods: Final[SimpleQueue[str]] = SimpleQueue()
|
||||
|
||||
async def respond(request: httpx2.Request) -> httpx2.Response:
|
||||
payload: Final = _JSONRPC_MESSAGE_ADAPTER.validate_json(request.content)
|
||||
assert isinstance(payload, JSONRPCRequest)
|
||||
methods.put(payload.method)
|
||||
if payload.method == "tools/list":
|
||||
listing_started.set()
|
||||
await hold_listing.wait()
|
||||
assert payload.method == "server/discover", "A cancelled listing must not dispatch a tool call"
|
||||
discovery: Final = DiscoverResult(
|
||||
supported_versions=["2026-07-28"], capabilities=ServerCapabilities(tools=ToolsCapability())
|
||||
)
|
||||
return httpx2.Response(
|
||||
200,
|
||||
json={"jsonrpc": "2.0", "id": payload.id, "result": discovery.model_dump(by_alias=True, exclude_none=True)},
|
||||
)
|
||||
|
||||
client: Final = _MockTransportClient(respond, server_url="https://example.com/mcp", protocol_version="2026-07-28")
|
||||
task: Final = asyncio.create_task(
|
||||
client.call_tool(CallToolRequestParams(name="quote", arguments={}), raise_on_error=True)
|
||||
)
|
||||
observed_listing: Final = asyncio.create_task(listing_started.wait())
|
||||
try:
|
||||
await asyncio.wait((task, observed_listing), return_when=asyncio.FIRST_COMPLETED)
|
||||
assert listing_started.is_set()
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
assert tuple(methods.get_nowait() for _ in range(methods.qsize())) == ("server/discover", "tools/list")
|
||||
finally:
|
||||
task.cancel()
|
||||
observed_listing.cancel()
|
||||
await asyncio.gather(task, observed_listing, return_exceptions=True)
|
||||
|
||||
def test_modern_upstream_rejects_legacy_sse_transport() -> None:
|
||||
with pytest.raises(ValueError, match="transport"):
|
||||
MCPClient(protocol_version="2026-07-28", transport_type=MCPTransport.sse)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ API docs: https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.ht
|
|||
|
||||
import json
|
||||
import asyncio
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
|
||||
|
|
@ -16,6 +18,7 @@ from botocore.auth import SigV4Auth
|
|||
from botocore.awsrequest import AWSRequest
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.llms.bedrock_mantle.chat.transformation import BedrockMantleChatConfig
|
||||
from litellm.llms.bedrock.base_aws_llm import sign_request_off_loop_if_aws
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
|
@ -818,6 +821,196 @@ class TestBedrockMantleProviderResolution:
|
|||
)
|
||||
|
||||
|
||||
def _row_cost(key: str, input_tokens: int, output_tokens: int) -> float:
|
||||
row: Final[Mapping[str, float]] = litellm.model_cost[key]
|
||||
return input_tokens * row["input_cost_per_token"] + output_tokens * row["output_cost_per_token"]
|
||||
|
||||
|
||||
def _anthropic_message(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
status_code=200,
|
||||
json={
|
||||
"id": "msg_test",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "anthropic.claude-opus-5-5",
|
||||
"content": [{"type": "text", "text": "ok"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5},
|
||||
},
|
||||
request=request,
|
||||
)
|
||||
|
||||
|
||||
def _anthropic_event_stream(request: httpx.Request) -> httpx.Response:
|
||||
events = (
|
||||
(
|
||||
"message_start",
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": "msg_test",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "anthropic.claude-opus-5-5",
|
||||
"content": [],
|
||||
"usage": {"input_tokens": 10, "output_tokens": 0},
|
||||
},
|
||||
},
|
||||
),
|
||||
(
|
||||
"content_block_start",
|
||||
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
|
||||
),
|
||||
(
|
||||
"content_block_delta",
|
||||
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "streamed"}},
|
||||
),
|
||||
(
|
||||
"message_delta",
|
||||
{"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 3}},
|
||||
),
|
||||
("message_stop", {"type": "message_stop"}),
|
||||
)
|
||||
body = "".join(f"event: {name}\ndata: {json.dumps(data)}\n\n" for name, data in events)
|
||||
return httpx.Response(
|
||||
status_code=200, content=body.encode(), headers={"content-type": "text/event-stream"}, request=request
|
||||
)
|
||||
|
||||
|
||||
class TestBedrockMantleClaudeChatRoute:
|
||||
def test_claude_completion_uses_native_messages_endpoint(self, monkeypatch, local_cost_map):
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "mantle-key")
|
||||
monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False)
|
||||
handler = Mock(side_effect=_anthropic_message)
|
||||
|
||||
response = litellm.completion(
|
||||
model="bedrock_mantle/anthropic.claude-opus-5-5",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
max_tokens=64,
|
||||
aws_region_name="us-east-2",
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler))),
|
||||
)
|
||||
|
||||
sent = handler.call_args.args[0]
|
||||
assert str(sent.url) == "https://bedrock-mantle.us-east-2.api.aws/anthropic/v1/messages"
|
||||
assert sent.headers["Authorization"] == "Bearer mantle-key"
|
||||
assert json.loads(sent.content) == {
|
||||
"model": "anthropic.claude-opus-5-5",
|
||||
"messages": [{"role": "user", "content": [{"type": "text", "text": "hello"}]}],
|
||||
"max_tokens": 64,
|
||||
"anthropic_version": "bedrock-2023-05-31",
|
||||
}
|
||||
assert response.choices[0].message.content == "ok"
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(
|
||||
_row_cost("bedrock_mantle/anthropic.claude-opus-5-5", 10, 5)
|
||||
)
|
||||
assert response._hidden_params["response_cost"] != pytest.approx(_row_cost("anthropic.claude-opus-5-5", 10, 5))
|
||||
|
||||
def test_claude_streaming_completion_uses_native_messages_endpoint(self, monkeypatch, local_cost_map):
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "mantle-key")
|
||||
monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False)
|
||||
handler = Mock(side_effect=_anthropic_event_stream)
|
||||
|
||||
stream = litellm.completion(
|
||||
model="bedrock_mantle/anthropic.claude-opus-5-5",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
max_tokens=64,
|
||||
stream=True,
|
||||
aws_region_name="us-east-2",
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler))),
|
||||
)
|
||||
assert isinstance(stream, CustomStreamWrapper)
|
||||
text = "".join(chunk.choices[0].delta.content or "" for chunk in stream)
|
||||
|
||||
sent = handler.call_args.args[0]
|
||||
assert str(sent.url) == "https://bedrock-mantle.us-east-2.api.aws/anthropic/v1/messages"
|
||||
assert json.loads(sent.content)["stream"] is True
|
||||
assert text == "streamed"
|
||||
|
||||
def test_claude_region_prefixed_model_sends_bare_model_to_that_region(self, monkeypatch, local_cost_map):
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
for var in (
|
||||
"BEDROCK_MANTLE_API_KEY",
|
||||
"AWS_BEARER_TOKEN_BEDROCK",
|
||||
"BEDROCK_MANTLE_API_BASE",
|
||||
"BEDROCK_MANTLE_REGION",
|
||||
"AWS_REGION_NAME",
|
||||
"AWS_REGION",
|
||||
"AWS_PROFILE",
|
||||
):
|
||||
monkeypatch.delenv(var, raising=False)
|
||||
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAEXAMPLE")
|
||||
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "c2VjcmV0LXRlc3Qtc2VjcmV0LXRlc3Qtc2VjcmV0")
|
||||
handler = Mock(side_effect=_anthropic_message)
|
||||
|
||||
response = litellm.completion(
|
||||
model="bedrock_mantle/us-gov-west-1/anthropic.claude-opus-5-5",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
max_tokens=64,
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler))),
|
||||
)
|
||||
|
||||
sent = handler.call_args.args[0]
|
||||
assert str(sent.url) == "https://bedrock-mantle.us-gov-west-1.api.aws/anthropic/v1/messages"
|
||||
assert json.loads(sent.content)["model"] == "anthropic.claude-opus-5-5"
|
||||
assert "/us-gov-west-1/bedrock/aws4_request" in sent.headers["Authorization"]
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(
|
||||
_row_cost("bedrock_mantle/us-gov-west-1/anthropic.claude-opus-5-5", 10, 5)
|
||||
)
|
||||
|
||||
def test_non_claude_completion_stays_on_chat_completions(self, monkeypatch, local_cost_map):
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "mantle-key")
|
||||
monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False)
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
status_code=200,
|
||||
json={
|
||||
"id": "chatcmpl-test",
|
||||
"object": "chat.completion",
|
||||
"created": 1733529600,
|
||||
"model": "openai.gpt-oss-120b",
|
||||
"choices": [
|
||||
{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}
|
||||
],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
|
||||
},
|
||||
request=request,
|
||||
)
|
||||
|
||||
handler = Mock(side_effect=respond)
|
||||
response = litellm.completion(
|
||||
model="bedrock_mantle/openai.gpt-oss-120b",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
aws_region_name="us-east-2",
|
||||
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler))),
|
||||
)
|
||||
|
||||
sent = handler.call_args.args[0]
|
||||
assert str(sent.url) == "https://bedrock-mantle.us-east-2.api.aws/v1/chat/completions"
|
||||
assert response.choices[0].message.content == "ok"
|
||||
|
||||
@pytest.mark.parametrize("request_type", ["chat_completion", "embeddings"])
|
||||
def test_supported_openai_params_follow_the_route_the_model_takes(self, request_type):
|
||||
claude_params = litellm.get_supported_openai_params(
|
||||
model="anthropic.claude-opus-5-5", custom_llm_provider="bedrock_mantle", request_type=request_type
|
||||
)
|
||||
open_weight_params = litellm.get_supported_openai_params(
|
||||
model="openai.gpt-oss-120b", custom_llm_provider="bedrock_mantle", request_type=request_type
|
||||
)
|
||||
|
||||
assert claude_params is not None and open_weight_params is not None
|
||||
assert "thinking" in claude_params
|
||||
assert "thinking" not in open_weight_params
|
||||
|
||||
|
||||
class TestBedrockMantlePricing:
|
||||
"""Tests that verify Bedrock Mantle uses correct AWS Bedrock pricing, not OpenAI pricing."""
|
||||
|
||||
|
|
|
|||
|
|
@ -6637,20 +6637,11 @@ class TestMCPServerManager:
|
|||
server.mcp_info = {"server_name": "test-server"}
|
||||
|
||||
# Mock tools returned from manager (3 tools, but only 2 are allowed)
|
||||
tool1 = MagicMock()
|
||||
tool1.name = "allowed_tool_1"
|
||||
tool1.description = "This tool is allowed"
|
||||
tool1.input_schema = {}
|
||||
tool1 = MCPTool(name="allowed_tool_1", description="This tool is allowed", inputSchema={})
|
||||
|
||||
tool2 = MagicMock()
|
||||
tool2.name = "blocked_tool"
|
||||
tool2.description = "This tool is not allowed"
|
||||
tool2.input_schema = {}
|
||||
tool2 = MCPTool(name="blocked_tool", description="This tool is not allowed", inputSchema={})
|
||||
|
||||
tool3 = MagicMock()
|
||||
tool3.name = "allowed_tool_2"
|
||||
tool3.description = "This tool is also allowed"
|
||||
tool3.input_schema = {}
|
||||
tool3 = MCPTool(name="allowed_tool_2", description="This tool is also allowed", inputSchema={})
|
||||
|
||||
# Mock the global_mcp_server_manager._get_tools_from_server
|
||||
from litellm.proxy._experimental.mcp_server import rest_endpoints
|
||||
|
|
@ -6687,20 +6678,11 @@ class TestMCPServerManager:
|
|||
server.mcp_info = {"server_name": "test-server"}
|
||||
|
||||
# Mock tools returned from manager
|
||||
tool1 = MagicMock()
|
||||
tool1.name = "tool_1"
|
||||
tool1.description = "Tool 1"
|
||||
tool1.input_schema = {}
|
||||
tool1 = MCPTool(name="tool_1", description="Tool 1", inputSchema={})
|
||||
|
||||
tool2 = MagicMock()
|
||||
tool2.name = "tool_2"
|
||||
tool2.description = "Tool 2"
|
||||
tool2.input_schema = {}
|
||||
tool2 = MCPTool(name="tool_2", description="Tool 2", inputSchema={})
|
||||
|
||||
tool3 = MagicMock()
|
||||
tool3.name = "tool_3"
|
||||
tool3.description = "Tool 3"
|
||||
tool3.input_schema = {}
|
||||
tool3 = MCPTool(name="tool_3", description="Tool 3", inputSchema={})
|
||||
|
||||
# Mock the global_mcp_server_manager._get_tools_from_server
|
||||
from litellm.proxy._experimental.mcp_server import rest_endpoints
|
||||
|
|
@ -6737,15 +6719,9 @@ class TestMCPServerManager:
|
|||
server.mcp_info = {"server_name": "test-server"}
|
||||
|
||||
# Mock tools returned from manager
|
||||
tool1 = MagicMock()
|
||||
tool1.name = "tool_1"
|
||||
tool1.description = "Tool 1"
|
||||
tool1.input_schema = {}
|
||||
tool1 = MCPTool(name="tool_1", description="Tool 1", inputSchema={})
|
||||
|
||||
tool2 = MagicMock()
|
||||
tool2.name = "tool_2"
|
||||
tool2.description = "Tool 2"
|
||||
tool2.input_schema = {}
|
||||
tool2 = MCPTool(name="tool_2", description="Tool 2", inputSchema={})
|
||||
|
||||
# Mock the global_mcp_server_manager._get_tools_from_server
|
||||
from litellm.proxy._experimental.mcp_server import rest_endpoints
|
||||
|
|
|
|||
|
|
@ -11,12 +11,13 @@ Covers:
|
|||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from typing import Any, Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from pydantic import JsonValue
|
||||
from mcp.types import Tool
|
||||
|
||||
import litellm
|
||||
|
|
@ -60,6 +61,90 @@ SAMPLE_TOOLS = _make_tools(
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("schema", "arguments", "error"),
|
||||
(
|
||||
(
|
||||
{
|
||||
"type": "object",
|
||||
"$defs": {"amount": {"type": "number", "minimum": 0.25, "multipleOf": 0.25}},
|
||||
"properties": {"amount": {"$ref": "#/$defs/amount"}},
|
||||
"required": ["amount"],
|
||||
},
|
||||
{"amount": 0.75},
|
||||
None,
|
||||
),
|
||||
({"type": "object", "anyOf": [{"required": ["amount"]}, {"required": ["trace"]}]}, {"trace": "a"}, None),
|
||||
(
|
||||
{"type": "object", "properties": {"amount": {"type": "number", "minimum": 0.25}}},
|
||||
{"amount": 0.1},
|
||||
"Invalid arguments:",
|
||||
),
|
||||
({"$ref": "https://schemas.example.invalid/amount"}, {}, "Unable to validate"),
|
||||
({"$ref": "#/$defs/missing"}, {}, "Unable to validate"),
|
||||
({"$ref": "#"}, {}, "Unable to validate"),
|
||||
({"type": "not-a-type"}, {}, "Unable to validate"),
|
||||
({"properties": {"value": {"pattern": "["}}}, {"value": "a"}, "Unable to validate"),
|
||||
),
|
||||
)
|
||||
def test_offline_schema_validation_preserves_supported_constraints(
|
||||
schema: Mapping[str, JsonValue], arguments: Mapping[str, JsonValue], error: str | None
|
||||
) -> None:
|
||||
from litellm.proxy._experimental.mcp_server.tool_search import _validate_tool_arguments
|
||||
|
||||
result: Final = _validate_tool_arguments(schema, arguments)
|
||||
if error is None:
|
||||
assert result is None
|
||||
else:
|
||||
assert result is not None and result.startswith(error)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("count", "allowed"), ((4_999, True), (5_000, False)))
|
||||
def test_validation_node_limit(count: int, allowed: bool) -> None:
|
||||
from litellm.proxy._experimental.mcp_server.tool_search import _validation_limit_error
|
||||
|
||||
result: Final = _validation_limit_error({}, {str(index): None for index in range(count)})
|
||||
assert (result is None) is allowed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("size", "allowed"), ((1_048_575, True), (1_048_576, False)))
|
||||
def test_validation_text_limit(size: int, allowed: bool) -> None:
|
||||
from litellm.proxy._experimental.mcp_server.tool_search import _validation_limit_error
|
||||
|
||||
assert (_validation_limit_error({}, {"x": "a" * size}) is None) is allowed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("depth", "allowed"), ((64, True), (65, False)))
|
||||
def test_validation_depth_limit(depth: int, allowed: bool) -> None:
|
||||
from functools import reduce
|
||||
|
||||
from litellm.proxy._experimental.mcp_server.tool_search import _validation_limit_error
|
||||
|
||||
nested: Final = reduce(lambda value, _: {"x": value}, range(depth), {})
|
||||
assert (_validation_limit_error({}, nested) is None) is allowed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_oversized_arguments_are_rejected_before_worker_submission() -> None:
|
||||
from litellm.proxy._experimental.mcp_server.tool_search import _tool_argument_validation_error
|
||||
|
||||
with patch("anyio.to_process.run_sync", new_callable=AsyncMock) as submit:
|
||||
result: Final = await _tool_argument_validation_error({}, {"value": "x" * 1_048_576})
|
||||
assert result == "Tool schema or arguments exceed validation size or depth limits"
|
||||
submit.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_validation_worker_returns_tool_error() -> None:
|
||||
from anyio import BrokenWorkerProcess
|
||||
|
||||
from litellm.proxy._experimental.mcp_server.tool_search import _tool_argument_validation_error
|
||||
|
||||
with patch("anyio.to_process.run_sync", side_effect=BrokenWorkerProcess):
|
||||
result: Final = await _tool_argument_validation_error({}, {})
|
||||
assert result == "Tool argument validation worker failed"
|
||||
|
||||
|
||||
FX_TOOL = Tool(
|
||||
name="treasury-get_rates",
|
||||
description="Get foreign exchange rates for a currency pair",
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ if sys.version_info < (3, 11): # BaseExceptionGroup is a builtin only from 3.11
|
|||
import httpx
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from mcp.types import CallToolResult, TextContent
|
||||
from mcp.types import CallToolResult, TextContent, Tool
|
||||
from starlette.requests import Request
|
||||
|
||||
from litellm.constants import MCP_TOOL_LISTING_TIMEOUT
|
||||
|
|
@ -3404,17 +3404,10 @@ class TestGetToolsForSingleServer:
|
|||
from litellm.proxy._types import LiteLLM_ObjectPermissionTable
|
||||
from litellm.types.mcp import MCPTransport
|
||||
|
||||
# Create mock tools
|
||||
class MockTool:
|
||||
def __init__(self, name, description):
|
||||
self.name = name
|
||||
self.description = description
|
||||
self.input_schema = {}
|
||||
|
||||
mock_tools = [
|
||||
MockTool("tool1", "First tool"),
|
||||
MockTool("tool2", "Second tool"),
|
||||
MockTool("tool3", "Third tool"),
|
||||
Tool(name="tool1", description="First tool", inputSchema={}),
|
||||
Tool(name="tool2", description="Second tool", inputSchema={}),
|
||||
Tool(name="tool3", description="Third tool", inputSchema={}),
|
||||
]
|
||||
|
||||
# Mock _get_tools_from_server to return all tools
|
||||
|
|
@ -3466,15 +3459,9 @@ class TestGetToolsForSingleServer:
|
|||
from litellm.proxy._experimental.mcp_server.server import MCPServer
|
||||
from litellm.types.mcp import MCPTransport
|
||||
|
||||
class MockTool:
|
||||
def __init__(self, name, description):
|
||||
self.name = name
|
||||
self.description = description
|
||||
self.input_schema = {}
|
||||
|
||||
mock_tools = [
|
||||
MockTool("tool1", "First tool"),
|
||||
MockTool("tool2", "Second tool"),
|
||||
Tool(name="tool1", description="First tool", inputSchema={}),
|
||||
Tool(name="tool2", description="Second tool", inputSchema={}),
|
||||
]
|
||||
|
||||
async def fake_get_tools_from_server(**kwargs):
|
||||
|
|
@ -3514,15 +3501,9 @@ class TestGetToolsForSingleServer:
|
|||
from litellm.proxy._types import LiteLLM_ObjectPermissionTable
|
||||
from litellm.types.mcp import MCPTransport
|
||||
|
||||
class MockTool:
|
||||
def __init__(self, name, description):
|
||||
self.name = name
|
||||
self.description = description
|
||||
self.input_schema = {}
|
||||
|
||||
mock_tools = [
|
||||
MockTool("tool1", "First tool"),
|
||||
MockTool("tool2", "Second tool"),
|
||||
Tool(name="tool1", description="First tool", inputSchema={}),
|
||||
Tool(name="tool2", description="Second tool", inputSchema={}),
|
||||
]
|
||||
|
||||
async def fake_get_tools_from_server(**kwargs):
|
||||
|
|
@ -3567,15 +3548,9 @@ class TestGetToolsForSingleServer:
|
|||
from litellm.proxy._types import LiteLLM_ObjectPermissionTable
|
||||
from litellm.types.mcp import MCPTransport
|
||||
|
||||
class MockTool:
|
||||
def __init__(self, name, description):
|
||||
self.name = name
|
||||
self.description = description
|
||||
self.input_schema = {}
|
||||
|
||||
mock_tools = [
|
||||
MockTool("tool1", "First tool"),
|
||||
MockTool("tool2", "Second tool"),
|
||||
Tool(name="tool1", description="First tool", inputSchema={}),
|
||||
Tool(name="tool2", description="Second tool", inputSchema={}),
|
||||
]
|
||||
|
||||
async def fake_get_tools_from_server(**kwargs):
|
||||
|
|
@ -3620,17 +3595,11 @@ class TestGetToolsForSingleServer:
|
|||
from litellm.proxy._types import LiteLLM_ObjectPermissionTable
|
||||
from litellm.types.mcp import MCPTransport
|
||||
|
||||
class MockTool:
|
||||
def __init__(self, name, description):
|
||||
self.name = name
|
||||
self.description = description
|
||||
self.input_schema = {}
|
||||
|
||||
mock_tools = [
|
||||
MockTool("tool1", "First tool"),
|
||||
MockTool("tool2", "Second tool"),
|
||||
MockTool("tool3", "Third tool"),
|
||||
MockTool("tool4", "Fourth tool"),
|
||||
Tool(name="tool1", description="First tool", inputSchema={}),
|
||||
Tool(name="tool2", description="Second tool", inputSchema={}),
|
||||
Tool(name="tool3", description="Third tool", inputSchema={}),
|
||||
Tool(name="tool4", description="Fourth tool", inputSchema={}),
|
||||
]
|
||||
|
||||
async def fake_get_tools_from_server(**kwargs):
|
||||
|
|
@ -3682,13 +3651,7 @@ class TestGetToolsForSingleServer:
|
|||
from litellm.proxy._experimental.mcp_server.server import MCPServer
|
||||
from litellm.types.mcp import MCPTransport
|
||||
|
||||
class MockTool:
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
self.description = name
|
||||
self.input_schema = {}
|
||||
|
||||
mock_tools = [MockTool("tool1"), MockTool("tool2"), MockTool("tool3")]
|
||||
mock_tools = [Tool(name="tool1", description="tool1", inputSchema={}), Tool(name="tool2", description="tool2", inputSchema={}), Tool(name="tool3", description="tool3", inputSchema={})]
|
||||
|
||||
async def fake_get_tools_from_server(**kwargs):
|
||||
return mock_tools
|
||||
|
|
@ -4401,6 +4364,31 @@ class TestToolResponseMcpInfoEnrichment:
|
|||
"alias": None,
|
||||
}
|
||||
|
||||
def test_preserves_complete_sdk_tool_definition(self) -> None:
|
||||
from mcp.types import Tool
|
||||
|
||||
tool: Final = Tool.model_validate(
|
||||
{
|
||||
"name": "quote",
|
||||
"title": "Quote",
|
||||
"description": "Return a quote",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"$defs": {"amount": {"type": "number", "minimum": 0.25}},
|
||||
"properties": {"amount": {"$ref": "#/$defs/amount"}},
|
||||
"anyOf": [{"required": ["amount"]}, {"maxProperties": 0}],
|
||||
},
|
||||
"outputSchema": {"type": "object", "properties": {"price": {"type": "number", "multipleOf": 0.25}}},
|
||||
"annotations": {"readOnlyHint": True},
|
||||
"_meta": {"display": {"priority": 0.75}},
|
||||
"icons": [{"src": "https://example.com/icon.png"}],
|
||||
}
|
||||
)
|
||||
original: Final = tool.model_dump(by_alias=True)
|
||||
server: Final = MCPServer(server_id="quotes", name="quotes", transport=MCPTransport.http)
|
||||
response: Final = rest_endpoints._create_tool_response_objects([tool], server)[0]
|
||||
assert response.model_dump(by_alias=True, exclude={"mcp_info"}) == original
|
||||
assert tool.model_dump(by_alias=True) == original
|
||||
|
||||
class TestRestListToolsetFiltering:
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
535
tests/unit/proxy/common_utils/test_codex_model_catalog.py
Normal file
535
tests/unit/proxy/common_utils/test_codex_model_catalog.py
Normal file
|
|
@ -0,0 +1,535 @@
|
|||
import json
|
||||
from types import MappingProxyType
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.proxy.common_utils.codex_model_catalog import (
|
||||
CODEX_CATALOG_BYTE_LIMIT,
|
||||
CodexCatalogRow,
|
||||
CodexServiceTier,
|
||||
CodexStockModel,
|
||||
CodexStockUpgrade,
|
||||
bundled_codex_models,
|
||||
codex_catalog_rows,
|
||||
codex_model_list_body,
|
||||
codex_models_response_json,
|
||||
configured_service_tiers,
|
||||
)
|
||||
|
||||
_PROMPT = "base prompt"
|
||||
_FAST = {"id": "priority", "name": "Fast", "description": "2x speed, increased usage"}
|
||||
|
||||
|
||||
def _stock(slug, *, visibility="list", supported_in_api=True, upgrade=None, service_tiers=(_FAST,), default_tier=None):
|
||||
return CodexStockModel(
|
||||
slug=slug,
|
||||
display_name=slug.upper(),
|
||||
priority=99,
|
||||
visibility=visibility,
|
||||
supported_in_api=supported_in_api,
|
||||
upgrade=CodexStockUpgrade(model=upgrade) if upgrade else None,
|
||||
service_tiers=tuple(CodexServiceTier(**tier) for tier in service_tiers),
|
||||
default_service_tier=default_tier,
|
||||
model_messages={"instructions_template": f"{slug} prompt"},
|
||||
)
|
||||
|
||||
|
||||
_STOCK = MappingProxyType(
|
||||
{
|
||||
model.slug: model
|
||||
for model in (
|
||||
_stock("gpt-5.5", upgrade="gpt-6-sol"),
|
||||
_stock("gpt-6-sol", default_tier="priority"),
|
||||
_stock("gpt-daybreak", visibility="hide", supported_in_api=False),
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _row(model_id, **overrides):
|
||||
"""A one-deployment row; `service_tiers` is that deployment's raw value, `deployments` a per-deployment tuple."""
|
||||
per_deployment = (
|
||||
{"service_tiers": (overrides["service_tiers"],)}
|
||||
if "service_tiers" in overrides
|
||||
else {"service_tiers": overrides["deployments"]}
|
||||
if "deployments" in overrides
|
||||
else {}
|
||||
)
|
||||
listing = {key: value for key, value in overrides.items() if key not in ("service_tiers", "deployments")}
|
||||
return CodexCatalogRow(
|
||||
**{"id": model_id, "mode": "responses", "max_input_tokens": 272000, **listing, **per_deployment}
|
||||
)
|
||||
|
||||
|
||||
def _body(*rows, **kwargs):
|
||||
return codex_models_response_json(rows, stock=_STOCK, instructions=_PROMPT, **kwargs)
|
||||
|
||||
|
||||
def _models(*rows, **kwargs):
|
||||
return json.loads(_body(*rows, **kwargs).json)["models"]
|
||||
|
||||
|
||||
def _tiers(entry):
|
||||
return [(tier["id"], tier["name"], tier["description"]) for tier in entry["service_tiers"]]
|
||||
|
||||
|
||||
def test_body_is_exactly_codex_models_response():
|
||||
body = json.loads(_body(_row("gpt-6-astra")).json)
|
||||
|
||||
assert list(body) == ["models"]
|
||||
assert [entry["slug"] for entry in body["models"]] == ["gpt-6-astra"]
|
||||
|
||||
|
||||
def test_unknown_model_entry_carries_every_field_codex_deserializes_without_a_default():
|
||||
"""Codex 0.159.3 `codex-rs/protocol/src/openai_models.rs` ModelInfo, the fields with no
|
||||
`#[serde(default)]`, read on 2026-10-01; `base_instructions` because its `ModelsResponse`
|
||||
decoder rejects an entry with neither it nor `model_messages.instructions_template`."""
|
||||
(entry,) = _models(_row("gpt-6-astra"))
|
||||
|
||||
assert entry.keys() >= {
|
||||
"slug",
|
||||
"display_name",
|
||||
"description",
|
||||
"supported_reasoning_levels",
|
||||
"shell_type",
|
||||
"visibility",
|
||||
"supported_in_api",
|
||||
"priority",
|
||||
"availability_nux",
|
||||
"upgrade",
|
||||
"support_verbosity",
|
||||
"default_verbosity",
|
||||
"apply_patch_tool_type",
|
||||
"truncation_policy",
|
||||
"experimental_supported_tools",
|
||||
}
|
||||
assert (entry["slug"], entry["display_name"], entry["visibility"], entry["supported_in_api"]) == (
|
||||
"gpt-6-astra",
|
||||
"gpt-6-astra",
|
||||
"list",
|
||||
True,
|
||||
)
|
||||
assert (entry["context_window"], entry["base_instructions"], entry["service_tiers"]) == (272000, _PROMPT, [])
|
||||
|
||||
|
||||
def test_known_model_keeps_codex_stock_entry_listed_under_the_served_id():
|
||||
(entry,) = _models(_row("gpt-daybreak"))
|
||||
|
||||
assert entry["model_messages"] == {"instructions_template": "gpt-daybreak prompt"}
|
||||
assert "base_instructions" not in entry
|
||||
assert (entry["slug"], entry["display_name"], entry["visibility"], entry["supported_in_api"]) == (
|
||||
"gpt-daybreak",
|
||||
"GPT-DAYBREAK",
|
||||
"list",
|
||||
True,
|
||||
)
|
||||
assert _tiers(entry) == [("priority", "Fast", "2x speed, increased usage")]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("upstream", ["gpt-5.5", "openai/gpt-5.5"])
|
||||
def test_alias_of_a_known_upstream_model_takes_stock_metadata_under_its_own_name(upstream):
|
||||
(entry,) = _models(_row("team-55", upstream_model=upstream))
|
||||
|
||||
assert entry["model_messages"] == {"instructions_template": "gpt-5.5 prompt"}
|
||||
assert (entry["slug"], entry["display_name"]) == ("team-55", "team-55")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("row", "expected"),
|
||||
[
|
||||
(_row("gpt-5.5", display_name="Our 5.5"), "Our 5.5"),
|
||||
(_row("gpt-5.5"), "GPT-5.5"),
|
||||
(_row("fast-55", upstream_model="gpt-5.5"), "fast-55"),
|
||||
(_row("mystery", display_name="Mystery"), "Mystery"),
|
||||
(_row("mystery"), "mystery"),
|
||||
],
|
||||
)
|
||||
def test_display_name_is_configured_then_stock_for_its_own_slug_then_the_id(row, expected):
|
||||
(entry,) = _models(row)
|
||||
|
||||
assert entry["display_name"] == expected
|
||||
|
||||
|
||||
def test_priority_is_listing_order():
|
||||
entries = _models(_row("gpt-6-sol"), _row("mystery"), _row("gpt-5.5"))
|
||||
|
||||
assert [(entry["slug"], entry["priority"]) for entry in entries] == [
|
||||
("gpt-6-sol", 0),
|
||||
("mystery", 1),
|
||||
("gpt-5.5", 2),
|
||||
]
|
||||
|
||||
|
||||
def test_upgrade_nudge_survives_only_when_its_target_is_served():
|
||||
with_target, without_target = (
|
||||
_models(_row("gpt-5.5"), _row("gpt-6-sol"))[0],
|
||||
_models(_row("gpt-5.5"))[0],
|
||||
)
|
||||
|
||||
assert with_target["upgrade"]["model"] == "gpt-6-sol"
|
||||
assert without_target["upgrade"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("row", "expected"),
|
||||
[
|
||||
(
|
||||
_row("gpt-6-astra", service_tiers=["ultrafast"]),
|
||||
[("ultrafast", "Ultrafast", "Sends service_tier=ultrafast upstream")],
|
||||
),
|
||||
(
|
||||
_row("gpt-6-astra", service_tiers=[{"id": "ultrafast", "name": "Ultra fast", "description": "Fastest"}]),
|
||||
[("ultrafast", "Ultra fast", "Fastest")],
|
||||
),
|
||||
(
|
||||
_row("gpt-6-astra", service_tiers=[{"id": "ultrafast"}]),
|
||||
[("ultrafast", "Ultrafast", "Sends service_tier=ultrafast upstream")],
|
||||
),
|
||||
(
|
||||
_row("gpt-5.5", service_tiers=["ultrafast"]),
|
||||
[("ultrafast", "Ultrafast", "Sends service_tier=ultrafast upstream")],
|
||||
),
|
||||
(
|
||||
_row("gpt-5.5", service_tiers=["priority", "ultrafast"]),
|
||||
[
|
||||
("priority", "Fast", "2x speed, increased usage"),
|
||||
("ultrafast", "Ultrafast", "Sends service_tier=ultrafast upstream"),
|
||||
],
|
||||
),
|
||||
(_row("gpt-5.5", service_tiers=[]), []),
|
||||
(
|
||||
_row("gpt-5.5", service_tiers=["ultrafast", "priority", "ultrafast", {"id": "priority"}]),
|
||||
[
|
||||
("ultrafast", "Ultrafast", "Sends service_tier=ultrafast upstream"),
|
||||
("priority", "Fast", "2x speed, increased usage"),
|
||||
],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_configured_service_tiers_replace_stock_tiers_and_a_known_id_keeps_its_codex_name(row, expected):
|
||||
(entry,) = _models(row)
|
||||
|
||||
assert _tiers(entry) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"invalid", ["ultrafast", [""], [" "], [1], [{"name": "no id"}], [{"id": "x", "bogus": 1}], {"id": "x"}]
|
||||
)
|
||||
def test_invalid_service_tiers_offer_no_tier_even_on_a_model_codex_ships_tiers_for(invalid):
|
||||
"""Keeping the stock tiers would offer Codex's `/fast` on a model whose operator declared something else."""
|
||||
stock_entry, fallback_entry = _models(
|
||||
_row("gpt-5.5", service_tiers=invalid), _row("mystery", service_tiers=invalid)
|
||||
)
|
||||
|
||||
assert _tiers(stock_entry) == []
|
||||
assert fallback_entry["service_tiers"] == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("deployments", "expected"),
|
||||
[
|
||||
((["ultrafast"], ["ultrafast"]), [("ultrafast", "Ultrafast", "Sends service_tier=ultrafast upstream")]),
|
||||
(
|
||||
(["priority", "ultrafast"], ["ultrafast", "flex", "priority"]),
|
||||
[
|
||||
("priority", "Fast", "2x speed, increased usage"),
|
||||
("ultrafast", "Ultrafast", "Sends service_tier=ultrafast upstream"),
|
||||
],
|
||||
),
|
||||
(
|
||||
(["priority", "ultrafast"], ["ultrafast"]),
|
||||
[("ultrafast", "Ultrafast", "Sends service_tier=ultrafast upstream")],
|
||||
),
|
||||
((["ultrafast"], ["priority"]), []),
|
||||
((["ultrafast"], None), []),
|
||||
((None, ["ultrafast"], None), []),
|
||||
],
|
||||
)
|
||||
def test_a_tier_is_offered_only_when_every_deployment_of_the_model_lists_it(deployments, expected):
|
||||
"""Any deployment can serve the request, so the first deployment's order is kept but a tier one of them
|
||||
lacks, or a deployment that lists none, offers nothing Codex could send to the wrong deployment."""
|
||||
(entry,) = _models(_row("gpt-5.5", deployments=deployments))
|
||||
|
||||
assert _tiers(entry) == expected
|
||||
|
||||
|
||||
def test_deployments_that_all_leave_service_tiers_unset_keep_the_stock_tiers():
|
||||
stock_entry, fallback_entry = _models(
|
||||
_row("gpt-5.5", deployments=(None, None)), _row("mystery", deployments=(None, None))
|
||||
)
|
||||
|
||||
assert _tiers(stock_entry) == [("priority", "Fast", "2x speed, increased usage")]
|
||||
assert fallback_entry["service_tiers"] == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deployments", [(["ultrafast"], "not-a-list"), ("not-a-list", ["ultrafast"]), (["priority"], "not-a-list")]
|
||||
)
|
||||
def test_an_invalid_value_on_one_deployment_offers_no_tier_for_the_whole_model(deployments):
|
||||
"""The deployment with the typo lists nothing Codex could send, and the stock `/fast` stays off too."""
|
||||
(entry,) = _models(_row("gpt-5.5", deployments=deployments))
|
||||
|
||||
assert _tiers(entry) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("configured", "expected"),
|
||||
[(None, "priority"), (["priority", "ultrafast"], "priority"), (["ultrafast"], None), ([], None)],
|
||||
)
|
||||
def test_default_service_tier_survives_only_while_still_offered(configured, expected):
|
||||
(entry,) = _models(_row("gpt-6-sol", service_tiers=configured))
|
||||
|
||||
assert entry["default_service_tier"] == expected
|
||||
|
||||
|
||||
def test_non_chat_and_wildcard_rows_are_left_out_of_the_catalog():
|
||||
body = _body(
|
||||
_row("gpt-6-astra"),
|
||||
CodexCatalogRow(id="text-embedding-3-small", mode="embedding"),
|
||||
CodexCatalogRow(id="openai/*"),
|
||||
CodexCatalogRow(id="no-mode-model"),
|
||||
_row("gpt-5.5", mode="chat"),
|
||||
)
|
||||
|
||||
assert body.listed == ("gpt-6-astra", "no-mode-model", "gpt-5.5")
|
||||
assert body.left_out == ()
|
||||
|
||||
|
||||
def test_body_leaves_out_the_entry_that_would_cross_the_byte_limit():
|
||||
rows = tuple(_row(f"model-{index}") for index in range(6))
|
||||
unlimited = _body(*rows, byte_limit=None)
|
||||
limit = len(unlimited.json.encode()) - 1
|
||||
|
||||
body = _body(*rows, byte_limit=limit)
|
||||
one_more = _body(*rows[: len(body.listed) + 1], byte_limit=None)
|
||||
|
||||
assert len(body.json.encode()) <= limit < len(one_more.json.encode())
|
||||
assert body.listed + body.left_out == tuple(row.id for row in rows)
|
||||
assert body.left_out == ("model-5",)
|
||||
assert [entry["slug"] for entry in json.loads(body.json)["models"]] == list(body.listed)
|
||||
|
||||
|
||||
def test_entries_offering_a_tier_survive_the_byte_limit_ahead_of_the_rest_and_keep_their_listing_place():
|
||||
rows = (
|
||||
_row("plain-first"),
|
||||
_row("gpt-5.5"),
|
||||
_row("plain-middle"),
|
||||
_row("tiered-last", service_tiers=["ultrafast"]),
|
||||
)
|
||||
unlimited = _models(*rows, byte_limit=None)
|
||||
two_entries_only = len(_body(rows[1], rows[3], byte_limit=None).json.encode())
|
||||
|
||||
body = _body(*rows, byte_limit=two_entries_only)
|
||||
models = json.loads(body.json)["models"]
|
||||
|
||||
assert body.listed == ("gpt-5.5", "tiered-last") and body.left_out == ("plain-first", "plain-middle")
|
||||
assert [entry["slug"] for entry in models] == ["gpt-5.5", "tiered-last"]
|
||||
assert [entry["priority"] for entry in models] == [1, 3]
|
||||
assert models == [unlimited[1], unlimited[3]]
|
||||
assert len(body.json.encode()) <= two_entries_only
|
||||
|
||||
|
||||
def test_an_entry_too_large_for_the_bytes_left_is_passed_over_and_the_smaller_ones_after_it_are_kept():
|
||||
"""A tier description longer than the whole byte limit sorts first (tiered) and can never fit,
|
||||
and a long display name fits an empty body but not what is left after the entries ahead of it."""
|
||||
kept_rows = (
|
||||
_row("plain-first"),
|
||||
_row("tiered-small", service_tiers=["ultrafast"]),
|
||||
_row("plain-last"),
|
||||
)
|
||||
limit = len(_body(*kept_rows, byte_limit=None).json.encode())
|
||||
oversized_tier = {"id": "huge", "name": "Huge", "description": "x" * limit}
|
||||
rows = (
|
||||
kept_rows[0],
|
||||
_row("tiered-oversized", service_tiers=[oversized_tier]),
|
||||
kept_rows[1],
|
||||
_row("plain-wide", display_name="w" * 200),
|
||||
kept_rows[2],
|
||||
)
|
||||
unlimited = _models(*rows, byte_limit=None)
|
||||
assert len(_body(rows[3], byte_limit=None).json.encode()) < limit
|
||||
|
||||
body = _body(*rows, byte_limit=limit)
|
||||
models = json.loads(body.json)["models"]
|
||||
|
||||
assert body.listed == ("plain-first", "tiered-small", "plain-last")
|
||||
assert body.left_out == ("tiered-oversized", "plain-wide")
|
||||
assert models == [unlimited[0], unlimited[2], unlimited[4]]
|
||||
assert [entry["priority"] for entry in models] == [0, 2, 4]
|
||||
assert len(body.json.encode()) <= limit
|
||||
|
||||
|
||||
def test_unlimited_body_keeps_every_entry():
|
||||
rows = tuple(_row(f"model-{index}") for index in range(3))
|
||||
|
||||
body = _body(*rows, byte_limit=None)
|
||||
|
||||
assert (body.listed, body.left_out) == (tuple(row.id for row in rows), ())
|
||||
assert CODEX_CATALOG_BYTE_LIMIT == 1024 * 1024
|
||||
|
||||
|
||||
def test_vendored_codex_catalog_entries_decode_on_codex():
|
||||
"""Every stock entry served keeps the `model_messages` Codex's `ModelsResponse` decoder
|
||||
needs in place of `base_instructions`, and the catalog is read from the vendored file."""
|
||||
stock = bundled_codex_models()
|
||||
rows = tuple(CodexCatalogRow(id=slug, mode="responses") for slug in stock)
|
||||
|
||||
body = codex_models_response_json(rows, instructions=_PROMPT, byte_limit=None)
|
||||
entries = json.loads(body.json)["models"]
|
||||
|
||||
assert len(entries) == len(stock) > 0
|
||||
assert all(entry["model_messages"]["instructions_template"] for entry in entries)
|
||||
assert all(entry["visibility"] == "list" and entry["supported_in_api"] is True for entry in entries)
|
||||
assert all("base_instructions" not in entry for entry in entries)
|
||||
|
||||
|
||||
def test_configured_service_tiers_without_known_tiers_names_strings_after_their_id():
|
||||
assert configured_service_tiers((["ultrafast"],), "m") == (
|
||||
CodexServiceTier(id="ultrafast", name="Ultrafast", description="Sends service_tier=ultrafast upstream"),
|
||||
)
|
||||
assert configured_service_tiers((None,), "m") is None
|
||||
assert configured_service_tiers((), "m") is None
|
||||
|
||||
|
||||
def test_catalog_rows_read_the_router_by_each_entry_lookup_id():
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "model_name_team-1_c0ffee",
|
||||
"litellm_params": {"model": "openai/gpt-5.5"},
|
||||
"model_info": {"display_name": "Team 5.5", "service_tiers": ["ultrafast"]},
|
||||
},
|
||||
{
|
||||
"model_name": "model_name_team-1_c0ffee",
|
||||
"litellm_params": {"model": "openai/gpt-5.5", "api_base": "https://second.example"},
|
||||
},
|
||||
{"model_name": "plain", "litellm_params": {"model": "openai/some-unmapped-model"}},
|
||||
]
|
||||
)
|
||||
listing = (
|
||||
{
|
||||
"id": "gpt-5.5-team",
|
||||
"object": "model",
|
||||
"created": 0,
|
||||
"owned_by": "openai",
|
||||
"mode": "chat",
|
||||
"max_input_tokens": 7,
|
||||
},
|
||||
{"id": "plain", "object": "model", "created": 0, "owned_by": "openai"},
|
||||
)
|
||||
|
||||
rows = codex_catalog_rows(listing, (("gpt-5.5-team", "model_name_team-1_c0ffee"), ("plain", "plain")), router)
|
||||
|
||||
assert rows == (
|
||||
CodexCatalogRow(
|
||||
id="gpt-5.5-team",
|
||||
mode="chat",
|
||||
max_input_tokens=7,
|
||||
upstream_model="openai/gpt-5.5",
|
||||
display_name="Team 5.5",
|
||||
service_tiers=(["ultrafast"], None),
|
||||
),
|
||||
CodexCatalogRow(id="plain", upstream_model="openai/some-unmapped-model", service_tiers=(None,)),
|
||||
)
|
||||
|
||||
|
||||
def test_catalog_rows_read_an_alias_off_its_target_under_the_alias_name():
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-6-astra",
|
||||
"litellm_params": {"model": "openai/gpt-6-astra"},
|
||||
"model_info": {"display_name": "Astra", "service_tiers": ["ultrafast"]},
|
||||
}
|
||||
],
|
||||
model_group_alias={"gpt-6": "gpt-6-astra"},
|
||||
)
|
||||
listing = (
|
||||
{"id": "gpt-6", "object": "model", "created": 0, "owned_by": "openai", "mode": "chat", "max_input_tokens": 9},
|
||||
)
|
||||
|
||||
(row,) = codex_catalog_rows(listing, (("gpt-6", "gpt-6"),), router)
|
||||
|
||||
assert row == CodexCatalogRow(
|
||||
id="gpt-6", mode="chat", max_input_tokens=9, upstream_model="openai/gpt-6-astra", service_tiers=(["ultrafast"],)
|
||||
)
|
||||
(entry,) = json.loads(codex_model_list_body(listing, (("gpt-6", "gpt-6"),), router))["models"]
|
||||
assert (entry["slug"], entry["display_name"], [tier["id"] for tier in entry["service_tiers"]]) == (
|
||||
"gpt-6",
|
||||
"gpt-6",
|
||||
["ultrafast"],
|
||||
)
|
||||
assert entry["model_messages"] == bundled_codex_models()["gpt-6-astra"].model_dump(mode="json")["model_messages"]
|
||||
|
||||
|
||||
def test_model_list_body_offers_a_tier_only_off_the_deployments_the_key_team_can_route_to():
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-6-astra",
|
||||
"litellm_params": {"model": "openai/gpt-6-astra"},
|
||||
"model_info": {"service_tiers": ["ultrafast"]},
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-6-astra",
|
||||
"litellm_params": {"model": "openai/gpt-6-astra", "api_base": "https://team-2.example"},
|
||||
"model_info": {"team_id": "team-2"},
|
||||
},
|
||||
]
|
||||
)
|
||||
listing = ({"id": "gpt-6-astra", "object": "model", "created": 0, "owned_by": "openai", "mode": "chat"},)
|
||||
entries = (("gpt-6-astra", "gpt-6-astra"),)
|
||||
|
||||
def offered(team_id):
|
||||
(entry,) = json.loads(codex_model_list_body(listing, entries, router, team_id))["models"]
|
||||
return [tier["id"] for tier in entry["service_tiers"]]
|
||||
|
||||
assert offered("team-1") == ["ultrafast"]
|
||||
assert offered("team-2") == []
|
||||
assert offered(None) == ["ultrafast"]
|
||||
|
||||
|
||||
def test_model_list_body_takes_stock_metadata_off_the_deployment_the_key_team_routes_to():
|
||||
"""Two teams own a deployment of one name: Codex's stock gpt-5.5 entry goes only to the team whose
|
||||
requests reach gpt-5.5, under the name and under a `model_group_alias` of it."""
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "coding-model",
|
||||
"litellm_params": {"model": "openai/gpt-5.5"},
|
||||
"model_info": {"team_id": "team-1"},
|
||||
},
|
||||
{
|
||||
"model_name": "coding-model",
|
||||
"litellm_params": {"model": "openai/some-unmapped-model"},
|
||||
"model_info": {"team_id": "team-2", "service_tiers": ["flex"]},
|
||||
},
|
||||
],
|
||||
model_group_alias={"coding": "coding-model"},
|
||||
)
|
||||
listing = tuple(
|
||||
{"id": name, "object": "model", "created": 0, "owned_by": "openai", "mode": "chat"}
|
||||
for name in ("coding-model", "coding")
|
||||
)
|
||||
entries = (("coding-model", "coding-model"), ("coding", "coding"))
|
||||
stock = bundled_codex_models()["gpt-5.5"].model_dump(mode="json")
|
||||
|
||||
def served(team_id):
|
||||
return json.loads(codex_model_list_body(listing, entries, router, team_id))["models"]
|
||||
|
||||
for entry in served("team-1"):
|
||||
assert entry["model_messages"] == stock["model_messages"]
|
||||
assert entry["supported_reasoning_levels"] == stock["supported_reasoning_levels"] != []
|
||||
assert entry["service_tiers"] == stock["service_tiers"] != []
|
||||
for entry in served("team-2"):
|
||||
assert "model_messages" not in entry and entry["base_instructions"]
|
||||
assert entry["supported_reasoning_levels"] == []
|
||||
assert [tier["id"] for tier in entry["service_tiers"]] == ["flex"]
|
||||
assert [entry["slug"] for entry in served("team-2")] == ["coding-model", "coding"]
|
||||
|
||||
|
||||
def test_catalog_rows_without_a_router_carry_only_the_listing():
|
||||
listing = ({"id": "plain", "object": "model", "created": 0, "owned_by": "openai", "mode": "chat"},)
|
||||
|
||||
assert codex_catalog_rows(listing, (("plain", "plain"),), None) == (CodexCatalogRow(id="plain", mode="chat"),)
|
||||
|
|
@ -888,6 +888,17 @@ def test_token_refresh_params_keep_the_prisma_tls_dialect_but_not_the_schema():
|
|||
}
|
||||
|
||||
|
||||
def test_token_refresh_params_keep_the_options_carrying_the_server_timeouts() -> None:
|
||||
kept: Final = token_refresh_params_from_url(
|
||||
"postgresql://u:TOKEN@db.example.com:5432/litellm_db"
|
||||
"?schema=tenant&connection_limit=5&options=-c%20statement_timeout%3D5000%20-c%20idle_in_transaction_session_timeout%3D60000"
|
||||
)
|
||||
assert dict(kept) == {
|
||||
"connection_limit": "5",
|
||||
"options": "-c statement_timeout=5000 -c idle_in_transaction_session_timeout=60000",
|
||||
}
|
||||
|
||||
|
||||
def _issue_cert(
|
||||
subject: str, issuer: x509.Certificate | None, issuer_key: ec.EllipticCurvePrivateKey | None, ca: bool
|
||||
) -> tuple[x509.Certificate, ec.EllipticCurvePrivateKey]:
|
||||
|
|
@ -1199,3 +1210,42 @@ def test_reader_keeps_its_own_pinned_idle_lifetime(monkeypatch):
|
|||
assert os.environ["DATABASE_URL_READ_REPLICA"] == (
|
||||
"postgresql://u:p@reader.example.com:5432/db?max_idle_connection_lifetime=120"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"writer_limit, reader_limit, num_workers, expected",
|
||||
[
|
||||
("10", None, "4", "4 worker(s) x writer connection_limit 10 = up to 40 connections"),
|
||||
(
|
||||
"10",
|
||||
"10",
|
||||
"4",
|
||||
"4 worker(s) x (writer connection_limit 10 + reader connection_limit 10) = up to 80 connections",
|
||||
),
|
||||
(
|
||||
"10",
|
||||
"50",
|
||||
"4",
|
||||
"4 worker(s) x (writer connection_limit 10 + reader connection_limit 50) = up to 240 connections",
|
||||
),
|
||||
("10", None, "0", "1 worker(s) x writer connection_limit 10 = up to 10 connections"),
|
||||
],
|
||||
ids=["writer_only", "reader_doubles_engines", "reader_pins_its_own_limit", "worker_floor"],
|
||||
)
|
||||
def test_connection_budget_message_sums_the_engines_each_worker_owns(
|
||||
writer_limit: str, reader_limit: str | None, num_workers: str, expected: str
|
||||
) -> None:
|
||||
from litellm.proxy.db.db_url_settings import postgres_connection_budget_message
|
||||
|
||||
message: Final = postgres_connection_budget_message(
|
||||
writer_limit=writer_limit, reader_limit=reader_limit, num_workers=num_workers
|
||||
)
|
||||
assert expected in message
|
||||
assert "max_connections minus superuser_reserved_connections" in message
|
||||
|
||||
|
||||
def test_connection_budget_message_survives_an_unparseable_limit() -> None:
|
||||
from litellm.proxy.db.db_url_settings import postgres_connection_budget_message
|
||||
|
||||
message: Final = postgres_connection_budget_message(writer_limit="ten", reader_limit=None, num_workers="2")
|
||||
assert "writer='ten'" in message
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import httpx
|
|||
import pytest
|
||||
from fastapi import HTTPException, Request
|
||||
from prisma import errors as prisma_errors
|
||||
from prisma.engine.errors import BinaryNotFoundError, EngineConnectionError
|
||||
from prisma.engine.errors import BinaryNotFoundError, EngineConnectionError, EngineRequestError
|
||||
from prisma.errors import (
|
||||
ClientNotConnectedError,
|
||||
DataError,
|
||||
|
|
@ -789,3 +789,159 @@ def test_db_lookup_deadline_is_a_connection_and_unavailability_error_but_never_a
|
|||
assert PrismaDBExceptionHandler.is_database_service_unavailable_error(deadline) is True
|
||||
assert PrismaDBExceptionHandler.is_database_transport_error(deadline) is False
|
||||
assert "temporarily unreachable" in PrismaDBExceptionHandler.database_unavailable_message(deadline)
|
||||
|
||||
|
||||
_TOO_MANY_CLIENTS: Final = "Error in connector: Error querying the database: FATAL: sorry, too many clients already"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error",
|
||||
[
|
||||
DataError(data={"user_facing_error": {"message": _TOO_MANY_CLIENTS}}),
|
||||
DataError(
|
||||
data={
|
||||
"user_facing_error": {
|
||||
"message": "Error querying the database: FATAL: zu viele Verbindungen",
|
||||
"meta": {"code": "53300", "message": "zu viele Verbindungen"},
|
||||
}
|
||||
}
|
||||
),
|
||||
DataError(
|
||||
data={
|
||||
"user_facing_error": {
|
||||
"message": "Error in connector: Error querying the database: FATAL: remaining connection slots are reserved for roles with the SUPERUSER attribute"
|
||||
}
|
||||
}
|
||||
),
|
||||
DataError(
|
||||
data={
|
||||
"user_facing_error": {
|
||||
"message": 'Error occurred during query execution: ConnectorError(ConnectorError { user_facing_error: None, kind: QueryError(PostgresError { code: "53300", message: "too many connections for role \\"litellm\\"", severity: "FATAL" }) })'
|
||||
}
|
||||
}
|
||||
),
|
||||
DataError(
|
||||
data={
|
||||
"user_facing_error": {
|
||||
"message": 'Error in connector: Error querying the database: FATAL: too many connections for role "litellm"'
|
||||
}
|
||||
}
|
||||
),
|
||||
DataError(
|
||||
data={
|
||||
"user_facing_error": {
|
||||
"message": 'Error in connector: Error querying the database: FATAL: too many connections for database "litellm"'
|
||||
}
|
||||
}
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_postgres_connection_capacity_refusal_is_service_unavailable_not_a_data_error(error: DataError) -> None:
|
||||
"""Postgres refusing a new connection (SQLSTATE 53300) reaches the proxy as a
|
||||
bare ``DataError`` with no SQLSTATE in ``meta``. The server is up but full, so
|
||||
the failure is service-unavailable (the spend-log flush re-raises and requeues
|
||||
instead of bisecting the batch row by row against a full server) while not a
|
||||
transport error, which would make auth and the health check tear the engine
|
||||
down and open yet more connections against it."""
|
||||
assert PrismaDBExceptionHandler.is_database_capacity_error(error) is True
|
||||
assert PrismaDBExceptionHandler.is_database_service_unavailable_error(error) is True
|
||||
assert PrismaDBExceptionHandler.is_database_transport_error(error) is False
|
||||
assert PrismaDBExceptionHandler.is_prisma_data_error(error) is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error",
|
||||
[
|
||||
DataError(data={"user_facing_error": {"message": "invalid byte sequence for encoding UTF8: 0x00"}}),
|
||||
UniqueViolationError(data={"user_facing_error": {"error_code": "P2002", "meta": {"table": "t"}}}),
|
||||
PrismaError("can't reach database server"),
|
||||
httpx.ConnectError("connection refused"),
|
||||
RuntimeError(_TOO_MANY_CLIENTS),
|
||||
],
|
||||
)
|
||||
def test_is_database_capacity_error_excludes_other_failures(error: Exception) -> None:
|
||||
assert PrismaDBExceptionHandler.is_database_capacity_error(error) is False
|
||||
|
||||
|
||||
def _capacity_error(message: str) -> DataError:
|
||||
"""The shape prisma-client-py raises when Postgres refuses the session: the
|
||||
connector message with no SQLSTATE in ``meta`` and no P-code, so it falls
|
||||
through ``handle_response_errors`` to the base ``DataError``."""
|
||||
return DataError(
|
||||
data={
|
||||
"error": f"Error occurred during query execution:\nConnectorError(ConnectorError {{ user_facing_error: None, kind: QueryError({message}) }})",
|
||||
"user_facing_error": {
|
||||
"is_panic": False,
|
||||
"message": f"Error in connector: Error querying the database: FATAL: {message}",
|
||||
"backtrace": None,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _pool_timeout_error() -> DataError:
|
||||
return DataError(
|
||||
data={
|
||||
"error": "Error in connector: Error creating a database connection. (Timed out fetching a connection from the pool (connection limit: 10, in use: 10, pool timeout 60))",
|
||||
"user_facing_error": {
|
||||
"is_panic": False,
|
||||
"message": "Timed out fetching a new connection from the connection pool. More info: http://pris.ly/d/connection-pool (Current connection pool timeout: 60, connection limit: 10)",
|
||||
"meta": {"connection_limit": 10, "timeout": 60},
|
||||
"error_code": "P2024",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error",
|
||||
[
|
||||
_capacity_error("sorry, too many clients already"),
|
||||
_pool_timeout_error(),
|
||||
EngineRequestError(
|
||||
MagicMock(status=500),
|
||||
'{"is_panic":false,"message":"Error in connector: Error querying the database: FATAL: sorry, too many clients already","backtrace":null}',
|
||||
),
|
||||
RawQueryError(
|
||||
data={
|
||||
"user_facing_error": {
|
||||
"message": 'Raw query failed. Code: `53300`. Message: `db error: FATAL: sorry, too many clients already`',
|
||||
"meta": {"code": "53300", "message": "FATAL: sorry, too many clients already"},
|
||||
"error_code": "P2010",
|
||||
}
|
||||
}
|
||||
),
|
||||
],
|
||||
ids=["53300_connector_dataerror", "P2024_pool_timeout", "engine_500", "raw_query_53300"],
|
||||
)
|
||||
def test_is_database_capacity_error_recognises_postgres_and_pool_exhaustion(error: Exception) -> None:
|
||||
"""SQLSTATE 53300 and prisma P2024 mean the statement was never sent because
|
||||
the deployment is over its connection budget. Both are infrastructure
|
||||
failures (503, never 401), neither is a reason to recreate the engine (that
|
||||
opens another pool against a server that is already full), and both must
|
||||
reach the spend-log writer as "requeue", not "bisect"."""
|
||||
assert PrismaDBExceptionHandler.is_database_capacity_error(error) is True
|
||||
assert PrismaDBExceptionHandler.is_database_service_unavailable_error(error) is True
|
||||
assert PrismaDBExceptionHandler.is_database_connection_error(error) is True
|
||||
assert PrismaDBExceptionHandler.is_database_transport_error(error) is False
|
||||
assert PrismaDBExceptionHandler.is_permanent_database_fault(error) is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error",
|
||||
[
|
||||
PrismaError("timed out while connecting"),
|
||||
PrismaError("can't reach database server"),
|
||||
DataError(data={"user_facing_error": {"message": "Can't reach database server at `127.0.0.1`:`5499`"}}),
|
||||
httpx.ConnectError("conn refused"),
|
||||
],
|
||||
)
|
||||
def test_capacity_check_leaves_reachability_failures_on_the_reconnect_path(error: Exception) -> None:
|
||||
assert PrismaDBExceptionHandler.is_database_capacity_error(error) is False
|
||||
assert PrismaDBExceptionHandler.is_database_transport_error(error) is True
|
||||
|
||||
|
||||
def test_capacity_error_is_seen_through_a_wrapping_exception() -> None:
|
||||
wrapped: Final = RuntimeError("spend flush failed")
|
||||
wrapped.__cause__ = _capacity_error("sorry, too many clients already")
|
||||
assert PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(wrapped) is True
|
||||
|
|
|
|||
|
|
@ -1,13 +1,24 @@
|
|||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from pydantic import ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm import Router
|
||||
from litellm.proxy.lens.endpoints import list_agents, user_scope, validate_model, worker_supports_model
|
||||
from litellm.proxy.lens.models import LensSettings
|
||||
from litellm.proxy.lens.endpoints import (
|
||||
list_agents,
|
||||
run_settings,
|
||||
run_window,
|
||||
user_scope,
|
||||
watchable,
|
||||
watching,
|
||||
validate_model,
|
||||
worker_supports_model,
|
||||
)
|
||||
from litellm.proxy.lens.models import Lens, LensSettings, RunRequest, Scope
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -167,6 +178,79 @@ def test_regular_keys_cannot_read_lens_results(role: LitellmUserRoles | None) ->
|
|||
assert error.value.status_code == 403
|
||||
|
||||
|
||||
def saved_lens() -> Lens:
|
||||
now: Final = datetime(2026, 1, 15, tzinfo=timezone.utc)
|
||||
return Lens(
|
||||
id="lens",
|
||||
scope=Scope(all_teams=True),
|
||||
settings=LensSettings(name="Support", model="analysis", context="Answer questions", agent_name="support"),
|
||||
created_at=now,
|
||||
next_run_at=now,
|
||||
budget_month="2026-01",
|
||||
)
|
||||
|
||||
|
||||
def test_run_now_agent_override_only_changes_the_agent_for_that_run() -> None:
|
||||
lens: Final = saved_lens()
|
||||
overridden: Final = run_settings(lens, RunRequest(agent_name="billing"))
|
||||
assert overridden is not None
|
||||
assert overridden.agent_name == "billing"
|
||||
assert overridden.model_copy(update={"agent_name": "support"}) == lens.settings
|
||||
|
||||
|
||||
def test_run_now_without_overrides_keeps_the_saved_settings() -> None:
|
||||
assert run_settings(saved_lens(), RunRequest()) is None
|
||||
|
||||
|
||||
def test_run_now_rejects_a_window_that_is_missing_an_edge_or_backwards() -> None:
|
||||
now: Final = datetime(2026, 1, 15, tzinfo=timezone.utc)
|
||||
with pytest.raises(ValidationError, match="both a start and an end"):
|
||||
RunRequest(start=now)
|
||||
with pytest.raises(ValidationError, match="before end"):
|
||||
RunRequest(start=now, end=now - timedelta(hours=1))
|
||||
|
||||
|
||||
def test_watching_switches_a_paused_investigation_on_and_records_the_change() -> None:
|
||||
paused: Final = saved_lens().model_copy(
|
||||
update={"settings": saved_lens().settings.model_copy(update={"enabled": False})}
|
||||
)
|
||||
watched: Final = watching(paused)
|
||||
assert watched.settings.enabled is True
|
||||
assert watched.revision == paused.revision + 1
|
||||
assert watched.settings.model_copy(update={"enabled": False}) == paused.settings
|
||||
|
||||
|
||||
def test_watching_leaves_an_investigation_that_is_already_on_untouched() -> None:
|
||||
on: Final = saved_lens()
|
||||
assert watching(on) is on
|
||||
|
||||
|
||||
async def test_watch_all_skips_an_investigation_whose_model_is_gone_instead_of_failing_them_all(
|
||||
analysis_router: Router,
|
||||
) -> None:
|
||||
stale: Final = saved_lens().model_copy(
|
||||
update={"settings": saved_lens().settings.model_copy(update={"model": "retired-model", "enabled": False})}
|
||||
)
|
||||
skipped: Final = await watchable(stale, UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN))
|
||||
assert skipped is not None
|
||||
assert skipped.id == stale.id
|
||||
assert skipped.reason
|
||||
|
||||
|
||||
def test_run_now_since_last_run_keeps_scanning_only_new_traces_even_with_an_agent_override() -> None:
|
||||
now: Final = datetime(2026, 1, 15, 12, tzinfo=timezone.utc)
|
||||
resumed: Final = saved_lens().model_copy(update={"last_scan_at": now - timedelta(hours=1)})
|
||||
window: Final = run_window(resumed, RunRequest(agent_name="billing"), now)
|
||||
assert window is not None
|
||||
assert window[0] == now - timedelta(hours=1)
|
||||
|
||||
|
||||
def test_run_now_with_a_lookback_scans_that_lookback_instead_of_since_last_run() -> None:
|
||||
now: Final = datetime(2026, 1, 15, 12, tzinfo=timezone.utc)
|
||||
resumed: Final = saved_lens().model_copy(update={"last_scan_at": now - timedelta(hours=1)})
|
||||
assert run_window(resumed, RunRequest(lookback_hours=24), now) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider", (False, True))
|
||||
def test_model_errors_reach_worker_with_status_and_redacted_provider_message(provider: bool) -> None:
|
||||
import httpx
|
||||
|
|
|
|||
|
|
@ -4,7 +4,8 @@ import pytest
|
|||
from fastapi import HTTPException
|
||||
|
||||
import litellm
|
||||
from litellm.proxy.lens.inference import Deployment, DeploymentParams, completion_charge, quote
|
||||
from litellm.proxy.lens.inference import Deployment, DeploymentParams, completion_charge, model_step, quote
|
||||
from litellm.proxy.lens.models import ModelRequest
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
|
||||
|
|
@ -120,3 +121,17 @@ def test_unknown_model_capacity_requires_explicit_operator_metadata() -> None:
|
|||
assert "model_info.max_output_tokens" in error.value.detail
|
||||
configured: Final = Deployment(litellm_params=params, model_info=ModelCapacity(max_output_tokens=32000))
|
||||
assert output_tokens(configured) == 32000
|
||||
|
||||
|
||||
def test_a_model_step_records_the_serving_model_and_its_tokens() -> None:
|
||||
response: Final = ModelResponse(model="gpt-5.6", usage={"prompt_tokens": 1200, "completion_tokens": 80})
|
||||
step: Final = model_step(response, ModelRequest(prompt="review", purpose="extract"), "analysis", 0.02)
|
||||
assert (step.model, step.prompt_tokens, step.completion_tokens, step.cost) == ("gpt-5.6", 1200, 80, 0.02)
|
||||
|
||||
|
||||
def test_a_response_without_usage_still_records_a_step_instead_of_failing_settlement() -> None:
|
||||
response: Final = ModelResponse(model="gpt-5.6")
|
||||
unpriced: Final = response.model_copy(update={"usage": None})
|
||||
step: Final = model_step(unpriced, ModelRequest(prompt="review", purpose="cluster"), "analysis", 0.0)
|
||||
assert (step.prompt_tokens, step.completion_tokens) == (0, 0)
|
||||
assert step.label == "Compared observations"
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
from datetime import datetime, timedelta, timezone
|
||||
from functools import reduce
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.lens.models import (
|
||||
MAX_STEPS,
|
||||
AgentTestCase,
|
||||
Check,
|
||||
Evidence,
|
||||
|
|
@ -12,9 +14,19 @@ from litellm.proxy.lens.models import (
|
|||
Lens,
|
||||
LensSettings,
|
||||
Scope,
|
||||
Step,
|
||||
Worker,
|
||||
)
|
||||
from litellm.proxy.lens.state import can_access, claim_job, current_job, merge_finding, queue_job, renew_budget
|
||||
from litellm.proxy.lens.state import (
|
||||
add_step,
|
||||
can_access,
|
||||
claim_job,
|
||||
current_job,
|
||||
merge_finding,
|
||||
next_scan_start,
|
||||
queue_job,
|
||||
renew_budget,
|
||||
)
|
||||
|
||||
NOW: Final = datetime(2026, 1, 15, tzinfo=timezone.utc)
|
||||
|
||||
|
|
@ -163,15 +175,42 @@ def test_monthly_budget_renews_without_erasing_job_costs() -> None:
|
|||
|
||||
|
||||
@pytest.mark.parametrize("hours", (24, 168, 720, 4800, 8760))
|
||||
def test_every_scan_uses_the_configured_lookback_window(hours: int) -> None:
|
||||
def test_first_scan_covers_the_configured_lookback_window(hours: int) -> None:
|
||||
original: Final = lens()
|
||||
configured: Final = original.model_copy(
|
||||
update={"settings": LensSettings.model_validate({**original.settings.model_dump(), "lookback_hours": hours})}
|
||||
)
|
||||
first: Final = queue_job(configured, NOW, "first")
|
||||
assert first.jobs[0].start == NOW - timedelta(hours=hours)
|
||||
resumed: Final = configured.model_copy(update={"last_scan_at": NOW - timedelta(hours=1)})
|
||||
assert queue_job(resumed, NOW, "next").jobs[0].start == NOW - timedelta(hours=hours)
|
||||
assert first.jobs[0].trigger == "schedule"
|
||||
|
||||
|
||||
def test_later_scheduled_scans_only_cover_traces_since_the_last_scan() -> None:
|
||||
resumed: Final = lens().model_copy(update={"last_scan_at": NOW - timedelta(hours=1)})
|
||||
job: Final = queue_job(resumed, NOW, "next").jobs[0]
|
||||
assert job.start == NOW - timedelta(hours=1)
|
||||
assert job.end == NOW - timedelta(minutes=2)
|
||||
|
||||
|
||||
def test_a_scan_after_a_long_outage_never_reaches_past_the_lookback_window() -> None:
|
||||
stale: Final = lens().model_copy(update={"last_scan_at": NOW - timedelta(days=400)})
|
||||
assert queue_job(stale, NOW, "next").jobs[0].start == NOW - timedelta(hours=stale.settings.lookback_hours)
|
||||
|
||||
|
||||
def test_run_now_with_an_exact_window_scans_that_window_and_is_marked_manual() -> None:
|
||||
window: Final = (NOW - timedelta(hours=5), NOW - timedelta(hours=3))
|
||||
job: Final = queue_job(lens(), NOW, "manual", window=window, trigger="manual").jobs[0]
|
||||
assert (job.start, job.end) == window
|
||||
assert job.trigger == "manual"
|
||||
|
||||
|
||||
def test_steps_keep_only_the_most_recent_entries() -> None:
|
||||
job: Final = queue_job(lens(), NOW, "job").jobs[0]
|
||||
steps: Final = tuple(Step(at=NOW, kind="stage", label=f"step {i}") for i in range(MAX_STEPS + 5))
|
||||
grown: Final = reduce(add_step, steps, job)
|
||||
assert len(grown.steps) == MAX_STEPS
|
||||
assert grown.steps[0].label == "step 5"
|
||||
assert grown.steps[-1].label == f"step {MAX_STEPS + 4}"
|
||||
|
||||
|
||||
def test_finding_keeps_uncertainty_separate_from_the_main_summary() -> None:
|
||||
|
|
@ -287,6 +326,17 @@ def test_legacy_finding_identity_preserves_feedback_only_for_same_kind_and_check
|
|||
assert separate.status == "open" and separate.reason == ""
|
||||
|
||||
|
||||
def test_only_successful_scheduled_scans_move_the_next_scan_forward() -> None:
|
||||
previous: Final = lens().model_copy(update={"last_scan_at": NOW - timedelta(hours=3)})
|
||||
scheduled: Final = queue_job(previous, NOW, "scheduled").jobs[0]
|
||||
manual: Final = queue_job(
|
||||
previous, NOW, "manual", window=(NOW - timedelta(hours=2), NOW - timedelta(hours=1)), trigger="manual"
|
||||
).jobs[0]
|
||||
assert next_scan_start(previous, scheduled, failed=False) == scheduled.end
|
||||
assert next_scan_start(previous, scheduled, failed=True) == previous.last_scan_at
|
||||
assert next_scan_start(previous, manual, failed=False) == previous.last_scan_at
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("lookback_hours", "interval_minutes"))
|
||||
def test_calendar_overflow_is_rejected_without_the_old_history_and_interval_caps(field: str) -> None:
|
||||
from pydantic import ValidationError
|
||||
|
|
|
|||
|
|
@ -46,7 +46,9 @@ def patched_models(monkeypatch):
|
|||
deployment = MagicMock()
|
||||
deployment.litellm_params.model = "gpt-4"
|
||||
router.get_deployment_by_model_group_name = MagicMock(return_value=deployment)
|
||||
router.get_routable_upstream_model = MagicMock(return_value="gpt-4")
|
||||
router.get_configured_display_name = MagicMock(return_value=None)
|
||||
router.get_configured_service_tiers = MagicMock(return_value=())
|
||||
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", MagicMock())
|
||||
|
|
@ -397,3 +399,118 @@ def test_anthropic_format_keeps_served_ids_for_other_anthropic_clients(client, a
|
|||
|
||||
assert response.status_code == 200
|
||||
assert [m["id"] for m in response.json()["data"]] == ["gpt-4", "claude-sonnet"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/v1/models", "/models"])
|
||||
@pytest.mark.parametrize("params", [{}, {"scope": "expand"}])
|
||||
def test_codex_format_when_client_version_present(client, auth_as, patched_models, path, params):
|
||||
"""Codex CLI fetches a provider's catalog as ``GET /v1/models?client_version=<its version>`` and
|
||||
decodes Codex's own ``{"models": [...]}`` shape; the same request without the parameter keeps the
|
||||
OpenAI shape byte for byte."""
|
||||
with auth_as():
|
||||
codex_response = client.get(path, params={**params, "client_version": "0.159.3"})
|
||||
openai_response = client.get(path, params=params)
|
||||
|
||||
assert codex_response.status_code == 200
|
||||
assert codex_response.headers["content-type"] == "application/json"
|
||||
body = codex_response.json()
|
||||
assert list(body) == ["models"]
|
||||
assert [(m["slug"], m["display_name"], m["priority"]) for m in body["models"]] == [
|
||||
("gpt-4", "gpt-4", 0),
|
||||
("claude-sonnet", "claude-sonnet", 1),
|
||||
]
|
||||
assert all(m["base_instructions"] and m["visibility"] == "list" for m in body["models"])
|
||||
|
||||
assert openai_response.status_code == 200
|
||||
assert normalize(openai_response.json()) == {
|
||||
"data": [
|
||||
{"id": "<VOLATILE>", "object": "model", "created": "<VOLATILE>", "owned_by": "openai"},
|
||||
{"id": "<VOLATILE>", "object": "model", "created": "<VOLATILE>", "owned_by": "openai"},
|
||||
],
|
||||
"object": "list",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/v1/models", "/models"])
|
||||
def test_codex_format_wins_over_the_anthropic_header(client, auth_as, patched_models, path):
|
||||
with auth_as():
|
||||
response = client.get(path, params={"client_version": "0.159.3"}, headers={"anthropic-version": "2023-06-01"})
|
||||
|
||||
assert response.status_code == 200
|
||||
assert list(response.json()) == ["models"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/v1/models", "/models"])
|
||||
def test_codex_format_carries_configured_service_tiers(client, auth_as, patched_models, path):
|
||||
"""A deployment's ``model_info.service_tiers`` becomes the entry's ``service_tiers``, which Codex
|
||||
offers as slash commands; a model without one offers none, and the OpenAI shape gains no field."""
|
||||
patched_models.get_configured_service_tiers = MagicMock(
|
||||
side_effect=lambda model_name, team_id=None: (["ultrafast"],) if model_name == "gpt-4" else (None,)
|
||||
)
|
||||
|
||||
with auth_as():
|
||||
codex_response = client.get(path, params={"client_version": "0.159.3"})
|
||||
openai_response = client.get(path)
|
||||
|
||||
gpt_4, claude = codex_response.json()["models"]
|
||||
assert gpt_4["service_tiers"] == [
|
||||
{"id": "ultrafast", "name": "Ultrafast", "description": "Sends service_tier=ultrafast upstream"}
|
||||
]
|
||||
assert claude["service_tiers"] == []
|
||||
assert all("service_tiers" not in m for m in openai_response.json()["data"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("params", [{}, {"scope": "expand"}])
|
||||
def test_codex_service_tiers_are_read_for_the_key_team(client, auth_as, patched_models, params):
|
||||
"""A tier and the upstream model that picks Codex's stock entry are read off the deployments the
|
||||
key's team can route to, so both listing paths hand the router the key's team, and no team for a
|
||||
key without one."""
|
||||
patched_models.get_configured_service_tiers = MagicMock(
|
||||
side_effect=lambda model_name, team_id=None: (["ultrafast"],) if team_id == "team-1" else (None,)
|
||||
)
|
||||
patched_models.get_routable_upstream_model = MagicMock(
|
||||
side_effect=lambda model_name, team_id=None: "openai/gpt-5.5" if team_id == "team-1" else "gpt-4"
|
||||
)
|
||||
|
||||
with auth_as(team_id="team-1"):
|
||||
team_response = client.get("/v1/models", params={**params, "client_version": "0.159.3"})
|
||||
with auth_as():
|
||||
teamless_response = client.get("/v1/models", params={**params, "client_version": "0.159.3"})
|
||||
|
||||
assert [[t["id"] for t in m["service_tiers"]] for m in team_response.json()["models"]] == [["ultrafast"]] * 2
|
||||
assert [m["service_tiers"] for m in teamless_response.json()["models"]] == [[], []]
|
||||
assert all(m["supported_reasoning_levels"] for m in team_response.json()["models"])
|
||||
assert [m["supported_reasoning_levels"] for m in teamless_response.json()["models"]] == [[], []]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("params", [{}, {"scope": "expand"}])
|
||||
def test_codex_service_tiers_resolved_via_internal_team_key(client, auth_as, patched_models, monkeypatch, params):
|
||||
"""A team-scoped row's tiers are looked up by the internal routing key while the entry is keyed by
|
||||
the public name Codex sends back as the model."""
|
||||
from litellm.proxy import utils as proxy_utils
|
||||
from litellm.proxy.auth import model_checks
|
||||
|
||||
internal_name = "model_name_team-1_c0ffee"
|
||||
|
||||
patched_models.get_model_list = MagicMock(
|
||||
return_value=[
|
||||
{"model_name": internal_name, "model_info": {"team_id": "team-1", "team_public_model_name": "gpt-4-team"}}
|
||||
]
|
||||
)
|
||||
patched_models.get_model_names = MagicMock(return_value=[internal_name])
|
||||
patched_models.get_configured_service_tiers = MagicMock(
|
||||
side_effect=lambda model_name, team_id=None: (["ultrafast"],) if model_name == internal_name else ()
|
||||
)
|
||||
|
||||
async def _fake_get_available_models_for_user(**kwargs):
|
||||
return [internal_name]
|
||||
|
||||
monkeypatch.setattr(proxy_utils, "get_available_models_for_user", _fake_get_available_models_for_user)
|
||||
monkeypatch.setattr(model_checks, "get_complete_model_list", lambda **kwargs: [internal_name])
|
||||
|
||||
with auth_as():
|
||||
response = client.get("/v1/models", params={**params, "client_version": "0.159.3"})
|
||||
|
||||
assert response.status_code == 200
|
||||
(entry,) = response.json()["models"]
|
||||
assert (entry["slug"], [tier["id"] for tier in entry["service_tiers"]]) == ("gpt-4-team", ["ultrafast"])
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import os
|
|||
from contextlib import nullcontext
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import click
|
||||
|
|
@ -2964,6 +2965,32 @@ class TestPostgresStatementTimeoutOptions:
|
|||
|
||||
assert _pg_options_with_timeouts(existing, statement_timeout, lock_timeout) == expected
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"existing, idle_timeout, expected",
|
||||
[
|
||||
("", 30, "-c statement_timeout=60000 -c lock_timeout=15000 -c idle_in_transaction_session_timeout=30000"),
|
||||
("", None, "-c statement_timeout=60000 -c lock_timeout=15000"),
|
||||
(
|
||||
"-c idle_in_transaction_session_timeout=5000",
|
||||
30,
|
||||
"-c idle_in_transaction_session_timeout=5000 -c statement_timeout=60000 -c lock_timeout=15000",
|
||||
),
|
||||
],
|
||||
ids=["idle_set", "idle_unset", "pinned_idle_wins"],
|
||||
)
|
||||
def test_pg_options_with_idle_in_transaction_timeout(
|
||||
self,
|
||||
existing: str,
|
||||
idle_timeout: int | None,
|
||||
expected: str,
|
||||
) -> None:
|
||||
"""A transaction that opened and then stalled holds its connection and its
|
||||
locks for as long as the client stays silent; ``idle_in_transaction_session_timeout``
|
||||
is the only server-side bound on that, so it rides the same ``options`` string."""
|
||||
from litellm.proxy.proxy_cli import _pg_options_with_timeouts
|
||||
|
||||
assert _pg_options_with_timeouts(existing, 60, 15, idle_timeout) == expected
|
||||
|
||||
def test_timeouts_reach_the_database_url_from_general_settings(self, tmp_path):
|
||||
"""The whole point of the setting: it has to land on DATABASE_URL."""
|
||||
import yaml
|
||||
|
|
@ -2976,6 +3003,7 @@ class TestPostgresStatementTimeoutOptions:
|
|||
"general_settings": {
|
||||
"database_statement_timeout": 60,
|
||||
"database_lock_timeout": 15,
|
||||
"database_idle_in_transaction_session_timeout": 30,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
|
@ -2986,6 +3014,7 @@ class TestPostgresStatementTimeoutOptions:
|
|||
options = urlparse.parse_qs(urlparse.urlparse(modified_url).query)["options"][0]
|
||||
assert "-c statement_timeout=60000" in options
|
||||
assert "-c lock_timeout=15000" in options
|
||||
assert "-c idle_in_transaction_session_timeout=30000" in options
|
||||
|
||||
def test_no_options_param_when_unset(self, tmp_path):
|
||||
"""Unset must mean today's behavior, not an empty options string."""
|
||||
|
|
@ -3066,6 +3095,7 @@ def _run_server_and_capture_urls(
|
|||
database_url: str = "postgresql://t:t@localhost:5432/t",
|
||||
direct_url: str | None = None,
|
||||
read_replica_url: str | None = None,
|
||||
extra_args: tuple[str, ...] = (),
|
||||
) -> dict:
|
||||
loaded_config = yaml.safe_load(Path(config_path).read_text())
|
||||
mock_proxy_config = MagicMock()
|
||||
|
|
@ -3098,7 +3128,7 @@ def _run_server_and_capture_urls(
|
|||
patch("litellm.proxy.db.check_migration.check_prisma_schema_diff"),
|
||||
):
|
||||
run_server.main(
|
||||
["--config", config_path, "--local", "--skip_server_startup"],
|
||||
["--config", config_path, "--local", "--skip_server_startup", *extra_args],
|
||||
standalone_mode=False,
|
||||
)
|
||||
return {k: os.environ[k] for k in _CAPTURED_DB_ENV_VARS if k in os.environ}
|
||||
|
|
@ -3172,6 +3202,47 @@ class TestReadReplicaConnectionParams:
|
|||
assert query["connection_limit"] == ["50"]
|
||||
assert query["pool_timeout"] == ["20"]
|
||||
|
||||
def test_connection_budget_line_counts_the_limits_the_final_urls_carry(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
import yaml
|
||||
|
||||
config_path: Final = tmp_path / "config.yaml"
|
||||
config_path.write_text(
|
||||
yaml.dump({"model_list": [], "general_settings": {"database_connection_pool_limit": 3}})
|
||||
)
|
||||
|
||||
_run_server_and_capture_urls(
|
||||
str(config_path),
|
||||
read_replica_url="postgresql://t:t@reader:5432/t?connection_limit=50",
|
||||
)
|
||||
|
||||
assert (
|
||||
"1 worker(s) x (writer connection_limit 3 + reader connection_limit 50) = up to 53 connections"
|
||||
in capsys.readouterr().out
|
||||
)
|
||||
|
||||
def test_connection_budget_line_counts_one_worker_under_hypercorn(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
import yaml
|
||||
|
||||
config_path: Final = tmp_path / "config.yaml"
|
||||
config_path.write_text(
|
||||
yaml.dump({"model_list": [], "general_settings": {"database_connection_pool_limit": 3}})
|
||||
)
|
||||
|
||||
_run_server_and_capture_urls(
|
||||
str(config_path),
|
||||
extra_args=("--run_hypercorn", "--num_workers", "4"),
|
||||
)
|
||||
|
||||
assert "1 worker(s) x writer connection_limit 3 = up to 3 connections" in capsys.readouterr().out
|
||||
|
||||
def test_extra_connection_params_never_carry_a_schema_override_to_the_reader(self, tmp_path):
|
||||
"""database_extra_connection_params is an untyped passthrough, so it can carry a
|
||||
search_path. The writer keeps it, the reader must not inherit it, or replica
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ from __future__ import annotations
|
|||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Iterator
|
||||
from typing import Any, Dict, List
|
||||
from collections.abc import Callable, Iterator
|
||||
from typing import Any, Dict, Final, List
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
|
@ -917,3 +917,38 @@ async def test_update_spend_logs_parks_failed_batch_in_redis_with_wire_safe_date
|
|||
parked = await buffer.get_spend_logs_from_redis_buffer(limit=10)
|
||||
assert mock_prisma_client.spend_log_transactions == []
|
||||
assert [(row["request_id"], row["startTime"]) for row in parked] == [("a", started.isoformat())]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_spend_logs_requeues_batch_when_postgres_is_out_of_connections(
|
||||
mock_prisma_client: MagicMock,
|
||||
make_spend_log_row: Callable[..., Dict[str, Any]],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Postgres refusing the pool a new connection (SQLSTATE 53300, "too many
|
||||
clients already") surfaces as a bare ``DataError``. It is the server being
|
||||
full, not a row being bad: the flush must stop after the one failed insert
|
||||
and put the batch back at the head of the queue for the next interval,
|
||||
rather than bisecting it (hundreds more connection attempts against a full
|
||||
server) and dropping every row as poisoned."""
|
||||
sleep: Final = AsyncMock(return_value=None)
|
||||
monkeypatch.setattr(utils_mod.asyncio, "sleep", sleep)
|
||||
err: Final = _data_error("Error in connector: Error querying the database: FATAL: sorry, too many clients already")
|
||||
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=err)
|
||||
proxy_logging: Final = MagicMock()
|
||||
proxy_logging.failure_handler = AsyncMock()
|
||||
mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="e")]
|
||||
logs: Final = [make_spend_log_row(request_id=f"r{i}") for i in range(4)]
|
||||
|
||||
with pytest.raises(type(err)):
|
||||
await ProxyUpdateSpend.update_spend_logs(
|
||||
n_retry_times=2,
|
||||
prisma_client=mock_prisma_client,
|
||||
db_writer_client=None,
|
||||
proxy_logging_obj=proxy_logging,
|
||||
logs_to_process=logs,
|
||||
)
|
||||
|
||||
assert mock_prisma_client.db.litellm_spendlogs.create_many.await_count == 1
|
||||
sleep.assert_not_awaited()
|
||||
assert [row["request_id"] for row in mock_prisma_client.spend_log_transactions] == ["r0", "r1", "r2", "r3", "e"]
|
||||
|
|
|
|||
|
|
@ -12,7 +12,9 @@ from __future__ import annotations
|
|||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from contextlib import suppress
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, Final, List
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
|
|
@ -968,3 +970,119 @@ async def test_monitor_spend_logs_queue_pulls_parked_rows_before_each_flush(
|
|||
)
|
||||
|
||||
assert seen == [["parked"]]
|
||||
|
||||
|
||||
def _postgres_out_of_connections() -> Exception:
|
||||
"""prisma's shape for Postgres SQLSTATE 53300: a base ``DataError`` whose only
|
||||
hint is the connector message."""
|
||||
from prisma.errors import DataError
|
||||
|
||||
return DataError(
|
||||
data={
|
||||
"user_facing_error": {
|
||||
"is_panic": False,
|
||||
"message": "Error in connector: Error querying the database: FATAL: sorry, too many clients already",
|
||||
"backtrace": None,
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_spend_logs_job_requeues_whole_batch_when_postgres_is_out_of_connections(
|
||||
mock_prisma_client: MagicMock,
|
||||
make_spend_log_row: Callable[..., dict[str, object]],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""53300 is not a poison row: bisecting it would issue one failing statement
|
||||
per row (each a fresh connection attempt against a full server) and drop
|
||||
every row. The batch goes back to the queue head untouched, in one attempt,
|
||||
without the in-job retry loop hammering the server."""
|
||||
sleeps: Final[list[float]] = []
|
||||
|
||||
async def _no_sleep(seconds: float, *_: object, **__: object) -> None:
|
||||
sleeps.append(seconds)
|
||||
|
||||
monkeypatch.setattr(asyncio, "sleep", _no_sleep)
|
||||
proxy_logging: Final = MagicMock()
|
||||
proxy_logging.failure_handler = AsyncMock()
|
||||
mock_prisma_client.spend_log_transactions = [
|
||||
make_spend_log_row(request_id="r1"),
|
||||
make_spend_log_row(request_id="r2"),
|
||||
make_spend_log_row(request_id="r3"),
|
||||
]
|
||||
mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_postgres_out_of_connections())
|
||||
|
||||
with pytest.raises(Exception, match="too many clients already"):
|
||||
await update_spend_logs_job(
|
||||
prisma_client=mock_prisma_client,
|
||||
db_writer_client=None,
|
||||
proxy_logging_obj=proxy_logging,
|
||||
)
|
||||
|
||||
assert {
|
||||
"create_many_calls": mock_prisma_client.db.litellm_spendlogs.create_many.await_count,
|
||||
"queue_after": [row["request_id"] for row in mock_prisma_client.spend_log_transactions],
|
||||
"backoff_sleeps": sleeps,
|
||||
} == {"create_many_calls": 1, "queue_after": ["r1", "r2", "r3"], "backoff_sleeps": []}
|
||||
|
||||
|
||||
def _tool_usage_transaction(request_id: str) -> object:
|
||||
from litellm.proxy.db.spend_log_tool_index import ToolUsageTransaction
|
||||
|
||||
return ToolUsageTransaction(
|
||||
request_id=request_id,
|
||||
date="2026-10-02",
|
||||
start_time=datetime(2026, 10, 2, tzinfo=timezone.utc),
|
||||
tool_names=("get_weather",),
|
||||
spend=0.01,
|
||||
total_tokens=12,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_usage_flush_requeues_when_postgres_is_out_of_connections(
|
||||
mock_prisma_client: MagicMock,
|
||||
) -> None:
|
||||
"""A tool-usage batch Postgres had no connection for was never sent, so it is
|
||||
safe to keep; dropping it loses the rollup increments for good. The job stops
|
||||
there, like the spend-log write, so a drain loop does not re-hit the full server."""
|
||||
from prisma.errors import DataError
|
||||
|
||||
mock_prisma_client.db.litellm_spendlogtoolindex.create_many = AsyncMock(side_effect=_postgres_out_of_connections())
|
||||
mock_prisma_client.spend_log_transactions = []
|
||||
first, second = _tool_usage_transaction("r1"), _tool_usage_transaction("r2")
|
||||
mock_prisma_client.tool_usage_transactions = [first, second]
|
||||
|
||||
with pytest.raises(DataError, match="too many clients already"):
|
||||
await update_spend_logs_job(
|
||||
prisma_client=mock_prisma_client,
|
||||
db_writer_client=None,
|
||||
proxy_logging_obj=MagicMock(),
|
||||
)
|
||||
|
||||
assert {
|
||||
"index_writes": mock_prisma_client.db.litellm_spendlogtoolindex.create_many.await_count,
|
||||
"queue_after": mock_prisma_client.tool_usage_transactions,
|
||||
} == {"index_writes": 1, "queue_after": [first, second]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_usage_flush_still_drops_ambiguous_failures(
|
||||
mock_prisma_client: MagicMock,
|
||||
) -> None:
|
||||
"""Anything other than a connection refusal may have reached the server, and
|
||||
the rollup increments are not idempotent, so the batch is not replayed."""
|
||||
mock_prisma_client.db.litellm_spendlogtoolindex.create_many = AsyncMock(
|
||||
side_effect=RuntimeError("engine returned a malformed payload")
|
||||
)
|
||||
mock_prisma_client.spend_log_transactions = []
|
||||
mock_prisma_client.tool_usage_transactions = [_tool_usage_transaction("r1")]
|
||||
|
||||
await update_spend_logs_job(
|
||||
prisma_client=mock_prisma_client,
|
||||
db_writer_client=None,
|
||||
proxy_logging_obj=MagicMock(),
|
||||
)
|
||||
|
||||
assert mock_prisma_client.tool_usage_transactions == []
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import datetime
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType, SimpleNamespace
|
||||
from typing import Final, cast
|
||||
|
|
@ -3864,6 +3865,35 @@ def test_completion_cost_mantle_native_messages_prices_unversioned_claude_from_t
|
|||
) == pytest.approx(expected), model
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["anthropic.claude-opus-5-5", "anthropic.claude-sonnet-5-5"])
|
||||
def test_completion_cost_region_without_its_own_row_prices_mantle_claude_from_the_mantle_row(
|
||||
_local_model_cost_map, model: str
|
||||
):
|
||||
"""The proxy resolves a Mantle region for every call. A region with no
|
||||
bedrock_mantle/<region>/<model> row must fall back to the model's own bedrock_mantle/ row, not to the
|
||||
bare Bedrock row that the bedrock provider family also matches."""
|
||||
|
||||
response = litellm.ModelResponse(
|
||||
id="msg_x",
|
||||
choices=[{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
|
||||
model=model,
|
||||
usage={"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110},
|
||||
)
|
||||
mantle: Final[Mapping[str, float]] = litellm.model_cost[f"bedrock_mantle/{model}"]
|
||||
bedrock: Final[Mapping[str, float]] = litellm.model_cost[model]
|
||||
expected: Final = 100 * mantle["input_cost_per_token"] + 10 * mantle["output_cost_per_token"]
|
||||
assert expected != 100 * bedrock["input_cost_per_token"] + 10 * bedrock["output_cost_per_token"]
|
||||
|
||||
for deployment in (model, f"bedrock_mantle/{model}", f"bedrock_mantle/us-east-1/{model}"):
|
||||
assert litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model=deployment,
|
||||
custom_llm_provider="bedrock_mantle",
|
||||
region_name="us-east-1",
|
||||
) == pytest.approx(expected), deployment
|
||||
assert litellm.get_model_info(f"bedrock_mantle/us-east-1/{model}", "bedrock_mantle")["key"] == f"bedrock_mantle/{model}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["anthropic.claude-opus-5-5", "anthropic.claude-sonnet-5-5"])
|
||||
def test_cost_per_token_gov_region_prices_mantle_claude_on_the_gov_row(_local_model_cost_map, model):
|
||||
"""A bedrock_mantle/ deployment in us-gov-west-1 must price from the
|
||||
|
|
|
|||
43
tests/unit/test_openrouter_gpt_image_model_metadata.py
Normal file
43
tests/unit/test_openrouter_gpt_image_model_metadata.py
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm import get_model_info
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
REPO_ROOT: Final = Path(__file__).parents[2]
|
||||
COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
|
||||
MODELS: Final = (
|
||||
"openrouter/openai/gpt-image-2",
|
||||
"openrouter/openai/gpt-image-2.5-flare",
|
||||
"openrouter/openai/gpt-image-2.5-sunburst",
|
||||
)
|
||||
PRICE_FIELDS: Final = ("input_cost_per_token", "input_cost_per_image_token", "output_cost_per_image_token")
|
||||
|
||||
|
||||
def _cost_map(path: Path) -> dict[str, dict[str, object]]:
|
||||
return COST_MAP_ADAPTER.validate_json(path.read_bytes())
|
||||
|
||||
|
||||
MAIN_COST_MAP: Final = _cost_map(REPO_ROOT / "model_prices_and_context_window.json")
|
||||
BACKUP_COST_MAP: Final = _cost_map(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_openrouter_gpt_image_row_is_identical_in_main_and_backup(model: str) -> None:
|
||||
assert model in MAIN_COST_MAP, f"{model} is missing from model_prices_and_context_window.json"
|
||||
assert BACKUP_COST_MAP.get(model) == MAIN_COST_MAP[model]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("local_model_cost_map")
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_openrouter_gpt_image_model_info_comes_from_the_openrouter_row(model: str) -> None:
|
||||
routed_model, provider, _, _ = get_llm_provider(model=model)
|
||||
assert (routed_model, provider) == (model.removeprefix("openrouter/"), "openrouter")
|
||||
|
||||
info = get_model_info(model=routed_model, custom_llm_provider=provider)
|
||||
row = MAIN_COST_MAP[model]
|
||||
assert (info["key"], info["litellm_provider"], info["mode"]) == (model, "openrouter", "image_generation")
|
||||
assert {field: info[field] for field in PRICE_FIELDS} == {field: row[field] for field in PRICE_FIELDS}
|
||||
|
|
@ -10276,6 +10276,200 @@ def test_get_configured_display_name_skips_wildcard_pattern_matching():
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"configured",
|
||||
[["ultrafast"], ["priority", {"id": "ultrafast", "name": "Ultrafast", "description": "Fastest"}], [], "not-a-list"],
|
||||
)
|
||||
def test_get_configured_service_tiers_returns_the_deployment_model_info_value_as_set(configured):
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-6-astra",
|
||||
"litellm_params": {"model": "openai/gpt-6-astra"},
|
||||
"model_info": {"service_tiers": configured},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
assert router.get_configured_service_tiers("gpt-6-astra") == (configured,)
|
||||
|
||||
|
||||
def test_get_configured_service_tiers_returns_one_value_per_deployment_in_model_list_order():
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-6-astra",
|
||||
"litellm_params": {"model": "openai/gpt-6-astra"},
|
||||
"model_info": {"service_tiers": ["ultrafast"]},
|
||||
},
|
||||
{"model_name": "gpt-6-astra", "litellm_params": {"model": "openai/gpt-6-astra", "api_base": "https://a.example"}},
|
||||
{
|
||||
"model_name": "gpt-6-astra",
|
||||
"litellm_params": {"model": "openai/gpt-6-astra", "api_base": "https://b.example"},
|
||||
"model_info": {"service_tiers": ["priority", "ultrafast"]},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
assert router.get_configured_service_tiers("gpt-6-astra") == (["ultrafast"], None, ["priority", "ultrafast"])
|
||||
|
||||
|
||||
def test_get_configured_service_tiers_returns_none_for_an_unset_deployment_and_nothing_for_an_unknown_name():
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "no-tiers-model",
|
||||
"litellm_params": {"model": "openai/some-unmapped-model"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
assert router.get_configured_service_tiers("no-tiers-model") == (None,)
|
||||
assert router.get_configured_service_tiers("not-a-real-model") == ()
|
||||
|
||||
|
||||
def test_get_configured_service_tiers_does_not_apply_a_wildcard_deployment_to_matched_names():
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {"model": "openai/*"},
|
||||
"model_info": {"service_tiers": ["ultrafast"]},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
assert router.get_configured_service_tiers("openai/gpt-6-astra") == ()
|
||||
|
||||
|
||||
def test_get_configured_service_tiers_reads_only_the_deployments_a_request_can_route_to():
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-6-astra",
|
||||
"litellm_params": {"model": "openai/gpt-6-astra"},
|
||||
"model_info": {"service_tiers": ["ultrafast"]},
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-6-astra",
|
||||
"litellm_params": {"model": "openai/gpt-6-astra", "api_base": "https://paused.example"},
|
||||
"model_info": {"blocked": True},
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-6-astra",
|
||||
"litellm_params": {"model": "openai/gpt-6-astra", "api_base": "https://team-2.example"},
|
||||
"model_info": {"team_id": "team-2"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
assert router.get_configured_service_tiers("gpt-6-astra", team_id="team-1") == (["ultrafast"],)
|
||||
assert router.get_configured_service_tiers("gpt-6-astra", team_id="team-2") == (["ultrafast"], None)
|
||||
assert router.get_configured_service_tiers("gpt-6-astra") == (["ultrafast"],)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"alias_value, expected_group",
|
||||
[
|
||||
("gpt-6-astra", "gpt-6-astra"),
|
||||
({"model": "gpt-6-astra", "hidden": True}, "gpt-6-astra"),
|
||||
({"model": "", "hidden": False}, "gpt-6"),
|
||||
],
|
||||
ids=["string-alias", "item-alias", "malformed-alias-is-itself"],
|
||||
)
|
||||
def test_routable_model_group_is_the_alias_target_else_the_name_itself(alias_value, expected_group):
|
||||
router = litellm.Router(
|
||||
model_list=[{"model_name": "gpt-6-astra", "litellm_params": {"model": "openai/gpt-6-astra"}}],
|
||||
model_group_alias={"gpt-6": alias_value},
|
||||
)
|
||||
|
||||
assert router.routable_model_group("gpt-6") == expected_group
|
||||
assert router.routable_model_group("gpt-6-astra") == "gpt-6-astra"
|
||||
assert router.routable_model_group("not-a-real-model") == "not-a-real-model"
|
||||
|
||||
|
||||
def test_get_configured_service_tiers_reads_an_alias_off_its_target_deployments():
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-6-astra",
|
||||
"litellm_params": {"model": "openai/gpt-6-astra"},
|
||||
"model_info": {"service_tiers": ["ultrafast"]},
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-6-astra",
|
||||
"litellm_params": {"model": "openai/gpt-6-astra", "api_base": "https://team-2.example"},
|
||||
"model_info": {"team_id": "team-2", "service_tiers": ["priority"]},
|
||||
},
|
||||
],
|
||||
model_group_alias={"gpt-6": "gpt-6-astra", "gpt-6-quiet": {"model": "gpt-6-astra", "hidden": True}},
|
||||
)
|
||||
|
||||
assert router.get_configured_service_tiers("gpt-6") == router.get_configured_service_tiers("gpt-6-astra")
|
||||
assert router.get_configured_service_tiers("gpt-6", team_id="team-1") == (["ultrafast"],)
|
||||
assert router.get_configured_service_tiers("gpt-6", team_id="team-2") == (["ultrafast"], ["priority"])
|
||||
assert router.get_configured_service_tiers("gpt-6-quiet", team_id="team-1") == (["ultrafast"],)
|
||||
|
||||
|
||||
def _router_with_team_owned_deployments():
|
||||
return litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "owned-by-teams",
|
||||
"litellm_params": {"model": "openai/gpt-5.5"},
|
||||
"model_info": {"team_id": "team-1", "service_tiers": ["priority"]},
|
||||
},
|
||||
{
|
||||
"model_name": "owned-by-teams",
|
||||
"litellm_params": {"model": "openai/paused-model"},
|
||||
"model_info": {"team_id": "team-2", "blocked": True, "service_tiers": ["paused"]},
|
||||
},
|
||||
{
|
||||
"model_name": "owned-by-teams",
|
||||
"litellm_params": {"model": "openai/team-2-model"},
|
||||
"model_info": {"team_id": "team-2", "service_tiers": ["flex"]},
|
||||
},
|
||||
{
|
||||
"model_name": "owned-and-shared",
|
||||
"litellm_params": {"model": "openai/team-1-model"},
|
||||
"model_info": {"team_id": "team-1", "service_tiers": ["priority"]},
|
||||
},
|
||||
{
|
||||
"model_name": "owned-and-shared",
|
||||
"litellm_params": {"model": "openai/shared-model"},
|
||||
"model_info": {"service_tiers": ["flex"]},
|
||||
},
|
||||
{"model_name": "openai/*", "litellm_params": {"model": "openai/*"}},
|
||||
],
|
||||
model_group_alias={"nickname": "owned-by-teams"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, team_id, upstream_model, service_tiers",
|
||||
[
|
||||
("owned-by-teams", "team-1", "openai/gpt-5.5", (["priority"],)),
|
||||
("owned-by-teams", "team-2", "openai/team-2-model", (["flex"],)),
|
||||
("nickname", "team-1", "openai/gpt-5.5", (["priority"],)),
|
||||
("nickname", "team-2", "openai/team-2-model", (["flex"],)),
|
||||
("owned-by-teams", "team-3", None, ()),
|
||||
("owned-by-teams", None, "openai/gpt-5.5", (["priority"], ["flex"])),
|
||||
("owned-and-shared", "team-1", "openai/team-1-model", (["priority"], ["flex"])),
|
||||
("owned-and-shared", "team-2", "openai/shared-model", (["flex"],)),
|
||||
("owned-and-shared", None, "openai/shared-model", (["flex"],)),
|
||||
("openai/gpt-5.5", "team-1", None, ()),
|
||||
("not-a-real-model", None, None, ()),
|
||||
],
|
||||
)
|
||||
def test_upstream_model_and_service_tiers_are_read_off_the_deployments_the_team_can_route_to(
|
||||
model_name, team_id, upstream_model, service_tiers
|
||||
):
|
||||
router = _router_with_team_owned_deployments()
|
||||
|
||||
assert router.get_routable_upstream_model(model_name, team_id) == upstream_model
|
||||
assert router.get_configured_service_tiers(model_name, team_id) == service_tiers
|
||||
|
||||
|
||||
def test_get_configured_display_name_treats_malformed_values_as_absent():
|
||||
malformed = ["", " ", 12345, ["Kimi K3"], {"name": "Kimi K3"}, True]
|
||||
router = litellm.Router(
|
||||
|
|
|
|||
|
|
@ -32,9 +32,9 @@ describe("Lens navigation", () => {
|
|||
expect(screen.getByText("Trace polling active")).toBeVisible();
|
||||
});
|
||||
|
||||
it("opens existing lens links directly in investigations", () => {
|
||||
it("opens existing lens links on findings", () => {
|
||||
renderWithProviders(<LensPage />, { searchParams: "?lens=saved-lens" });
|
||||
expect(screen.getByRole("tab", { name: "Investigations" })).toHaveAttribute("aria-selected", "true");
|
||||
expect(screen.getByRole("tab", { name: "Findings" })).toHaveAttribute("aria-selected", "true");
|
||||
expect(screen.getByText("Manage investigations")).toBeVisible();
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ describe("Lens interactive demo", () => {
|
|||
await chooseSelectOption(user, screen.getByRole("combobox", { name: "Filter by status" }), "Failed");
|
||||
expect(within(screen.getByRole("table", { name: "Agent runs" })).getAllByRole("row")).toHaveLength(4);
|
||||
await user.click(screen.getByRole("tab", { name: "Investigations" }));
|
||||
expect(await screen.findByRole("button", { name: /Support quality/ })).toBeVisible();
|
||||
expect(await screen.findByRole("row", { name: /Support quality/ })).toBeVisible();
|
||||
expect(screen.queryByRole("button", { name: "New investigation" })).not.toBeInTheDocument();
|
||||
expect(network).not.toHaveBeenCalled();
|
||||
expect(onUrlUpdate).not.toHaveBeenCalled();
|
||||
|
|
@ -60,9 +60,9 @@ describe("Lens interactive demo", () => {
|
|||
});
|
||||
await screen.findByRole("button", { name: "Preview sample" });
|
||||
await user.click(screen.getByRole("button", { name: "Preview sample" }));
|
||||
await user.click(await screen.findByRole("button", { name: /Support quality/ }));
|
||||
await user.click(await screen.findByRole("tab", { name: "Findings" }));
|
||||
network.mockClear();
|
||||
await user.click(screen.getByRole("button", { name: /Repeated lookups leave customers without an answer/ }));
|
||||
await user.click(await screen.findByRole("row", { name: /Repeated lookups leave customers without an answer/ }));
|
||||
const finding = screen.getByRole("dialog");
|
||||
expect(within(finding).getByText(/The support agent retries/)).toBeVisible();
|
||||
const summaries = within(finding).getAllByText("support_agent", { exact: true });
|
||||
|
|
@ -80,9 +80,7 @@ describe("Lens interactive demo", () => {
|
|||
within(screen.getByRole("dialog", { name: "Original run" })).getByRole("button", { name: "Close" }),
|
||||
);
|
||||
await user.click(within(screen.getByRole("dialog")).getByRole("button", { name: "Close" }));
|
||||
await user.click(screen.getByRole("tab", { name: "History" }));
|
||||
await user.click(screen.getAllByRole("button", { name: /runs reviewed/ })[1]);
|
||||
expect(await screen.findByText(/1 linked run · high priority/)).toBeVisible();
|
||||
expect(await screen.findByRole("table", { name: "Findings" })).toBeVisible();
|
||||
expect(network).not.toHaveBeenCalled();
|
||||
expect(onUrlUpdate).not.toHaveBeenCalled();
|
||||
await user.click(screen.getByRole("button", { name: "Exit demo" }));
|
||||
|
|
@ -127,7 +125,7 @@ describe("Lens interactive demo", () => {
|
|||
expect(await screen.findByRole("button", { name: "Preview sample" })).toBeVisible();
|
||||
const tabs = within(screen.getByRole("tablist", { name: "Lens" }));
|
||||
await user.click(tabs.getByRole("tab", { name: "Investigations" }));
|
||||
expect(await screen.findByRole("button", { name: new RegExp(saved.settings.name) })).toBeVisible();
|
||||
expect(await screen.findByRole("row", { name: new RegExp(saved.settings.name) })).toBeVisible();
|
||||
expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument();
|
||||
await user.click(tabs.getByRole("tab", { name: "Traces" }));
|
||||
await user.click(await screen.findByRole("button", { name: "Preview sample" }));
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import { isProxyAdminRole, isProxyAdminTierRole } from "@/utils/roles";
|
|||
import { InvestigationsView } from "./investigations/InvestigationsView";
|
||||
import { createLensDemo } from "./demo/createLensDemo";
|
||||
|
||||
type Tab = "traces" | "investigations";
|
||||
type Tab = "traces" | "findings" | "investigations";
|
||||
type WorkspaceProps = { accessToken: string; userRole: string; readOnly: boolean };
|
||||
|
||||
export function LensWorkspace(props: WorkspaceProps) {
|
||||
|
|
@ -59,12 +59,12 @@ function LensContent({
|
|||
const demo = useLensDemo();
|
||||
const [tab, setTab] = useQueryState(
|
||||
"tab",
|
||||
parseAsStringLiteral(["traces", "investigations"]).withOptions({ history: "push" }),
|
||||
parseAsStringLiteral(["traces", "findings", "investigations"]).withOptions({ history: "push" }),
|
||||
);
|
||||
const [lensId] = useQueryState("lens", parseAsString);
|
||||
const [demoTab, setDemoTab] = useState(initialTab);
|
||||
const [previewTarget, setPreviewTarget] = useState<HTMLDivElement | null>(null);
|
||||
const defaultTab = lensId ? "investigations" : "traces";
|
||||
const defaultTab = lensId ? "findings" : "traces";
|
||||
const activeTab = demo ? demoTab : tab ?? defaultTab;
|
||||
const openDemo = onDemo ? () => onDemo(activeTab) : undefined;
|
||||
return (
|
||||
|
|
@ -86,6 +86,9 @@ function LensContent({
|
|||
<TabsTrigger value="traces" className="px-3">
|
||||
Traces
|
||||
</TabsTrigger>
|
||||
<TabsTrigger value="findings" className="px-3">
|
||||
Findings
|
||||
</TabsTrigger>
|
||||
<TabsTrigger value="investigations" className="px-3">
|
||||
Investigations
|
||||
</TabsTrigger>
|
||||
|
|
@ -102,19 +105,23 @@ function LensContent({
|
|||
onDemo={activeTab === "traces" ? openDemo : undefined}
|
||||
/>
|
||||
</TabsContent>
|
||||
<TabsContent value="investigations" keepMounted={!!demo}>
|
||||
{demo || isProxyAdminTierRole(userRole) ? (
|
||||
<InvestigationsView
|
||||
accessToken={accessToken}
|
||||
readOnly={readOnly || !isProxyAdminRole(userRole)}
|
||||
onDemo={activeTab === "investigations" ? openDemo : undefined}
|
||||
/>
|
||||
) : (
|
||||
<p className="py-6 text-sm text-muted-foreground">
|
||||
Investigations require proxy administrator access. You can still view your traces.
|
||||
</p>
|
||||
)}
|
||||
</TabsContent>
|
||||
{(["findings", "investigations"] as const).map((view) => (
|
||||
<TabsContent key={view} value={view} keepMounted={!!demo} className="flex min-h-0 flex-col">
|
||||
{demo || isProxyAdminTierRole(userRole) ? (
|
||||
<InvestigationsView
|
||||
view={view}
|
||||
active={activeTab === view}
|
||||
accessToken={accessToken}
|
||||
readOnly={readOnly || !isProxyAdminRole(userRole)}
|
||||
onDemo={activeTab === view ? openDemo : undefined}
|
||||
/>
|
||||
) : (
|
||||
<p className="py-6 text-sm text-muted-foreground">
|
||||
Investigations require proxy administrator access. You can still view your traces.
|
||||
</p>
|
||||
)}
|
||||
</TabsContent>
|
||||
))}
|
||||
</Tabs>
|
||||
</main>
|
||||
</LensPreviewTarget.Provider>
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { z } from "zod";
|
||||
import type { RunWindow } from "../investigations/detail/RunNowDialog";
|
||||
import type { ApiClient } from "@/lib/http/client";
|
||||
import type { components } from "@/lib/http/schema";
|
||||
import type { ActivitySelection, Job, Lens, LensList, Sample, Settings, WorkerCreated } from "../model/types";
|
||||
|
|
@ -44,7 +45,8 @@ export interface LensApi {
|
|||
keys(alias: string, page: number, signal: AbortSignal): Promise<KeyPage>;
|
||||
keyInfo(keyId: string): Promise<KeyInfo>;
|
||||
saveLens(id: string | undefined, settings: Settings): Promise<Lens>;
|
||||
startRun(lensId: string): Promise<void>;
|
||||
startRun(lensId: string, request?: RunWindow): Promise<void>;
|
||||
watchAll(): Promise<components["schemas"]["WatchAllResult"]>;
|
||||
cancelRun(lensId: string): Promise<void>;
|
||||
reviewFinding(lensId: string, findingId: string, status: FindingStatus, reason: string): Promise<void>;
|
||||
registerWorker(analysisKeyId: string | null): Promise<WorkerCreated>;
|
||||
|
|
@ -109,7 +111,9 @@ export function liveLensApi(apiClient: ApiClient, accessToken: string): LensApi
|
|||
keyInfoSchema.parse(await apiClient.get("/key/info", { accessToken, query: { key: keyId } })).info,
|
||||
saveLens: (id, settings) =>
|
||||
apiClient.request<Lens>(id ? "PUT" : "POST", id ? `/lens/${id}` : "/lens", { accessToken, body: settings }),
|
||||
startRun: (lensId) => apiClient.post(`/lens/${lensId}/runs`, { accessToken, body: {} }),
|
||||
startRun: (lensId, request = {}) => apiClient.post(`/lens/${lensId}/runs`, { accessToken, body: request }),
|
||||
watchAll: () =>
|
||||
apiClient.post<components["schemas"]["WatchAllResult"]>("/lens/watch-all", { accessToken, body: {} }),
|
||||
cancelRun: (lensId) => apiClient.post(`/lens/${lensId}/cancel`, { accessToken, body: {} }),
|
||||
reviewFinding: (lensId, findingId, status, reason) =>
|
||||
apiClient.patch(`/lens/${lensId}/findings/${findingId}`, { accessToken, body: { status, reason } }),
|
||||
|
|
|
|||
|
|
@ -296,6 +296,8 @@ export function createLensDemoData(now = Date.now()) {
|
|||
: [],
|
||||
})),
|
||||
attempts: 1,
|
||||
steps: [],
|
||||
trigger: "schedule" as const,
|
||||
error: "",
|
||||
cost: sample.length * 0.012,
|
||||
coverage: {
|
||||
|
|
@ -359,6 +361,7 @@ function demoLensApi(data: LensDemoData): LensApi {
|
|||
keyInfo: notInDemo,
|
||||
saveLens: readOnly,
|
||||
startRun: readOnly,
|
||||
watchAll: async () => ({ watching: [], skipped: [] }),
|
||||
cancelRun: readOnly,
|
||||
reviewFinding: readOnly,
|
||||
registerWorker: readOnly,
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import { type Finding, type Sample } from "../model/types";
|
|||
|
||||
export function FindingSheet({
|
||||
finding,
|
||||
agents = [],
|
||||
sampledRuns,
|
||||
readOnly,
|
||||
busy,
|
||||
|
|
@ -18,6 +19,7 @@ export function FindingSheet({
|
|||
changeFinding,
|
||||
}: {
|
||||
finding?: Finding;
|
||||
agents?: readonly string[];
|
||||
sampledRuns: Sample["executions"];
|
||||
readOnly: boolean;
|
||||
busy: boolean;
|
||||
|
|
@ -46,6 +48,7 @@ export function FindingSheet({
|
|||
<SheetHeader>
|
||||
<SheetTitle className="pr-8 text-xl leading-snug">{finding.title}</SheetTitle>
|
||||
<SheetDescription>
|
||||
{agents.length > 0 && <span className="font-medium text-foreground">{agents.join(", ")} · </span>}
|
||||
{finding.kind === "issue" ? `${finding.priority} priority` : "Pattern"} ·{" "}
|
||||
{finding.occurrences?.length ?? 0} linked {finding.occurrences?.length === 1 ? "run" : "runs"}
|
||||
</SheetDescription>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,147 @@
|
|||
"use client";
|
||||
|
||||
import { ChevronRight } from "lucide-react";
|
||||
import { useState } from "react";
|
||||
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import { useNow } from "@/hooks/useNow";
|
||||
import { formatActivityTimestamp } from "@/utils/activityTimestamp";
|
||||
import { agoLabel } from "@/components/view_logs/TraceView/lensField";
|
||||
import { cn } from "@/lib/cva.config";
|
||||
|
||||
import { ALL_AGENTS, filterInbox, inboxAgents, inboxRows, type InboxRow, type Priority } from "../model/inbox";
|
||||
import type { Lens } from "../model/types";
|
||||
|
||||
const PRIORITY_DOT = { high: "bg-[#e5484d]", medium: "bg-amber-500", low: "bg-muted-foreground/50" } as const;
|
||||
const PRIORITY_ITEMS: { value: Priority | "all"; label: string }[] = [
|
||||
{ value: "all", label: "All priorities" },
|
||||
{ value: "high", label: "High" },
|
||||
{ value: "medium", label: "Medium" },
|
||||
{ value: "low", label: "Low" },
|
||||
];
|
||||
|
||||
const TH = "px-3 font-medium";
|
||||
const TH_NUM = "px-3 text-right font-medium";
|
||||
|
||||
function FilterSelect<T extends string>({
|
||||
label,
|
||||
value,
|
||||
items,
|
||||
onChange,
|
||||
width,
|
||||
}: {
|
||||
label: string;
|
||||
value: T;
|
||||
items: { value: T; label: string }[];
|
||||
onChange: (value: T) => void;
|
||||
width: string;
|
||||
}) {
|
||||
return (
|
||||
<Select items={items} value={value} onValueChange={(next: T | null) => next !== null && onChange(next)}>
|
||||
<SelectTrigger size="sm" className={cn("h-7 text-[12px]", width)} aria-label={label}>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{items.map((item) => (
|
||||
<SelectItem key={item.value} value={item.value}>
|
||||
{item.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
);
|
||||
}
|
||||
|
||||
export function FindingsInbox({ lenses, onOpen }: { lenses: readonly Lens[]; onOpen: (row: InboxRow) => void }) {
|
||||
const [agent, setAgent] = useState(ALL_AGENTS);
|
||||
const [priority, setPriority] = useState<Priority | "all">("all");
|
||||
const now = useNow(30000);
|
||||
const all = inboxRows(lenses);
|
||||
const rows = filterInbox(all, { agent, priority });
|
||||
const agentItems = [
|
||||
{ value: ALL_AGENTS, label: "All agents" },
|
||||
...inboxAgents(all).map((a) => ({ value: a, label: a })),
|
||||
];
|
||||
return (
|
||||
<div
|
||||
className="flex min-h-[420px] flex-1 flex-col overflow-hidden border-y border-border bg-card"
|
||||
data-testid="findings-inbox"
|
||||
>
|
||||
<div className="flex min-h-10 shrink-0 flex-wrap items-center gap-2 border-b border-border bg-card px-3 py-1.5">
|
||||
<FilterSelect
|
||||
label="Filter by agent"
|
||||
value={agent}
|
||||
items={agentItems}
|
||||
onChange={setAgent}
|
||||
width="min-w-[150px]"
|
||||
/>
|
||||
<FilterSelect
|
||||
label="Filter by priority"
|
||||
value={priority}
|
||||
items={PRIORITY_ITEMS}
|
||||
onChange={setPriority}
|
||||
width="min-w-[130px]"
|
||||
/>
|
||||
</div>
|
||||
<div className="min-h-0 flex-1 overflow-auto">
|
||||
<table aria-label="Findings" className="w-full min-w-[900px] table-fixed border-collapse text-left">
|
||||
<thead className="sticky top-0 z-sticky bg-muted/40 backdrop-blur">
|
||||
<tr className="h-8 border-b border-border text-[10px] tracking-[0.08em] text-muted-foreground uppercase">
|
||||
<th className={`w-[96px] ${TH}`}>Priority</th>
|
||||
<th className={TH}>Finding</th>
|
||||
<th className={`w-[180px] ${TH}`}>Agent</th>
|
||||
<th className={`w-[64px] ${TH_NUM}`}>Runs</th>
|
||||
<th className={`w-[96px] ${TH}`}>Last seen</th>
|
||||
<th className="w-8" />
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows.map((row) => (
|
||||
<tr
|
||||
key={row.key}
|
||||
onClick={() => onOpen(row)}
|
||||
data-testid="inbox-row"
|
||||
aria-label={row.title}
|
||||
className="h-9 cursor-pointer border-b border-border/60 text-[12px] transition-colors duration-150 hover:bg-trace-row-hover motion-reduce:transition-none"
|
||||
>
|
||||
<td className="px-3">
|
||||
<span className="inline-flex items-center gap-1.5 text-muted-foreground">
|
||||
<span className={cn("size-1.5 rounded-full", PRIORITY_DOT[row.priority])} />
|
||||
{row.priority}
|
||||
</span>
|
||||
</td>
|
||||
<td className="px-3" title={row.suggestion ? `Fix: ${row.suggestion}` : undefined}>
|
||||
<span className="block truncate text-foreground">{row.title}</span>
|
||||
</td>
|
||||
<td className="truncate px-3 text-muted-foreground" title={row.agents.join(", ")}>
|
||||
{row.agents.join(", ")}
|
||||
</td>
|
||||
<td className="px-3 text-right font-mono tabular-nums text-foreground">{row.runs}</td>
|
||||
<td
|
||||
className="px-3 font-mono text-[11px] tabular-nums text-muted-foreground"
|
||||
title={formatActivityTimestamp(row.lastSeen)}
|
||||
>
|
||||
{agoLabel(Date.parse(row.lastSeen), now)}
|
||||
</td>
|
||||
<td>
|
||||
<ChevronRight className="size-3 text-muted-foreground/60" />
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
{rows.length === 0 && (
|
||||
<div className="py-16 text-center text-[12px] text-muted-foreground">
|
||||
{all.length === 0
|
||||
? "No open findings yet. New problems show up here as soon as an investigation spots them."
|
||||
: "No findings match these filters."}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<footer className="flex h-8 shrink-0 items-center border-t border-border bg-muted/40 px-3 font-mono text-[11px] text-muted-foreground">
|
||||
{rows.length} {rows.length === 1 ? "finding" : "findings"}
|
||||
{rows.length !== all.length && ` of ${all.length}`}
|
||||
</footer>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
"use client";
|
||||
|
||||
import { useContext, type ReactNode } from "react";
|
||||
import { createPortal } from "react-dom";
|
||||
|
||||
import { LensPreviewTarget } from "../LensPreviewButton";
|
||||
|
||||
export function HeaderActions({ children }: { children: ReactNode }) {
|
||||
const target = useContext(LensPreviewTarget);
|
||||
const actions = <div className="flex flex-wrap items-center gap-3">{children}</div>;
|
||||
if (target === null) return null;
|
||||
return target ? createPortal(actions, target) : actions;
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue