mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge remote-tracking branch 'origin/main' into litellm_wt_0918_be27
Only conflict was the litellm.types.utils import list in tests/test_litellm/test_utils.py: main added ImageResponse for its image call-type coverage while this branch added EmbeddingResponse for the partial embedding cache tests. Kept both. litellm/utils.py and litellm/caching/caching_handler.py auto-merged; the diff against the new base is unchanged from before the merge.
This commit is contained in:
commit
cb69cebccd
158 changed files with 8997 additions and 1720 deletions
1
.github/e2e-stack/select_tests.py
vendored
1
.github/e2e-stack/select_tests.py
vendored
|
|
@ -11,6 +11,7 @@ UNSUPPORTED: Final = re.compile(
|
|||
r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"
|
||||
r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$"
|
||||
r"|^tests/e2e/logging/test_langsmith_batch_serialization_e2e\.py$"
|
||||
r"|^tests/e2e/secret_manager/"
|
||||
)
|
||||
HARNESS: Final = re.compile(
|
||||
r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"
|
||||
|
|
|
|||
32
.github/workflows/_test-unit-base.yml
vendored
32
.github/workflows/_test-unit-base.yml
vendored
|
|
@ -4,7 +4,13 @@ on:
|
|||
workflow_call:
|
||||
inputs:
|
||||
test-path:
|
||||
description: "Pytest path(s) to run"
|
||||
description: >-
|
||||
Space-separated pytest paths to run. A path that no longer exists is
|
||||
dropped with a warning instead of being passed to pytest, because one
|
||||
missing path makes pytest-xdist collect nothing and report exit 5, which
|
||||
the step treats as a drained shard. Options are passed through as
|
||||
written, so use the `--flag=value` form: a bare `--ignore path` would
|
||||
have its path existence-checked like any other token.
|
||||
required: true
|
||||
type: string
|
||||
workers:
|
||||
|
|
@ -165,14 +171,22 @@ jobs:
|
|||
DIST: ${{ inputs.dist }}
|
||||
COVERAGE_CORE: sysmon
|
||||
run: |
|
||||
found_path=false
|
||||
for path in ${TEST_PATH}; do
|
||||
if [ -e "${path%%::*}" ]; then
|
||||
found_path=true
|
||||
break
|
||||
fi
|
||||
pytest_args=()
|
||||
existing_paths=0
|
||||
for token in ${TEST_PATH:?}; do
|
||||
case "${token}" in
|
||||
-*) pytest_args+=("${token}") ;;
|
||||
*)
|
||||
if [ -e "${token%%::*}" ]; then
|
||||
pytest_args+=("${token}")
|
||||
existing_paths=$((existing_paths + 1))
|
||||
else
|
||||
echo "::warning::${token} does not exist; drop it from this shard's test-path"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
if [ "$found_path" = false ]; then
|
||||
if [ "${existing_paths}" -eq 0 ]; then
|
||||
echo "No path in TEST_PATH exists (${TEST_PATH}); nothing to run"
|
||||
exit 0
|
||||
fi
|
||||
|
|
@ -181,7 +195,7 @@ jobs:
|
|||
xdist_args=(-n "${WORKERS}" --dist="${DIST}")
|
||||
fi
|
||||
set +e
|
||||
uv run --no-sync pytest ${TEST_PATH:?} \
|
||||
uv run --no-sync pytest "${pytest_args[@]}" \
|
||||
--tb=short -vv \
|
||||
--maxfail="${MAX_FAILURES}" \
|
||||
"${xdist_args[@]}" \
|
||||
|
|
|
|||
8
.github/workflows/test-unit.yml
vendored
8
.github/workflows/test-unit.yml
vendored
|
|
@ -107,26 +107,18 @@ jobs:
|
|||
tests/test_litellm/batches
|
||||
tests/test_litellm/secret_managers
|
||||
tests/test_litellm/a2a_protocol
|
||||
tests/test_litellm/anthropic_interface
|
||||
tests/test_litellm/chat_completions
|
||||
tests/test_litellm/completion_extras
|
||||
tests/test_litellm/compression
|
||||
tests/test_litellm/containers
|
||||
tests/test_litellm/endpoints
|
||||
tests/test_litellm/models
|
||||
tests/test_litellm/repositories
|
||||
tests/test_litellm/images
|
||||
tests/test_litellm/interactions
|
||||
tests/test_litellm/messages
|
||||
tests/test_litellm/ocr
|
||||
tests/test_litellm/passthrough
|
||||
tests/test_litellm/rag
|
||||
tests/test_litellm/realtime_api
|
||||
tests/test_litellm/rerank_api
|
||||
tests/test_litellm/rust_bridge
|
||||
tests/test_litellm/sandbox
|
||||
tests/test_litellm/skills
|
||||
tests/test_litellm/test_router
|
||||
tests/test_litellm/vector_stores
|
||||
tests/test_litellm/videos
|
||||
tests/test_litellm/test_*.py
|
||||
|
|
|
|||
46
litellm-rust/Cargo.lock
generated
46
litellm-rust/Cargo.lock
generated
|
|
@ -2956,6 +2956,14 @@ dependencies = [
|
|||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-cost"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"proptest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-framing"
|
||||
version = "0.1.0"
|
||||
|
|
@ -3046,6 +3054,20 @@ dependencies = [
|
|||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-model-catalog"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"indexmap 2.14.0",
|
||||
"litellm-model-catalog",
|
||||
"rstest",
|
||||
"schemars 1.2.2",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.19",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-python-bridge"
|
||||
version = "0.1.0"
|
||||
|
|
@ -4748,10 +4770,23 @@ checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a"
|
|||
dependencies = [
|
||||
"dyn-clone",
|
||||
"ref-cast",
|
||||
"schemars_derive",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schemars_derive"
|
||||
version = "1.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"serde_derive_internals",
|
||||
"syn 3.0.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.2.0"
|
||||
|
|
@ -4840,6 +4875,17 @@ dependencies = [
|
|||
"syn 3.0.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive_internals"
|
||||
version = "0.30.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.150"
|
||||
|
|
|
|||
14
litellm-rust/crates/cost/Cargo.toml
Normal file
14
litellm-rust/crates/cost/Cargo.toml
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
[package]
|
||||
name = "litellm-cost"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
criterion.workspace = true
|
||||
proptest.workspace = true
|
||||
|
||||
[[bench]]
|
||||
name = "calculate"
|
||||
harness = false
|
||||
13
litellm-rust/crates/cost/README.md
Normal file
13
litellm-rust/crates/cost/README.md
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
# litellm-cost
|
||||
|
||||
This crate calculates text token charges from rates and usage supplied by its caller. It is standalone and has no Python bridge or proxy integration
|
||||
|
||||
Call `compile(&pricing)` once for an immutable plan, then `plan.calculate(&request)` for each supported request. `calculate(&pricing, &request)` compiles on each call. A successful result exposes pre-multiplier component costs, selected rates, the multiplier, and derived `input()`, `output()`, and `total()` values
|
||||
|
||||
The caller states whether `prompt_tokens` includes cache tokens. Threshold selection uses total input tokens for either convention and selects one rate for the whole request. Thresholds are sorted when compiled, and duplicate thresholds or tier overrides fail deterministically. `Fast` selects priority rates; unknown tiers use standard rates
|
||||
|
||||
`Rate::Missing`, `Rate::Null`, and `Rate::Value(0.0)` remain distinct. Missing cache rates fall back to the selected input rate, and an absent one-hour write rate falls back to the selected write rate. Missing input or output rates return typed errors, including for zero usage. Python's sparse-entry behavior remains outside this native contract
|
||||
|
||||
The supported off-peak shape is one non-wrapping UTC daily window. The caller supplies the applicable regional multiplier after provider-specific selection. Negative or non-finite rates, ambiguous rules, inconsistent cache counts, incomplete write splits, invalid windows and overflow return errors. Callers must decline unsupported inputs before native execution if their public contract accepts those shapes
|
||||
|
||||
This crate does not select models, read catalogs, fetch provider prices, normalize multimodal usage, process provider-reported costs, or calculate non-token charges. It does not change proxy behavior. The reference fixture was generated by `tests/generate_python_reference.py` against the Python implementation at the commit recorded in `tests/python_reference.tsv`, using synthetic rates and fixed usage
|
||||
66
litellm-rust/crates/cost/benches/calculate.rs
Normal file
66
litellm-rust/crates/cost/benches/calculate.rs
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
use criterion::{Criterion, criterion_group, criterion_main};
|
||||
use litellm_cost::{
|
||||
Pricing, PromptConvention, Rate, Rates, Request, ServiceTier, ThresholdPolicy, ThresholdRates,
|
||||
Usage, calculate, compile,
|
||||
};
|
||||
use std::hint::black_box;
|
||||
|
||||
fn bench(c: &mut Criterion) {
|
||||
let pricing = Pricing {
|
||||
standard: Rates {
|
||||
input: Rate::Value(0.000002),
|
||||
output: Rate::Value(0.000008),
|
||||
cache_read: Rate::Value(0.0000005),
|
||||
cache_write: Rate::Missing,
|
||||
cache_write_1h: Rate::Missing,
|
||||
},
|
||||
tiers: &[],
|
||||
thresholds: &[],
|
||||
off_peak: None,
|
||||
};
|
||||
let request = Request {
|
||||
usage: Usage {
|
||||
prompt_tokens: 1000,
|
||||
completion_tokens: 200,
|
||||
cache_read_tokens: 250,
|
||||
cache_write_tokens: 0,
|
||||
cache_write_5m_tokens: None,
|
||||
cache_write_1h_tokens: None,
|
||||
prompt_convention: PromptConvention::IncludesCache,
|
||||
},
|
||||
service_tier: ServiceTier::Standard,
|
||||
threshold_policy: ThresholdPolicy::Exclusive,
|
||||
region_multiplier: None,
|
||||
billed_at_utc_minute: None,
|
||||
};
|
||||
let plan = compile(&pricing).unwrap();
|
||||
c.bench_function("native_compiled_calculation", |b| {
|
||||
b.iter(|| black_box(plan.calculate(black_box(&request)).unwrap()))
|
||||
});
|
||||
c.bench_function("native_full_wrapper", |b| {
|
||||
b.iter(|| black_box(calculate(black_box(&pricing), black_box(&request)).unwrap()))
|
||||
});
|
||||
c.bench_function("native_rate_compilation", |b| {
|
||||
b.iter(|| black_box(compile(black_box(&pricing)).unwrap()))
|
||||
});
|
||||
let threshold = ThresholdRates {
|
||||
above_prompt_tokens: 1000,
|
||||
standard: Rates {
|
||||
input: Rate::Value(0.000004),
|
||||
output: Rate::Value(0.000016),
|
||||
..Rates::EMPTY
|
||||
},
|
||||
tiers: &[],
|
||||
};
|
||||
let threshold_pricing = Pricing {
|
||||
thresholds: &[threshold],
|
||||
..pricing
|
||||
};
|
||||
let threshold_plan = compile(&threshold_pricing).unwrap();
|
||||
c.bench_function("native_threshold_boundary", |b| {
|
||||
b.iter(|| black_box(threshold_plan.calculate(black_box(&request)).unwrap()))
|
||||
});
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench);
|
||||
criterion_main!(benches);
|
||||
40
litellm-rust/crates/cost/examples/charge.rs
Normal file
40
litellm-rust/crates/cost/examples/charge.rs
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
use litellm_cost::{
|
||||
Pricing, PromptConvention, Rate, Rates, Request, ServiceTier, ThresholdPolicy, Usage, compile,
|
||||
};
|
||||
|
||||
fn main() {
|
||||
let pricing = Pricing {
|
||||
standard: Rates {
|
||||
input: Rate::Value(2.0),
|
||||
output: Rate::Value(4.0),
|
||||
cache_read: Rate::Value(0.5),
|
||||
cache_write: Rate::Value(3.0),
|
||||
cache_write_1h: Rate::Missing,
|
||||
},
|
||||
tiers: &[],
|
||||
thresholds: &[],
|
||||
off_peak: None,
|
||||
};
|
||||
let request = Request {
|
||||
usage: Usage {
|
||||
prompt_tokens: 100,
|
||||
completion_tokens: 20,
|
||||
cache_read_tokens: 25,
|
||||
cache_write_tokens: 10,
|
||||
cache_write_5m_tokens: None,
|
||||
cache_write_1h_tokens: None,
|
||||
prompt_convention: PromptConvention::IncludesCache,
|
||||
},
|
||||
service_tier: ServiceTier::Standard,
|
||||
threshold_policy: ThresholdPolicy::Exclusive,
|
||||
region_multiplier: None,
|
||||
billed_at_utc_minute: None,
|
||||
};
|
||||
let cost = compile(&pricing).unwrap().calculate(&request).unwrap();
|
||||
println!(
|
||||
"input={} output={} total={}",
|
||||
cost.input(),
|
||||
cost.output(),
|
||||
cost.total()
|
||||
);
|
||||
}
|
||||
405
litellm-rust/crates/cost/src/lib.rs
Normal file
405
litellm-rust/crates/cost/src/lib.rs
Normal file
|
|
@ -0,0 +1,405 @@
|
|||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub enum Rate {
|
||||
Missing,
|
||||
Null,
|
||||
Value(f64),
|
||||
}
|
||||
|
||||
impl Rate {
|
||||
fn value(self) -> Option<f64> {
|
||||
match self {
|
||||
Self::Value(value) => Some(value),
|
||||
Self::Missing | Self::Null => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn or(self, fallback: Self) -> Self {
|
||||
if self.value().is_some() {
|
||||
self
|
||||
} else {
|
||||
fallback
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Rates {
|
||||
pub input: Rate,
|
||||
pub output: Rate,
|
||||
pub cache_read: Rate,
|
||||
pub cache_write: Rate,
|
||||
pub cache_write_1h: Rate,
|
||||
}
|
||||
|
||||
impl Rates {
|
||||
pub const EMPTY: Self = Self {
|
||||
input: Rate::Missing,
|
||||
output: Rate::Missing,
|
||||
cache_read: Rate::Missing,
|
||||
cache_write: Rate::Missing,
|
||||
cache_write_1h: Rate::Missing,
|
||||
};
|
||||
|
||||
fn overlay(self, base: Self) -> Self {
|
||||
Self {
|
||||
input: self.input.or(base.input),
|
||||
output: self.output.or(base.output),
|
||||
cache_read: self.cache_read.or(base.cache_read),
|
||||
cache_write: self.cache_write.or(base.cache_write),
|
||||
cache_write_1h: self.cache_write_1h.or(base.cache_write_1h),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum ServiceTier {
|
||||
Standard,
|
||||
Flex,
|
||||
Priority,
|
||||
Fast,
|
||||
Ultrafast,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum ThresholdPolicy {
|
||||
Exclusive,
|
||||
Inclusive,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum PromptConvention {
|
||||
IncludesCache,
|
||||
ExcludesCache,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Usage {
|
||||
pub prompt_tokens: u64,
|
||||
pub completion_tokens: u64,
|
||||
pub cache_read_tokens: u64,
|
||||
pub cache_write_tokens: u64,
|
||||
pub cache_write_5m_tokens: Option<u64>,
|
||||
pub cache_write_1h_tokens: Option<u64>,
|
||||
pub prompt_convention: PromptConvention,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct TierRates {
|
||||
pub tier: ServiceTier,
|
||||
pub rates: Rates,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct ThresholdRates<'a> {
|
||||
pub above_prompt_tokens: u64,
|
||||
pub standard: Rates,
|
||||
pub tiers: &'a [TierRates],
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct OffPeakRates {
|
||||
pub start_utc_minute: u16,
|
||||
pub end_utc_minute: u16,
|
||||
pub rates: Rates,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Pricing<'a> {
|
||||
pub standard: Rates,
|
||||
pub tiers: &'a [TierRates],
|
||||
pub thresholds: &'a [ThresholdRates<'a>],
|
||||
pub off_peak: Option<OffPeakRates>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Request {
|
||||
pub usage: Usage,
|
||||
pub service_tier: ServiceTier,
|
||||
pub threshold_policy: ThresholdPolicy,
|
||||
pub region_multiplier: Option<f64>,
|
||||
pub billed_at_utc_minute: Option<u16>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Cost {
|
||||
pub uncached_input: f64,
|
||||
pub cache_read: f64,
|
||||
pub cache_write_5m: f64,
|
||||
pub cache_write_1h: f64,
|
||||
pub output: f64,
|
||||
pub multiplier: f64,
|
||||
pub rates: EffectiveRates,
|
||||
}
|
||||
|
||||
impl Cost {
|
||||
pub fn input(self) -> f64 {
|
||||
(self.uncached_input + self.cache_read + self.cache_write_5m + self.cache_write_1h)
|
||||
* self.multiplier
|
||||
}
|
||||
|
||||
pub fn output(self) -> f64 {
|
||||
self.output * self.multiplier
|
||||
}
|
||||
|
||||
pub fn total(self) -> f64 {
|
||||
self.input() + self.output()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct EffectiveRates {
|
||||
pub input: f64,
|
||||
pub output: f64,
|
||||
pub cache_read: f64,
|
||||
pub cache_write_5m: f64,
|
||||
pub cache_write_1h: f64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum PricingError {
|
||||
MissingInputRate,
|
||||
MissingOutputRate,
|
||||
InvalidRate,
|
||||
InvalidRegionMultiplier,
|
||||
InvalidBillingTime,
|
||||
InvalidOffPeakWindow,
|
||||
CacheExceedsPrompt,
|
||||
InvalidCacheWriteDetails,
|
||||
TokenCountOverflow,
|
||||
DuplicateTier,
|
||||
DuplicateThreshold,
|
||||
DuplicateThresholdTier,
|
||||
}
|
||||
|
||||
fn selected_tier(tier: ServiceTier) -> ServiceTier {
|
||||
if tier == ServiceTier::Fast {
|
||||
ServiceTier::Priority
|
||||
} else {
|
||||
tier
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct CompiledThreshold {
|
||||
above_prompt_tokens: u64,
|
||||
standard: Rates,
|
||||
tiers: Vec<TierRates>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PricingPlan {
|
||||
standard: Rates,
|
||||
tiers: Vec<TierRates>,
|
||||
thresholds: Vec<CompiledThreshold>,
|
||||
off_peak: Option<OffPeakRates>,
|
||||
}
|
||||
|
||||
fn valid_rates(rates: Rates) -> bool {
|
||||
[
|
||||
rates.input,
|
||||
rates.output,
|
||||
rates.cache_read,
|
||||
rates.cache_write,
|
||||
rates.cache_write_1h,
|
||||
]
|
||||
.into_iter()
|
||||
.all(|rate| {
|
||||
rate.value()
|
||||
.is_none_or(|value| value.is_finite() && value >= 0.0)
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_tiers(tiers: &[TierRates], duplicate: PricingError) -> Result<(), PricingError> {
|
||||
if tiers.iter().any(|entry| !valid_rates(entry.rates)) {
|
||||
return Err(PricingError::InvalidRate);
|
||||
}
|
||||
if tiers.iter().enumerate().any(|(index, entry)| {
|
||||
matches!(
|
||||
entry.tier,
|
||||
ServiceTier::Standard | ServiceTier::Unknown | ServiceTier::Fast
|
||||
) || tiers[..index]
|
||||
.iter()
|
||||
.any(|previous| previous.tier == entry.tier)
|
||||
}) {
|
||||
return Err(duplicate);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn compile(pricing: &Pricing<'_>) -> Result<PricingPlan, PricingError> {
|
||||
if !valid_rates(pricing.standard) {
|
||||
return Err(PricingError::InvalidRate);
|
||||
}
|
||||
validate_tiers(pricing.tiers, PricingError::DuplicateTier)?;
|
||||
if let Some(window) = pricing.off_peak {
|
||||
if window.start_utc_minute >= 1440
|
||||
|| window.end_utc_minute > 1440
|
||||
|| window.start_utc_minute >= window.end_utc_minute
|
||||
{
|
||||
return Err(PricingError::InvalidOffPeakWindow);
|
||||
}
|
||||
if !valid_rates(window.rates) {
|
||||
return Err(PricingError::InvalidRate);
|
||||
}
|
||||
}
|
||||
let mut thresholds: Vec<_> = pricing
|
||||
.thresholds
|
||||
.iter()
|
||||
.map(|entry| {
|
||||
if !valid_rates(entry.standard) {
|
||||
return Err(PricingError::InvalidRate);
|
||||
}
|
||||
validate_tiers(entry.tiers, PricingError::DuplicateThresholdTier)?;
|
||||
Ok(CompiledThreshold {
|
||||
above_prompt_tokens: entry.above_prompt_tokens,
|
||||
standard: entry.standard,
|
||||
tiers: entry.tiers.to_vec(),
|
||||
})
|
||||
})
|
||||
.collect::<Result<_, _>>()?;
|
||||
thresholds.sort_unstable_by_key(|entry| entry.above_prompt_tokens);
|
||||
if thresholds
|
||||
.windows(2)
|
||||
.any(|pair| pair[0].above_prompt_tokens == pair[1].above_prompt_tokens)
|
||||
{
|
||||
return Err(PricingError::DuplicateThreshold);
|
||||
}
|
||||
Ok(PricingPlan {
|
||||
standard: pricing.standard,
|
||||
tiers: pricing.tiers.to_vec(),
|
||||
thresholds,
|
||||
off_peak: pricing.off_peak,
|
||||
})
|
||||
}
|
||||
|
||||
impl PricingPlan {
|
||||
fn resolve_rates(
|
||||
&self,
|
||||
request: &Request,
|
||||
threshold_tokens: u64,
|
||||
) -> Result<Rates, PricingError> {
|
||||
let tier = selected_tier(request.service_tier);
|
||||
let base = self
|
||||
.tiers
|
||||
.iter()
|
||||
.find(|entry| tier != ServiceTier::Standard && entry.tier == tier)
|
||||
.map_or(self.standard, |entry| entry.rates.overlay(self.standard));
|
||||
let threshold = self.thresholds.iter().rev().find(|entry| {
|
||||
threshold_tokens > entry.above_prompt_tokens
|
||||
|| (request.threshold_policy == ThresholdPolicy::Inclusive
|
||||
&& threshold_tokens == entry.above_prompt_tokens)
|
||||
});
|
||||
let selected = threshold.map_or(base, |entry| {
|
||||
let standard = entry.standard.overlay(base);
|
||||
entry
|
||||
.tiers
|
||||
.iter()
|
||||
.find(|specific| tier != ServiceTier::Standard && specific.tier == tier)
|
||||
.map_or(standard, |specific| specific.rates.overlay(standard))
|
||||
});
|
||||
match self.off_peak {
|
||||
None => Ok(selected),
|
||||
Some(window) => {
|
||||
if window.start_utc_minute >= 1440
|
||||
|| window.end_utc_minute > 1440
|
||||
|| window.start_utc_minute >= window.end_utc_minute
|
||||
{
|
||||
return Err(PricingError::InvalidOffPeakWindow);
|
||||
}
|
||||
let minute = request
|
||||
.billed_at_utc_minute
|
||||
.ok_or(PricingError::InvalidBillingTime)?;
|
||||
if minute >= 1440 {
|
||||
return Err(PricingError::InvalidBillingTime);
|
||||
}
|
||||
if (window.start_utc_minute..window.end_utc_minute).contains(&minute) {
|
||||
Ok(window.rates.overlay(selected))
|
||||
} else {
|
||||
Ok(selected)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn checked_rate(rate: Rate, missing: PricingError) -> Result<f64, PricingError> {
|
||||
let value = rate.value().ok_or(missing)?;
|
||||
if !value.is_finite() || value < 0.0 {
|
||||
return Err(PricingError::InvalidRate);
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
pub fn calculate(&self, request: &Request) -> Result<Cost, PricingError> {
|
||||
let usage = request.usage;
|
||||
let cached = usage
|
||||
.cache_read_tokens
|
||||
.checked_add(usage.cache_write_tokens)
|
||||
.ok_or(PricingError::TokenCountOverflow)?;
|
||||
let (regular, threshold_tokens) = match usage.prompt_convention {
|
||||
PromptConvention::IncludesCache => (
|
||||
usage
|
||||
.prompt_tokens
|
||||
.checked_sub(cached)
|
||||
.ok_or(PricingError::CacheExceedsPrompt)?,
|
||||
usage.prompt_tokens,
|
||||
),
|
||||
PromptConvention::ExcludesCache => (
|
||||
usage.prompt_tokens,
|
||||
usage
|
||||
.prompt_tokens
|
||||
.checked_add(cached)
|
||||
.ok_or(PricingError::TokenCountOverflow)?,
|
||||
),
|
||||
};
|
||||
let writes = match (usage.cache_write_5m_tokens, usage.cache_write_1h_tokens) {
|
||||
(None, None) => (usage.cache_write_tokens, 0),
|
||||
(Some(five), Some(one)) if five.checked_add(one) == Some(usage.cache_write_tokens) => {
|
||||
(five, one)
|
||||
}
|
||||
_ => return Err(PricingError::InvalidCacheWriteDetails),
|
||||
};
|
||||
let rates = self.resolve_rates(request, threshold_tokens)?;
|
||||
let input = Self::checked_rate(rates.input, PricingError::MissingInputRate)?;
|
||||
let output = Self::checked_rate(rates.output, PricingError::MissingOutputRate)?;
|
||||
let read = Self::checked_rate(
|
||||
rates.cache_read.or(rates.input),
|
||||
PricingError::MissingInputRate,
|
||||
)?;
|
||||
let write = Self::checked_rate(
|
||||
rates.cache_write.or(rates.input),
|
||||
PricingError::MissingInputRate,
|
||||
)?;
|
||||
let write_1h = Self::checked_rate(
|
||||
rates.cache_write_1h.or(rates.cache_write).or(rates.input),
|
||||
PricingError::MissingInputRate,
|
||||
)?;
|
||||
let multiplier = request.region_multiplier.unwrap_or(1.0);
|
||||
if !multiplier.is_finite() || multiplier <= 0.0 {
|
||||
return Err(PricingError::InvalidRegionMultiplier);
|
||||
}
|
||||
let cost = Cost {
|
||||
uncached_input: regular as f64 * input,
|
||||
cache_read: usage.cache_read_tokens as f64 * read,
|
||||
cache_write_5m: writes.0 as f64 * write,
|
||||
cache_write_1h: writes.1 as f64 * write_1h,
|
||||
output: usage.completion_tokens as f64 * output,
|
||||
multiplier,
|
||||
rates: EffectiveRates {
|
||||
input,
|
||||
output,
|
||||
cache_read: read,
|
||||
cache_write_5m: write,
|
||||
cache_write_1h: write_1h,
|
||||
},
|
||||
};
|
||||
if !cost.total().is_finite() {
|
||||
return Err(PricingError::TokenCountOverflow);
|
||||
}
|
||||
Ok(cost)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn calculate(pricing: &Pricing<'_>, request: &Request) -> Result<Cost, PricingError> {
|
||||
compile(pricing)?.calculate(request)
|
||||
}
|
||||
458
litellm-rust/crates/cost/tests/calculation.rs
Normal file
458
litellm-rust/crates/cost/tests/calculation.rs
Normal file
|
|
@ -0,0 +1,458 @@
|
|||
use litellm_cost::{
|
||||
OffPeakRates, Pricing, PricingError, PromptConvention, Rate, Rates, Request, ServiceTier,
|
||||
ThresholdPolicy, ThresholdRates, TierRates, Usage, calculate, compile,
|
||||
};
|
||||
|
||||
fn rates(input: Rate, output: Rate) -> Rates {
|
||||
Rates {
|
||||
input,
|
||||
output,
|
||||
..Rates::EMPTY
|
||||
}
|
||||
}
|
||||
|
||||
fn request() -> Request {
|
||||
Request {
|
||||
usage: Usage {
|
||||
prompt_tokens: 100,
|
||||
completion_tokens: 20,
|
||||
cache_read_tokens: 25,
|
||||
cache_write_tokens: 10,
|
||||
cache_write_5m_tokens: None,
|
||||
cache_write_1h_tokens: None,
|
||||
prompt_convention: PromptConvention::IncludesCache,
|
||||
},
|
||||
service_tier: ServiceTier::Standard,
|
||||
threshold_policy: ThresholdPolicy::Exclusive,
|
||||
region_multiplier: None,
|
||||
billed_at_utc_minute: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn pricing(standard: Rates) -> Pricing<'static> {
|
||||
Pricing {
|
||||
standard,
|
||||
tiers: &[],
|
||||
thresholds: &[],
|
||||
off_peak: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn breakdown_and_total_agree() {
|
||||
let standard = Rates {
|
||||
cache_read: Rate::Value(0.5),
|
||||
cache_write: Rate::Value(3.0),
|
||||
..rates(Rate::Value(2.0), Rate::Value(4.0))
|
||||
};
|
||||
let result = calculate(&pricing(standard), &request()).unwrap();
|
||||
assert_eq!(result.uncached_input, 65.0 * 2.0);
|
||||
assert_eq!(result.cache_read, 25.0 * 0.5);
|
||||
assert_eq!(result.cache_write_5m, 10.0 * 3.0);
|
||||
assert_eq!(result.output(), 20.0 * 4.0);
|
||||
assert_eq!(result.total(), result.input() + result.output());
|
||||
assert_eq!(result.rates.cache_read, 0.5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_null_and_zero_cache_rates_are_distinct() {
|
||||
let base = rates(Rate::Value(2.0), Rate::Value(4.0));
|
||||
for read in [Rate::Missing, Rate::Null] {
|
||||
let standard = Rates {
|
||||
cache_read: read,
|
||||
..base
|
||||
};
|
||||
assert_eq!(
|
||||
calculate(&pricing(standard), &request()).unwrap().input(),
|
||||
200.0
|
||||
);
|
||||
}
|
||||
let standard = Rates {
|
||||
cache_read: Rate::Value(0.0),
|
||||
cache_write: Rate::Value(0.0),
|
||||
..base
|
||||
};
|
||||
assert_eq!(
|
||||
calculate(&pricing(standard), &request()).unwrap().input(),
|
||||
130.0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn equivalent_prompt_conventions_select_the_same_threshold() {
|
||||
let threshold = ThresholdRates {
|
||||
above_prompt_tokens: 90,
|
||||
standard: rates(Rate::Value(5.0), Rate::Value(8.0)),
|
||||
tiers: &[],
|
||||
};
|
||||
let specification = Pricing {
|
||||
standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
|
||||
tiers: &[],
|
||||
thresholds: &[threshold],
|
||||
off_peak: None,
|
||||
};
|
||||
let included = request();
|
||||
let excluded = Request {
|
||||
usage: Usage {
|
||||
prompt_tokens: 65,
|
||||
prompt_convention: PromptConvention::ExcludesCache,
|
||||
..included.usage
|
||||
},
|
||||
..included
|
||||
};
|
||||
let plan = compile(&specification).unwrap();
|
||||
assert_eq!(plan.calculate(&included), plan.calculate(&excluded));
|
||||
assert_eq!(plan.calculate(&included).unwrap().rates.input, 5.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_writes_and_invalid_accounting() {
|
||||
let standard = Rates {
|
||||
cache_read: Rate::Value(0.5),
|
||||
cache_write: Rate::Value(3.0),
|
||||
cache_write_1h: Rate::Value(5.0),
|
||||
..rates(Rate::Value(2.0), Rate::Value(4.0))
|
||||
};
|
||||
let base = request();
|
||||
let split = Request {
|
||||
usage: Usage {
|
||||
cache_write_5m_tokens: Some(4),
|
||||
cache_write_1h_tokens: Some(6),
|
||||
..base.usage
|
||||
},
|
||||
..base
|
||||
};
|
||||
let result = calculate(&pricing(standard), &split).unwrap();
|
||||
assert_eq!(result.cache_write_5m, 12.0);
|
||||
assert_eq!(result.cache_write_1h, 30.0);
|
||||
let overlapping = Request {
|
||||
usage: Usage {
|
||||
prompt_tokens: 30,
|
||||
..split.usage
|
||||
},
|
||||
..split
|
||||
};
|
||||
assert_eq!(
|
||||
calculate(&pricing(standard), &overlapping),
|
||||
Err(PricingError::CacheExceedsPrompt)
|
||||
);
|
||||
let incomplete = Request {
|
||||
usage: Usage {
|
||||
cache_write_1h_tokens: None,
|
||||
..split.usage
|
||||
},
|
||||
..split
|
||||
};
|
||||
assert_eq!(
|
||||
calculate(&pricing(standard), &incomplete),
|
||||
Err(PricingError::InvalidCacheWriteDetails)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn threshold_tiers_and_boundaries() {
|
||||
let priority = TierRates {
|
||||
tier: ServiceTier::Priority,
|
||||
rates: rates(Rate::Value(3.0), Rate::Missing),
|
||||
};
|
||||
let threshold = ThresholdRates {
|
||||
above_prompt_tokens: 100,
|
||||
standard: rates(Rate::Value(5.0), Rate::Value(8.0)),
|
||||
tiers: &[
|
||||
TierRates {
|
||||
tier: ServiceTier::Priority,
|
||||
rates: rates(Rate::Value(7.0), Rate::Missing),
|
||||
},
|
||||
TierRates {
|
||||
tier: ServiceTier::Flex,
|
||||
rates: rates(Rate::Value(6.0), Rate::Missing),
|
||||
},
|
||||
],
|
||||
};
|
||||
let specification = Pricing {
|
||||
standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
|
||||
tiers: &[priority],
|
||||
thresholds: &[threshold],
|
||||
off_peak: None,
|
||||
};
|
||||
let base = request();
|
||||
let no_cache = Request {
|
||||
usage: Usage {
|
||||
cache_read_tokens: 0,
|
||||
cache_write_tokens: 0,
|
||||
..base.usage
|
||||
},
|
||||
..base
|
||||
};
|
||||
let fast = Request {
|
||||
service_tier: ServiceTier::Fast,
|
||||
..no_cache
|
||||
};
|
||||
let inclusive = Request {
|
||||
threshold_policy: ThresholdPolicy::Inclusive,
|
||||
..fast
|
||||
};
|
||||
let flex = Request {
|
||||
service_tier: ServiceTier::Flex,
|
||||
..inclusive
|
||||
};
|
||||
assert_eq!(calculate(&specification, &no_cache).unwrap().input(), 200.0);
|
||||
assert_eq!(calculate(&specification, &fast).unwrap().input(), 300.0);
|
||||
assert_eq!(
|
||||
calculate(&specification, &inclusive).unwrap().input(),
|
||||
700.0
|
||||
);
|
||||
assert_eq!(calculate(&specification, &flex).unwrap().input(), 600.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compile_rejects_ambiguous_rates() {
|
||||
let duplicate = ThresholdRates {
|
||||
above_prompt_tokens: 100,
|
||||
standard: Rates::EMPTY,
|
||||
tiers: &[],
|
||||
};
|
||||
let specification = Pricing {
|
||||
standard: rates(Rate::Value(1.0), Rate::Value(1.0)),
|
||||
tiers: &[],
|
||||
thresholds: &[duplicate, duplicate],
|
||||
off_peak: None,
|
||||
};
|
||||
assert_eq!(
|
||||
compile(&specification).err(),
|
||||
Some(PricingError::DuplicateThreshold)
|
||||
);
|
||||
let invalid = pricing(rates(Rate::Value(f64::NAN), Rate::Value(1.0)));
|
||||
assert_eq!(compile(&invalid).err(), Some(PricingError::InvalidRate));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn off_peak_is_one_non_wrapping_utc_window() {
|
||||
let specification = Pricing {
|
||||
standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
|
||||
tiers: &[],
|
||||
thresholds: &[],
|
||||
off_peak: Some(OffPeakRates {
|
||||
start_utc_minute: 60,
|
||||
end_utc_minute: 120,
|
||||
rates: rates(Rate::Value(1.0), Rate::Value(2.0)),
|
||||
}),
|
||||
};
|
||||
let base = request();
|
||||
let start = Request {
|
||||
billed_at_utc_minute: Some(60),
|
||||
..base
|
||||
};
|
||||
let end = Request {
|
||||
billed_at_utc_minute: Some(120),
|
||||
..base
|
||||
};
|
||||
assert_eq!(
|
||||
calculate(&specification, &base),
|
||||
Err(PricingError::InvalidBillingTime)
|
||||
);
|
||||
assert_eq!(calculate(&specification, &start).unwrap().input(), 100.0);
|
||||
assert_eq!(calculate(&specification, &end).unwrap().input(), 200.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_rates_and_free_rates_remain_distinct() {
|
||||
let base = request();
|
||||
let empty = Request {
|
||||
usage: Usage {
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
cache_write_tokens: 0,
|
||||
..base.usage
|
||||
},
|
||||
..base
|
||||
};
|
||||
assert_eq!(
|
||||
calculate(&pricing(Rates::EMPTY), &empty),
|
||||
Err(PricingError::MissingInputRate)
|
||||
);
|
||||
assert_eq!(
|
||||
calculate(&pricing(rates(Rate::Value(0.0), Rate::Missing)), &empty),
|
||||
Err(PricingError::MissingOutputRate)
|
||||
);
|
||||
assert_eq!(
|
||||
calculate(&pricing(rates(Rate::Value(0.0), Rate::Value(0.0))), &empty)
|
||||
.unwrap()
|
||||
.total(),
|
||||
0.0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_executed_python_reference_cases() {
|
||||
for row in include_str!("python_reference.tsv")
|
||||
.lines()
|
||||
.filter(|line| !line.starts_with('#'))
|
||||
{
|
||||
let fields: Vec<_> = row.split('\t').collect();
|
||||
let count = |index: usize| fields[index].parse::<u64>().unwrap();
|
||||
let number = |index: usize| fields[index].parse::<f64>().unwrap();
|
||||
let optional_rate = |index: usize| {
|
||||
if fields[index].is_empty() {
|
||||
Rate::Missing
|
||||
} else {
|
||||
Rate::Value(number(index))
|
||||
}
|
||||
};
|
||||
let threshold = ThresholdRates {
|
||||
above_prompt_tokens: if fields[9].is_empty() { 0 } else { count(9) },
|
||||
standard: rates(optional_rate(10), optional_rate(11)),
|
||||
tiers: &[],
|
||||
};
|
||||
let thresholds = if fields[9].is_empty() {
|
||||
&[][..]
|
||||
} else {
|
||||
std::slice::from_ref(&threshold)
|
||||
};
|
||||
let specification = Pricing {
|
||||
standard: Rates {
|
||||
cache_read: optional_rate(7),
|
||||
cache_write: optional_rate(8),
|
||||
..rates(Rate::Value(number(5)), Rate::Value(number(6)))
|
||||
},
|
||||
tiers: &[],
|
||||
thresholds,
|
||||
off_peak: None,
|
||||
};
|
||||
let base = request();
|
||||
let input = Request {
|
||||
usage: Usage {
|
||||
prompt_tokens: count(1),
|
||||
completion_tokens: count(2),
|
||||
cache_read_tokens: count(3),
|
||||
cache_write_tokens: count(4),
|
||||
..base.usage
|
||||
},
|
||||
..base
|
||||
};
|
||||
let actual = calculate(&specification, &input).unwrap();
|
||||
assert_eq!(actual.input(), number(12), "{}", fields[0]);
|
||||
assert_eq!(actual.output(), number(13), "{}", fields[0]);
|
||||
}
|
||||
}
|
||||
|
||||
proptest::proptest! {
|
||||
#[test]
|
||||
fn equivalent_usage_conventions_and_breakdown_agree(
|
||||
regular in 0_u64..1000,
|
||||
read in 0_u64..1000,
|
||||
write in 0_u64..1000,
|
||||
output in 0_u64..1000,
|
||||
) {
|
||||
let threshold = ThresholdRates {
|
||||
above_prompt_tokens: 1000,
|
||||
standard: rates(Rate::Value(5.0), Rate::Value(8.0)),
|
||||
tiers: &[],
|
||||
};
|
||||
let specification = Pricing {
|
||||
standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
|
||||
tiers: &[],
|
||||
thresholds: &[threshold],
|
||||
off_peak: None,
|
||||
};
|
||||
let base = request();
|
||||
let included = Request {
|
||||
usage: Usage {
|
||||
prompt_tokens: regular + read + write,
|
||||
completion_tokens: output,
|
||||
cache_read_tokens: read,
|
||||
cache_write_tokens: write,
|
||||
..base.usage
|
||||
},
|
||||
..base
|
||||
};
|
||||
let excluded = Request {
|
||||
usage: Usage {
|
||||
prompt_tokens: regular,
|
||||
prompt_convention: PromptConvention::ExcludesCache,
|
||||
..included.usage
|
||||
},
|
||||
..included
|
||||
};
|
||||
let plan = compile(&specification).unwrap();
|
||||
let left = plan.calculate(&included).unwrap();
|
||||
let right = plan.calculate(&excluded).unwrap();
|
||||
proptest::prop_assert_eq!(left, right);
|
||||
proptest::prop_assert_eq!(left.total(), left.input() + left.output());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn regional_multiplier_applies_after_input_components_are_summed() {
|
||||
let standard = Rates {
|
||||
cache_read: Rate::Value(0.5),
|
||||
cache_write: Rate::Value(3.0),
|
||||
..rates(Rate::Value(2.0), Rate::Value(4.0))
|
||||
};
|
||||
let base = request();
|
||||
let regional = Request {
|
||||
region_multiplier: Some(1.1),
|
||||
..base
|
||||
};
|
||||
let result = calculate(&pricing(standard), ®ional).unwrap();
|
||||
assert_eq!(result.input(), (65.0 * 2.0 + 25.0 * 0.5 + 10.0 * 3.0) * 1.1);
|
||||
assert_eq!(result.output(), 20.0 * 4.0 * 1.1);
|
||||
let invalid = Request {
|
||||
region_multiplier: Some(f64::NAN),
|
||||
..base
|
||||
};
|
||||
assert_eq!(
|
||||
calculate(&pricing(standard), &invalid),
|
||||
Err(PricingError::InvalidRegionMultiplier)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compilation_sorts_thresholds_and_rejects_duplicate_tiers() {
|
||||
let high = ThresholdRates {
|
||||
above_prompt_tokens: 200,
|
||||
standard: rates(Rate::Value(7.0), Rate::Missing),
|
||||
tiers: &[],
|
||||
};
|
||||
let low = ThresholdRates {
|
||||
above_prompt_tokens: 100,
|
||||
standard: rates(Rate::Value(5.0), Rate::Missing),
|
||||
tiers: &[],
|
||||
};
|
||||
let specification = Pricing {
|
||||
standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
|
||||
tiers: &[],
|
||||
thresholds: &[high, low],
|
||||
off_peak: None,
|
||||
};
|
||||
let base = request();
|
||||
let above_both = Request {
|
||||
usage: Usage {
|
||||
prompt_tokens: 201,
|
||||
cache_read_tokens: 0,
|
||||
cache_write_tokens: 0,
|
||||
..base.usage
|
||||
},
|
||||
..base
|
||||
};
|
||||
assert_eq!(
|
||||
compile(&specification)
|
||||
.unwrap()
|
||||
.calculate(&above_both)
|
||||
.unwrap()
|
||||
.rates
|
||||
.input,
|
||||
7.0
|
||||
);
|
||||
let duplicate = TierRates {
|
||||
tier: ServiceTier::Flex,
|
||||
rates: Rates::EMPTY,
|
||||
};
|
||||
let invalid = Pricing {
|
||||
tiers: &[duplicate, duplicate],
|
||||
thresholds: &[],
|
||||
..specification
|
||||
};
|
||||
assert_eq!(compile(&invalid).err(), Some(PricingError::DuplicateTier));
|
||||
}
|
||||
96
litellm-rust/crates/cost/tests/generate_python_reference.py
Normal file
96
litellm-rust/crates/cost/tests/generate_python_reference.py
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Case:
|
||||
name: str
|
||||
prompt: int
|
||||
completion: int
|
||||
cache_read: int
|
||||
cache_write: int
|
||||
input_rate: float
|
||||
output_rate: float
|
||||
cache_read_rate: float | None = None
|
||||
cache_write_rate: float | None = None
|
||||
threshold: int | None = None
|
||||
threshold_input_rate: float | None = None
|
||||
threshold_output_rate: float | None = None
|
||||
|
||||
|
||||
CASES = (
|
||||
Case("ordinary", 100, 20, 0, 0, 2.0, 4.0),
|
||||
Case("cache_fallback", 100, 20, 25, 10, 2.0, 4.0),
|
||||
Case("cache_specific", 100, 20, 25, 10, 2.0, 4.0, 0.5, 3.0),
|
||||
Case("free_cache", 100, 20, 25, 10, 2.0, 4.0, 0.0, 0.0),
|
||||
Case("threshold_below", 99, 20, 0, 0, 2.0, 4.0, threshold=100, threshold_input_rate=5.0, threshold_output_rate=8.0),
|
||||
Case("threshold_at", 100, 20, 0, 0, 2.0, 4.0, threshold=100, threshold_input_rate=5.0, threshold_output_rate=8.0),
|
||||
Case(
|
||||
"threshold_above", 101, 20, 0, 0, 2.0, 4.0, threshold=100, threshold_input_rate=5.0, threshold_output_rate=8.0
|
||||
),
|
||||
Case(
|
||||
"cache_threshold_above",
|
||||
101,
|
||||
20,
|
||||
25,
|
||||
10,
|
||||
2.0,
|
||||
4.0,
|
||||
threshold=100,
|
||||
threshold_input_rate=5.0,
|
||||
threshold_output_rate=8.0,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def reference(case: Case) -> tuple[float, float]:
|
||||
info = {"input_cost_per_token": case.input_rate, "output_cost_per_token": case.output_rate}
|
||||
if case.cache_read_rate is not None:
|
||||
info["cache_read_input_token_cost"] = case.cache_read_rate
|
||||
if case.cache_write_rate is not None:
|
||||
info["cache_creation_input_token_cost"] = case.cache_write_rate
|
||||
if case.threshold is not None:
|
||||
info[f"input_cost_per_token_above_{case.threshold}_tokens"] = case.threshold_input_rate
|
||||
info[f"output_cost_per_token_above_{case.threshold}_tokens"] = case.threshold_output_rate
|
||||
details = {"cached_tokens": case.cache_read, "cache_write_tokens": case.cache_write}
|
||||
usage = Usage(prompt_tokens=case.prompt, completion_tokens=case.completion, prompt_tokens_details=details)
|
||||
return generic_cost_per_token(
|
||||
model="synthetic",
|
||||
usage=usage,
|
||||
custom_llm_provider="openai",
|
||||
model_info=info,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
revision = subprocess.check_output(("git", "rev-parse", "HEAD"), text=True).strip()
|
||||
rows = ("# Python reference commit: " + revision,) + tuple(
|
||||
"\t".join(
|
||||
str(value) if value is not None else ""
|
||||
for value in (
|
||||
case.name,
|
||||
case.prompt,
|
||||
case.completion,
|
||||
case.cache_read,
|
||||
case.cache_write,
|
||||
case.input_rate,
|
||||
case.output_rate,
|
||||
case.cache_read_rate,
|
||||
case.cache_write_rate,
|
||||
case.threshold,
|
||||
case.threshold_input_rate,
|
||||
case.threshold_output_rate,
|
||||
*reference(case),
|
||||
)
|
||||
)
|
||||
for case in CASES
|
||||
)
|
||||
Path(__file__).with_name("python_reference.tsv").write_text("\n".join(rows) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
9
litellm-rust/crates/cost/tests/python_reference.tsv
Normal file
9
litellm-rust/crates/cost/tests/python_reference.tsv
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
# Python reference commit: dc4be2fd987c993aefcf16444e34f960c12c8627
|
||||
ordinary 100 20 0 0 2.0 4.0 200.0 80.0
|
||||
cache_fallback 100 20 25 10 2.0 4.0 200.0 80.0
|
||||
cache_specific 100 20 25 10 2.0 4.0 0.5 3.0 172.5 80.0
|
||||
free_cache 100 20 25 10 2.0 4.0 0.0 0.0 130.0 80.0
|
||||
threshold_below 99 20 0 0 2.0 4.0 100 5.0 8.0 198.0 80.0
|
||||
threshold_at 100 20 0 0 2.0 4.0 100 5.0 8.0 200.0 80.0
|
||||
threshold_above 101 20 0 0 2.0 4.0 100 5.0 8.0 505.0 160.0
|
||||
cache_threshold_above 101 20 25 10 2.0 4.0 100 5.0 8.0 505.0 160.0
|
||||
|
Can't render this file because it has a wrong number of fields in line 2.
|
25
litellm-rust/crates/model-catalog/Cargo.toml
Normal file
25
litellm-rust/crates/model-catalog/Cargo.toml
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
[package]
|
||||
name = "litellm-model-catalog"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[features]
|
||||
schema = ["dep:schemars"]
|
||||
|
||||
[dependencies]
|
||||
indexmap = { version = "2.14.0", features = ["serde"] }
|
||||
schemars = { version = "1.0", optional = true }
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
criterion.workspace = true
|
||||
rstest.workspace = true
|
||||
litellm-model-catalog = { path = ".", features = ["schema"] }
|
||||
|
||||
[[bench]]
|
||||
name = "catalog"
|
||||
harness = false
|
||||
25
litellm-rust/crates/model-catalog/README.md
Normal file
25
litellm-rust/crates/model-catalog/README.md
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
# Model catalog
|
||||
|
||||
`litellm-model-catalog` builds an immutable snapshot from caller supplied JSON bytes. It has no network, Python, registration, or refresh behavior. The caller supplies optional source, revision, and ETag provenance. Parse and validation are separate so small synthetic catalogs can use explicit integrity limits
|
||||
|
||||
The parser treats `sample_spec` and `fallback_generalizations` as reserved top level metadata. `fallback_rules()` exposes the typed rule array when present; this crate does not execute regex generalizations. Model entries retain all JSON fields except `aliases`, including unknown fields. `field()` returns `None` for an absent key and a JSON null, false, or zero value for a present key. The returned values are borrowed, so callers cannot mutate the snapshot
|
||||
|
||||
Each entry also deserializes into `ModelInfo`, a typed mirror of `model_prices_and_context_window.schema.json`'s `modelEntry` definition, reachable via `ModelEntry::info()`. All schema fields are optional on `ModelInfo`, including `litellm_provider` which the schema marks required, so small synthetic catalogs still parse. Unknown fields are not part of `ModelInfo`; they remain on `fields()`. Building with the `schema` feature adds `schemars` derives and exposes `model_entry_json_schema()` for emitting the entry's JSON Schema. Parse and validation failures are reported by the `Error` enum in `error.rs`, while catalog logic lives in `catalog.rs`
|
||||
|
||||
The integration tests read the repository's catalog and schema files at test time, assert every entry round-trips through `ModelInfo`, and verify that the generated schema's properties match the repository schema
|
||||
|
||||
Aliases point to their canonical entries. An alias that exactly matches any canonical key is skipped; the first canonical entry claiming an alias wins. Invalid alias lists and nonstring names are skipped and reported by `alias_issues()`. Exact lookup wins. For a case insensitive miss, the last key with the same lowercase spelling wins, following Python's lowercase map built after aliases are appended. This uses Rust Unicode lowercasing, which can differ from Python for unusual Unicode model IDs
|
||||
|
||||
`validate()` counts canonical entries before alias expansion and excludes both reserved keys. It enforces an explicit minimum and backup shrink ratio, with Python defaults of 50 models and 0.5. Parsing rejects nonobject model entries and known fields with the wrong JSON type, but ignores unknown fields. It does not enforce every constraint in the JSON schema, calculate prices, resolve providers, or check provenance authenticity. The caller decides how to handle validation failures
|
||||
|
||||
This snapshot does not represent Python's live mutable `litellm.model_cost`, nested dict and list mutation, or mutation of dicts previously returned by Python APIs. It has no bridge or runtime integration
|
||||
|
||||
## Benchmarks
|
||||
|
||||
`cargo bench -p litellm-model-catalog --bench catalog` measures parsing plus alias indexing and exact lookup. For a local Python baseline on the same fixture, use:
|
||||
|
||||
```sh
|
||||
python3 -m timeit -s 'import json, pathlib; body = pathlib.Path("../model_prices_and_context_window.json").read_bytes()' 'json.loads(body)'
|
||||
```
|
||||
|
||||
Run these commands from `litellm-rust`. Python's command measures JSON loading only, without alias expansion or snapshot construction. The Rust benchmark does not include future Python object materialization, so these numbers are not an end to end runtime comparison
|
||||
21
litellm-rust/crates/model-catalog/benches/catalog.rs
Normal file
21
litellm-rust/crates/model-catalog/benches/catalog.rs
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
use criterion::{Criterion, criterion_group, criterion_main};
|
||||
use litellm_model_catalog::{Catalog, Provenance};
|
||||
use std::hint::black_box;
|
||||
|
||||
fn benchmarks(c: &mut Criterion) {
|
||||
let body = include_bytes!("../../../../model_prices_and_context_window.json");
|
||||
c.bench_function("parse_current_catalog", |b| {
|
||||
b.iter(|| Catalog::parse(black_box(body), Provenance::default()).unwrap())
|
||||
});
|
||||
let catalog = Catalog::parse(body, Provenance::default()).unwrap();
|
||||
let key = catalog
|
||||
.model_names()
|
||||
.next()
|
||||
.expect("catalog must have a benchmark key");
|
||||
c.bench_function("lookup_catalog_key", |b| {
|
||||
b.iter(|| black_box(&catalog).lookup(black_box(key)))
|
||||
});
|
||||
}
|
||||
|
||||
criterion_group!(benches, benchmarks);
|
||||
criterion_main!(benches);
|
||||
241
litellm-rust/crates/model-catalog/src/catalog.rs
Normal file
241
litellm-rust/crates/model-catalog/src/catalog.rs
Normal file
|
|
@ -0,0 +1,241 @@
|
|||
use crate::error::Error;
|
||||
use crate::model_info::{FallbackGeneralizations, FallbackRule, ModelInfo};
|
||||
use indexmap::IndexMap;
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Map, Value};
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct Provenance {
|
||||
pub source: Option<String>,
|
||||
pub revision: Option<String>,
|
||||
pub etag: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct IntegrityLimits {
|
||||
pub backup_model_count: usize,
|
||||
pub min_model_count: usize,
|
||||
pub min_backup_ratio: f64,
|
||||
}
|
||||
|
||||
impl IntegrityLimits {
|
||||
pub fn python_defaults(backup_model_count: usize) -> Self {
|
||||
Self {
|
||||
backup_model_count,
|
||||
min_model_count: 50,
|
||||
min_backup_ratio: 0.5,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum AliasIssue {
|
||||
InvalidList { model: String },
|
||||
InvalidName { model: String },
|
||||
CanonicalCollision { model: String, alias: String },
|
||||
AliasCollision { model: String, alias: String },
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ModelEntry {
|
||||
fields: Map<String, Value>,
|
||||
info: ModelInfo,
|
||||
}
|
||||
|
||||
impl ModelEntry {
|
||||
pub fn field(&self, name: &str) -> Option<&Value> {
|
||||
self.fields.get(name)
|
||||
}
|
||||
pub fn fields(&self) -> &Map<String, Value> {
|
||||
&self.fields
|
||||
}
|
||||
/// The entry deserialized into the typed mirror of the catalog schema.
|
||||
pub fn info(&self) -> &ModelInfo {
|
||||
&self.info
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct ModelMatch<'a> {
|
||||
pub matched_key: &'a str,
|
||||
pub canonical_key: &'a str,
|
||||
pub entry: &'a ModelEntry,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Catalog {
|
||||
entries: IndexMap<String, ModelEntry>,
|
||||
aliases: IndexMap<String, String>,
|
||||
lowercase_keys: HashMap<String, String>,
|
||||
sample_spec: Option<Value>,
|
||||
fallback_generalizations: Option<FallbackGeneralizations>,
|
||||
provenance: Provenance,
|
||||
alias_issues: Vec<AliasIssue>,
|
||||
}
|
||||
|
||||
impl Catalog {
|
||||
pub fn parse(body: &[u8], provenance: Provenance) -> Result<Self, Error> {
|
||||
let root: IndexMap<String, Value> = serde_json::from_slice(body)?;
|
||||
if root.is_empty() {
|
||||
return Err(Error::Empty);
|
||||
}
|
||||
|
||||
let mut entries = IndexMap::with_capacity(root.len());
|
||||
let mut alias_lists = Vec::new();
|
||||
let mut alias_issues = Vec::new();
|
||||
let mut sample_spec = None;
|
||||
let mut fallback_generalizations = None;
|
||||
for (name, value) in root {
|
||||
match name.as_str() {
|
||||
"sample_spec" => {
|
||||
sample_spec = Some(value);
|
||||
continue;
|
||||
}
|
||||
"fallback_generalizations" => {
|
||||
fallback_generalizations =
|
||||
Some(serde_json::from_value::<FallbackGeneralizations>(value)?);
|
||||
continue;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
let Value::Object(ref object) = value else {
|
||||
return Err(Error::EntryNotObject { model: name });
|
||||
};
|
||||
let info = ModelInfo::deserialize(object)?;
|
||||
let Value::Object(mut fields) = value else {
|
||||
unreachable!("value checked is_object above")
|
||||
};
|
||||
if let Some(aliases) = fields.remove("aliases")
|
||||
&& !aliases.is_null()
|
||||
{
|
||||
match aliases {
|
||||
Value::Array(names) => alias_lists.push((name.clone(), names)),
|
||||
_ => alias_issues.push(AliasIssue::InvalidList {
|
||||
model: name.clone(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
entries.insert(name, ModelEntry { fields, info });
|
||||
}
|
||||
|
||||
let mut aliases = IndexMap::new();
|
||||
for (model, names) in alias_lists {
|
||||
for name in names {
|
||||
let Value::String(alias) = name else {
|
||||
alias_issues.push(AliasIssue::InvalidName {
|
||||
model: model.clone(),
|
||||
});
|
||||
continue;
|
||||
};
|
||||
if entries.contains_key(&alias) {
|
||||
alias_issues.push(AliasIssue::CanonicalCollision {
|
||||
model: model.clone(),
|
||||
alias,
|
||||
});
|
||||
} else if aliases.contains_key(&alias) {
|
||||
alias_issues.push(AliasIssue::AliasCollision {
|
||||
model: model.clone(),
|
||||
alias,
|
||||
});
|
||||
} else {
|
||||
aliases.insert(alias, model.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let lowercase_keys = entries
|
||||
.keys()
|
||||
.chain(aliases.keys())
|
||||
.map(|key| (key.to_lowercase(), key.clone()))
|
||||
.collect();
|
||||
Ok(Self {
|
||||
entries,
|
||||
aliases,
|
||||
lowercase_keys,
|
||||
sample_spec,
|
||||
fallback_generalizations,
|
||||
provenance,
|
||||
alias_issues,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn validate(&self, limits: IntegrityLimits) -> Result<(), Error> {
|
||||
if !limits.min_backup_ratio.is_finite() || !(0.0..=1.0).contains(&limits.min_backup_ratio) {
|
||||
return Err(Error::InvalidRatio);
|
||||
}
|
||||
let actual = self.entries.len();
|
||||
if actual < limits.min_model_count {
|
||||
return Err(Error::BelowMinimum {
|
||||
actual,
|
||||
minimum: limits.min_model_count,
|
||||
});
|
||||
}
|
||||
if limits.backup_model_count > 0
|
||||
&& (actual as f64) < (limits.backup_model_count as f64) * limits.min_backup_ratio
|
||||
{
|
||||
return Err(Error::Shrunk {
|
||||
actual,
|
||||
backup: limits.backup_model_count,
|
||||
ratio: limits.min_backup_ratio,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn lookup(&self, key: &str) -> Option<ModelMatch<'_>> {
|
||||
let matched_key = if self.entries.contains_key(key) || self.aliases.contains_key(key) {
|
||||
key
|
||||
} else {
|
||||
self.lowercase_keys.get(&key.to_lowercase())?.as_str()
|
||||
};
|
||||
let canonical_key = self
|
||||
.aliases
|
||||
.get(matched_key)
|
||||
.map(String::as_str)
|
||||
.unwrap_or(matched_key);
|
||||
let (canonical_key, entry) = self.entries.get_key_value(canonical_key)?;
|
||||
let matched_key = self
|
||||
.entries
|
||||
.get_key_value(matched_key)
|
||||
.map(|(key, _)| key.as_str())
|
||||
.or_else(|| {
|
||||
self.aliases
|
||||
.get_key_value(matched_key)
|
||||
.map(|(key, _)| key.as_str())
|
||||
})?;
|
||||
Some(ModelMatch {
|
||||
matched_key,
|
||||
canonical_key,
|
||||
entry,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn model_count(&self) -> usize {
|
||||
self.entries.len()
|
||||
}
|
||||
pub fn model_names(&self) -> impl Iterator<Item = &str> {
|
||||
self.entries.keys().map(String::as_str)
|
||||
}
|
||||
pub fn alias_count(&self) -> usize {
|
||||
self.aliases.len()
|
||||
}
|
||||
pub fn aliases(&self) -> &IndexMap<String, String> {
|
||||
&self.aliases
|
||||
}
|
||||
pub fn alias_issues(&self) -> &[AliasIssue] {
|
||||
&self.alias_issues
|
||||
}
|
||||
pub fn sample_spec(&self) -> Option<&Value> {
|
||||
self.sample_spec.as_ref()
|
||||
}
|
||||
pub fn fallback_generalizations(&self) -> Option<&FallbackGeneralizations> {
|
||||
self.fallback_generalizations.as_ref()
|
||||
}
|
||||
pub fn fallback_rules(&self) -> Option<&[FallbackRule]> {
|
||||
Some(self.fallback_generalizations.as_ref()?.rules.as_slice())
|
||||
}
|
||||
pub fn provenance(&self) -> &Provenance {
|
||||
&self.provenance
|
||||
}
|
||||
}
|
||||
28
litellm-rust/crates/model-catalog/src/error.rs
Normal file
28
litellm-rust/crates/model-catalog/src/error.rs
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
use thiserror::Error;
|
||||
|
||||
/// Failures from parsing or validating a catalog snapshot.
|
||||
#[derive(Debug, Error)]
|
||||
pub enum Error {
|
||||
/// The body is not valid JSON, or a model entry fails typed deserialization.
|
||||
#[error("invalid JSON: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
/// The catalog has no entries at all.
|
||||
#[error("catalog is empty")]
|
||||
Empty,
|
||||
/// A non-reserved top level value is not a JSON object.
|
||||
#[error("model {model:?} must be an object")]
|
||||
EntryNotObject { model: String },
|
||||
/// Canonical entry count is under the configured minimum.
|
||||
#[error("catalog has {actual} models, below minimum {minimum}")]
|
||||
BelowMinimum { actual: usize, minimum: usize },
|
||||
/// Canonical entry count is under the configured backup shrink ratio.
|
||||
#[error("catalog has {actual} models, below {ratio} of backup count {backup}")]
|
||||
Shrunk {
|
||||
actual: usize,
|
||||
backup: usize,
|
||||
ratio: f64,
|
||||
},
|
||||
/// The configured minimum backup ratio is not finite or outside `[0, 1]`.
|
||||
#[error("minimum backup ratio must be finite and between zero and one")]
|
||||
InvalidRatio,
|
||||
}
|
||||
16
litellm-rust/crates/model-catalog/src/lib.rs
Normal file
16
litellm-rust/crates/model-catalog/src/lib.rs
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
mod catalog;
|
||||
mod error;
|
||||
mod model_info;
|
||||
#[cfg(feature = "schema")]
|
||||
mod schema;
|
||||
|
||||
pub use catalog::{AliasIssue, Catalog, IntegrityLimits, ModelEntry, ModelMatch, Provenance};
|
||||
pub use error::Error;
|
||||
pub use model_info::{
|
||||
AudioFormat, FallbackGeneralizations, FallbackRule, InputModality, Mode, ModelInfo,
|
||||
OffPeakPricing, OffPeakWindow, OutputModality, ReasoningEffort, SearchContextCostPerQuery,
|
||||
TieredRate, UtcHours, VertexAiAudioApi, WebSearchBillingUnit, Weekday,
|
||||
};
|
||||
|
||||
#[cfg(feature = "schema")]
|
||||
pub use schema::model_entry_json_schema;
|
||||
665
litellm-rust/crates/model-catalog/src/model_info.rs
Normal file
665
litellm-rust/crates/model-catalog/src/model_info.rs
Normal file
|
|
@ -0,0 +1,665 @@
|
|||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// Primary API surface / task type of the model.
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Mode {
|
||||
AudioSpeech,
|
||||
AudioTranscription,
|
||||
Chat,
|
||||
Completion,
|
||||
Embedding,
|
||||
Evaluation,
|
||||
Guardrail,
|
||||
ImageEdit,
|
||||
ImageGeneration,
|
||||
Moderation,
|
||||
Ocr,
|
||||
Realtime,
|
||||
Rerank,
|
||||
Responses,
|
||||
Search,
|
||||
VectorStore,
|
||||
VideoGeneration,
|
||||
}
|
||||
|
||||
/// Reasoning effort level accepted or applied by the model.
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ReasoningEffort {
|
||||
None,
|
||||
Minimal,
|
||||
Low,
|
||||
Medium,
|
||||
High,
|
||||
Xhigh,
|
||||
Max,
|
||||
}
|
||||
|
||||
/// Gemini audio generation API the model is served through.
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum VertexAiAudioApi {
|
||||
LyriaPredict,
|
||||
LyriaInteractions,
|
||||
}
|
||||
|
||||
/// Whether web search is billed per query or per prompt.
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum WebSearchBillingUnit {
|
||||
PerQuery,
|
||||
PerPrompt,
|
||||
}
|
||||
|
||||
/// Audio container format the model can return.
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum AudioFormat {
|
||||
Mp3,
|
||||
Wav,
|
||||
}
|
||||
|
||||
/// Input modality the model accepts.
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum InputModality {
|
||||
Text,
|
||||
Image,
|
||||
Audio,
|
||||
Video,
|
||||
}
|
||||
|
||||
/// Output modality the model can produce.
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum OutputModality {
|
||||
Text,
|
||||
Image,
|
||||
Audio,
|
||||
Video,
|
||||
Code,
|
||||
}
|
||||
|
||||
/// UTC "HH:MM-HH:MM" window, or a list of them; a window may wrap past midnight.
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
|
||||
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
|
||||
#[serde(untagged)]
|
||||
pub enum UtcHours {
|
||||
Single(String),
|
||||
Multiple(Vec<String>),
|
||||
}
|
||||
|
||||
/// ISO-8601 weekday number (1 = Monday .. 7 = Sunday) or English day name.
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
|
||||
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
|
||||
#[serde(untagged)]
|
||||
pub enum Weekday {
|
||||
Number(u8),
|
||||
Name(String),
|
||||
}
|
||||
|
||||
/// One off-peak window entry inside `windows`.
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
|
||||
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct OffPeakWindow {
|
||||
pub hours_utc: UtcHours,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub weekdays: Option<Vec<Weekday>>,
|
||||
}
|
||||
|
||||
/// Rates that replace the same-named base fields inside the stated UTC windows.
|
||||
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
|
||||
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct OffPeakPricing {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub hours_utc: Option<UtcHours>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub windows: Option<Vec<OffPeakWindow>>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub weekday_timezone: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_token: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_token: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_reasoning_token: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read_input_token_cost: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_creation_input_token_cost: Option<f64>,
|
||||
}
|
||||
|
||||
/// USD cost per web search query, keyed by search context size.
|
||||
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
|
||||
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SearchContextCostPerQuery {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub search_context_size_low: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub search_context_size_medium: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub search_context_size_high: Option<f64>,
|
||||
}
|
||||
|
||||
/// One tier of a context-length or result-count tiered rate.
|
||||
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
|
||||
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct TieredRate {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub range: Option<[f64; 2]>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_results_range: Option<[f64; 2]>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_token: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_token: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_reasoning_token: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read_input_token_cost: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_creation_input_token_cost: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_query: Option<f64>,
|
||||
}
|
||||
|
||||
/// One regex rule generalizing unknown model ids to known families.
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
|
||||
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
|
||||
pub struct FallbackRule {
|
||||
pub name: String,
|
||||
pub pattern: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub description: Option<String>,
|
||||
#[serde(flatten)]
|
||||
pub extra: BTreeMap<String, Value>,
|
||||
}
|
||||
|
||||
/// Regex rules that generalize unknown model ids to known families; not a model entry.
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
|
||||
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct FallbackGeneralizations {
|
||||
pub rules: Vec<FallbackRule>,
|
||||
}
|
||||
|
||||
/// Typed mirror of one catalog model entry.
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
|
||||
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
|
||||
pub struct ModelInfo {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub annotation_cost_per_page: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub annotation_cost_per_page_batches: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub audio_transcription_config: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub bedrock_converse_supports_strict_tools: Option<bool>,
|
||||
/// Highest reasoning effort the Bedrock output_config accepts for this model.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub bedrock_output_config_effort_ceiling: Option<ReasoningEffort>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_creation_input_audio_token_cost: Option<f64>,
|
||||
/// USD per token written to the provider's prompt cache.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_creation_input_token_cost: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_creation_input_token_cost_above_128k_tokens: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_creation_input_token_cost_above_1hr: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_creation_input_token_cost_above_1hr_above_200k_tokens: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_creation_input_token_cost_above_200k_tokens: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_creation_input_token_cost_above_256k_tokens: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_creation_input_token_cost_above_272k_tokens: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_creation_input_token_cost_above_272k_tokens_batches: Option<f64>,
|
||||
/// Flex service-tier rate for the same-named base field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_creation_input_token_cost_above_272k_tokens_flex: Option<f64>,
|
||||
/// Priority service-tier rate for the same-named base field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_creation_input_token_cost_above_272k_tokens_priority: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_creation_input_token_cost_batches: Option<f64>,
|
||||
/// Flex service-tier rate for the same-named base field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_creation_input_token_cost_flex: Option<f64>,
|
||||
/// Priority service-tier rate for the same-named base field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_creation_input_token_cost_priority: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read_input_audio_token_cost: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read_input_image_token_cost: Option<f64>,
|
||||
/// USD per prompt token served from the provider's prompt cache.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read_input_token_cost: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read_input_token_cost_above_128k_tokens: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read_input_token_cost_above_200k_tokens: Option<f64>,
|
||||
/// Priority service-tier rate for the same-named base field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read_input_token_cost_above_200k_tokens_priority: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read_input_token_cost_above_256k_tokens: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read_input_token_cost_above_272k_tokens: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read_input_token_cost_above_272k_tokens_batches: Option<f64>,
|
||||
/// Flex service-tier rate for the same-named base field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read_input_token_cost_above_272k_tokens_flex: Option<f64>,
|
||||
/// Priority service-tier rate for the same-named base field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read_input_token_cost_above_272k_tokens_priority: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read_input_token_cost_above_512k_tokens: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read_input_token_cost_batches: Option<f64>,
|
||||
/// Flex service-tier rate for the same-named base field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read_input_token_cost_flex: Option<f64>,
|
||||
/// Priority service-tier rate for the same-named base field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read_input_token_cost_priority: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub citation_cost_per_token: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub code_interpreter_cost_per_session: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub comment: Option<String>,
|
||||
/// Reasoning effort the provider applies when the request omits reasoning_effort. Gates whether a non-default temperature or the top_p/logprobs sampling params are accepted, which hold only when the effort resolves to 'none'.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub default_reasoning_effort: Option<ReasoningEffort>,
|
||||
/// Date the provider deprecates the model, YYYY-MM-DD.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub deprecation_date: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub gemini_audio_only_live: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub gemini_native_audio: Option<bool>,
|
||||
/// USD per Grounding with Google Maps request; billed per query or per prompt per web_search_billing_unit.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub google_maps_grounding_cost_per_query: Option<f64>,
|
||||
/// USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub guardrail_cost_per_unit: Option<BTreeMap<String, f64>>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_audio_per_second: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_audio_per_second_above_128k_tokens: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_audio_token: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_audio_token_batches: Option<f64>,
|
||||
/// Priority service-tier rate for the same-named base field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_audio_token_priority: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_character: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_character_above_128k_tokens: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_image: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_image_above_128k_tokens: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_image_token: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_image_token_batches: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_pixel: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_query: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_request: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_second: Option<f64>,
|
||||
/// USD per prompt token.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_token: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_token_above_128k_tokens: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_token_above_200k_tokens: Option<f64>,
|
||||
/// Priority service-tier rate for the same-named base field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_token_above_200k_tokens_priority: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_token_above_256k_tokens: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_token_above_272k_tokens: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_token_above_272k_tokens_batches: Option<f64>,
|
||||
/// Flex service-tier rate for the same-named base field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_token_above_272k_tokens_flex: Option<f64>,
|
||||
/// Priority service-tier rate for the same-named base field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_token_above_272k_tokens_priority: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_token_above_512k_tokens: Option<f64>,
|
||||
/// USD per prompt token via the provider's batch API.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_token_batches: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_token_cache_hit: Option<f64>,
|
||||
/// Flex service-tier rate for the same-named base field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_token_flex: Option<f64>,
|
||||
/// Priority service-tier rate for the same-named base field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_token_priority: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_video_per_second: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_video_per_second_above_128k_tokens: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_video_per_second_above_15s_interval: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_video_per_second_above_8s_interval: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_video_token: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_video_token_batches: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub input_dbu_cost_per_token: Option<f64>,
|
||||
/// LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub litellm_provider: Option<String>,
|
||||
/// Maximum prompt/context tokens the model accepts.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_input_tokens: Option<u64>,
|
||||
/// Maximum tokens the model can generate in one response.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_output_tokens: Option<u64>,
|
||||
/// Legacy field: max output tokens if the provider specifies it, else max input tokens.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_tokens: Option<u64>,
|
||||
/// Free-form notes about the entry (e.g. pricing derivation).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub metadata: Option<BTreeMap<String, Value>>,
|
||||
/// Primary API surface / task type of the model.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub mode: Option<Mode>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub ocr_cost_per_credit: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub ocr_cost_per_page: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub ocr_cost_per_page_batches: Option<f64>,
|
||||
/// Rates that replace the same-named base fields while the request falls inside the stated UTC windows.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub off_peak_pricing: Option<OffPeakPricing>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_audio_token: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_character: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_character_above_128k_tokens: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_image: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_image_1024: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_image_1536: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_image_512: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_image_token: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_pixel: Option<f64>,
|
||||
/// USD per reasoning/thinking token, when billed separately.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_reasoning_token: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_second: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_second_1080p: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_second_2k: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_second_480p: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_second_4k: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_second_720p: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_second_768p: Option<f64>,
|
||||
/// USD per generated token.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_token: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_token_above_128k_tokens: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_token_above_200k_tokens: Option<f64>,
|
||||
/// Priority service-tier rate for the same-named base field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_token_above_200k_tokens_priority: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_token_above_256k_tokens: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_token_above_272k_tokens: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_token_above_272k_tokens_batches: Option<f64>,
|
||||
/// Flex service-tier rate for the same-named base field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_token_above_272k_tokens_flex: Option<f64>,
|
||||
/// Priority service-tier rate for the same-named base field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_token_above_272k_tokens_priority: Option<f64>,
|
||||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_token_above_512k_tokens: Option<f64>,
|
||||
/// USD per generated token via the provider's batch API.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_token_batches: Option<f64>,
|
||||
/// Flex service-tier rate for the same-named base field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_token_flex: Option<f64>,
|
||||
/// Priority service-tier rate for the same-named base field.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_token_priority: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_video_per_second: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_video_token: Option<f64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_dbu_cost_per_token: Option<f64>,
|
||||
/// Embedding dimension for embedding models.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub output_vector_size: Option<u64>,
|
||||
/// Smallest prefix the provider will actually cache; absent means the provider default applies.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub prompt_cache_min_tokens: Option<u64>,
|
||||
/// Provider-internal routing hints (e.g. bedrock_invocation_schema).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub provider_specific_entry: Option<BTreeMap<String, Value>>,
|
||||
/// Exact reasoning_effort levels this deployment accepts; wins over supports_* flags.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub reasoning_effort_levels: Option<Vec<ReasoningEffort>>,
|
||||
/// Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub regional_endpoint_uplift_multiplier: Option<f64>,
|
||||
/// Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub regional_processing_uplift_multiplier_eu: Option<f64>,
|
||||
/// Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub regional_processing_uplift_multiplier_us: Option<f64>,
|
||||
/// Provider default requests-per-minute limit.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub rpm: Option<u64>,
|
||||
/// USD cost per web search query, keyed by search context size.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub search_context_cost_per_query: Option<SearchContextCostPerQuery>,
|
||||
/// URL of the provider pricing/model page this entry was taken from.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub source: Option<String>,
|
||||
/// Audio container formats the model can return.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supported_audio_formats: Option<Vec<AudioFormat>>,
|
||||
/// OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supported_endpoints: Option<Vec<String>>,
|
||||
/// Input modalities the model accepts.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supported_modalities: Option<Vec<InputModality>>,
|
||||
/// Output modalities the model can produce.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supported_output_modalities: Option<Vec<OutputModality>>,
|
||||
/// Cloud regions the model is available in ('global' or region ids).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supported_regions: Option<Vec<String>>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_adaptive_thinking: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_anthropic_compaction: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_anthropic_thinking_payload: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_assistant_prefill: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_audio_input: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_audio_output: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_computer_use: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_embedding_image_input: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_fast_mode: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_forced_tool_use: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_function_calling: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_image_input: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_image_size: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_legacy_thinking: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_low_reasoning_effort: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_max_reasoning_effort: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_mid_conversation_system: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_minimal_reasoning_effort: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_multimodal: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_native_streaming: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_native_structured_output: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_none_reasoning_effort: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_nova_canvas_image_edit: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_output_config: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_parallel_function_calling: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_parallel_tool_use_config: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_pdf_input: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_prompt_cache_breakpoint: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_prompt_caching: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_reasoning: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_response_schema: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_sampling_params: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_speed: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_system_messages: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_thinking_cache_preservation: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_tool_choice: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_tool_search: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_url_context: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_video_input: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_vision: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_web_search: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supports_xhigh_reasoning_effort: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub thinking_always_on: Option<bool>,
|
||||
/// Context-length or result-count tiered rates; each tier's costs apply within its range.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub tiered_pricing: Option<Vec<TieredRate>>,
|
||||
/// Provider default tokens-per-minute limit.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub tpm: Option<u64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub use_openai_responses_path: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub uses_embed_content: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub vertex_ai_audio_api: Option<VertexAiAudioApi>,
|
||||
/// Whether web search is billed per query or per prompt.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub web_search_billing_unit: Option<WebSearchBillingUnit>,
|
||||
}
|
||||
7
litellm-rust/crates/model-catalog/src/schema.rs
Normal file
7
litellm-rust/crates/model-catalog/src/schema.rs
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
use crate::model_info::ModelInfo;
|
||||
|
||||
/// JSON Schema for one catalog model entry, mirroring
|
||||
/// `model_prices_and_context_window.schema.json`'s `modelEntry` definition.
|
||||
pub fn model_entry_json_schema() -> schemars::Schema {
|
||||
schemars::schema_for!(ModelInfo)
|
||||
}
|
||||
253
litellm-rust/crates/model-catalog/tests/catalog.rs
Normal file
253
litellm-rust/crates/model-catalog/tests/catalog.rs
Normal file
|
|
@ -0,0 +1,253 @@
|
|||
use std::path::{Path, PathBuf};
|
||||
|
||||
use litellm_model_catalog::{AliasIssue, Catalog, Error, IntegrityLimits, Provenance};
|
||||
use rstest::{fixture, rstest};
|
||||
use serde_json::json;
|
||||
|
||||
const ALPHA_FIXTURE: &[u8] = br#"{
|
||||
"sample_spec":{"explanation":"example"},
|
||||
"fallback_generalizations":{"rules":[{"name":"family","pattern":"^new-","model_info":{"mode":"chat"}}]},
|
||||
"Alpha":{"litellm_provider":"test","aliases":["short"],"price":0,"enabled":false,
|
||||
"optional":null,"unknown":{"nested":[1,{"x":true}]}}
|
||||
}"#;
|
||||
|
||||
#[fixture]
|
||||
fn repo_root() -> PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..")
|
||||
}
|
||||
|
||||
#[fixture]
|
||||
fn current_catalog(repo_root: PathBuf) -> Catalog {
|
||||
let body = std::fs::read(repo_root.join("model_prices_and_context_window.json")).unwrap();
|
||||
Catalog::parse(&body, Provenance::default()).unwrap()
|
||||
}
|
||||
|
||||
#[fixture]
|
||||
fn backup_catalog(repo_root: PathBuf) -> Catalog {
|
||||
let body = std::fs::read(repo_root.join("litellm/model_prices_and_context_window_backup.json"))
|
||||
.unwrap();
|
||||
Catalog::parse(&body, Provenance::default()).unwrap()
|
||||
}
|
||||
|
||||
#[fixture]
|
||||
fn fixture_catalog() -> Catalog {
|
||||
Catalog::parse(
|
||||
ALPHA_FIXTURE,
|
||||
Provenance {
|
||||
source: Some("fixture".into()),
|
||||
revision: Some("rev".into()),
|
||||
etag: None,
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn preserves_fields_and_metadata(fixture_catalog: Catalog) {
|
||||
let catalog = fixture_catalog;
|
||||
let entry = catalog.lookup("SHORT").unwrap();
|
||||
assert_eq!(entry.canonical_key, "Alpha");
|
||||
assert_eq!(entry.matched_key, "short");
|
||||
assert_eq!(entry.entry.field("price"), Some(&json!(0)));
|
||||
assert_eq!(entry.entry.field("enabled"), Some(&json!(false)));
|
||||
assert_eq!(entry.entry.field("optional"), Some(&json!(null)));
|
||||
assert_eq!(entry.entry.field("missing"), None);
|
||||
assert_eq!(
|
||||
entry.entry.field("unknown"),
|
||||
Some(&json!({"nested":[1,{"x":true}]}))
|
||||
);
|
||||
assert_eq!(entry.entry.field("aliases"), None);
|
||||
assert_eq!(entry.entry.info().litellm_provider.as_deref(), Some("test"));
|
||||
assert_eq!(
|
||||
catalog.sample_spec(),
|
||||
Some(&json!({"explanation":"example"}))
|
||||
);
|
||||
assert_eq!(catalog.fallback_rules().unwrap().len(), 1);
|
||||
assert_eq!(catalog.provenance().revision.as_deref(), Some("rev"));
|
||||
assert_eq!(catalog.model_count(), 1);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn snapshot_does_not_borrow_source() {
|
||||
let mut source = ALPHA_FIXTURE.to_vec();
|
||||
let catalog = Catalog::parse(&source, Provenance::default()).unwrap();
|
||||
source.fill(b' ');
|
||||
|
||||
let entry = catalog.lookup("short").unwrap();
|
||||
assert_eq!(entry.canonical_key, "Alpha");
|
||||
assert_eq!(entry.entry.field("price"), Some(&json!(0)));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("Shared", "First")]
|
||||
#[case("Second", "Second")]
|
||||
#[case("shared", "Second")]
|
||||
#[case("FIRST", "First")]
|
||||
#[case("sHaReD", "Second")]
|
||||
fn alias_collisions_and_case_fallback_follow_python_order(
|
||||
#[case] lookup: &str,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
let catalog = Catalog::parse(
|
||||
br#"{
|
||||
"First":{"aliases":["Shared","Second","first"],"value":1},
|
||||
"Second":{"aliases":["Shared","sHaReD"],"value":2},
|
||||
"SHARED":{"value":3}
|
||||
}"#,
|
||||
Provenance::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(catalog.lookup(lookup).unwrap().canonical_key, expected);
|
||||
assert_eq!(catalog.alias_count(), 3);
|
||||
assert!(
|
||||
catalog
|
||||
.alias_issues()
|
||||
.contains(&AliasIssue::CanonicalCollision {
|
||||
model: "First".into(),
|
||||
alias: "Second".into(),
|
||||
})
|
||||
);
|
||||
assert!(
|
||||
catalog
|
||||
.alias_issues()
|
||||
.contains(&AliasIssue::AliasCollision {
|
||||
model: "Second".into(),
|
||||
alias: "Shared".into(),
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum ValidationOutcome {
|
||||
Ok,
|
||||
Shrunk,
|
||||
BelowMinimum,
|
||||
InvalidRatio,
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case(
|
||||
IntegrityLimits {
|
||||
backup_model_count: 2,
|
||||
min_model_count: 1,
|
||||
min_backup_ratio: 0.5,
|
||||
},
|
||||
ValidationOutcome::Ok
|
||||
)]
|
||||
#[case(
|
||||
IntegrityLimits {
|
||||
backup_model_count: 3,
|
||||
min_model_count: 1,
|
||||
min_backup_ratio: 0.5,
|
||||
},
|
||||
ValidationOutcome::Shrunk
|
||||
)]
|
||||
#[case(
|
||||
IntegrityLimits {
|
||||
backup_model_count: 0,
|
||||
min_model_count: 2,
|
||||
min_backup_ratio: 0.5,
|
||||
},
|
||||
ValidationOutcome::BelowMinimum
|
||||
)]
|
||||
#[case(
|
||||
IntegrityLimits {
|
||||
backup_model_count: 0,
|
||||
min_model_count: 0,
|
||||
min_backup_ratio: f64::NAN,
|
||||
},
|
||||
ValidationOutcome::InvalidRatio
|
||||
)]
|
||||
fn integrity_uses_canonical_count_and_strict_shrink_boundary(
|
||||
#[case] limits: IntegrityLimits,
|
||||
#[case] expected: ValidationOutcome,
|
||||
) {
|
||||
let catalog = Catalog::parse(
|
||||
br#"{"sample_spec":{},"fallback_generalizations":{"rules":[]},"a":{"aliases":["b","c"]}}"#,
|
||||
Provenance::default(),
|
||||
)
|
||||
.unwrap();
|
||||
let actual = catalog.validate(limits);
|
||||
match expected {
|
||||
ValidationOutcome::Ok => assert!(actual.is_ok()),
|
||||
ValidationOutcome::Shrunk => {
|
||||
assert!(matches!(actual, Err(Error::Shrunk { actual: 1, .. })))
|
||||
}
|
||||
ValidationOutcome::BelowMinimum => {
|
||||
assert!(matches!(actual, Err(Error::BelowMinimum { actual: 1, .. })))
|
||||
}
|
||||
ValidationOutcome::InvalidRatio => assert!(matches!(actual, Err(Error::InvalidRatio))),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum MalformedOutcome {
|
||||
Empty,
|
||||
Json,
|
||||
EntryNotObject,
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::empty(b"{}", MalformedOutcome::Empty)]
|
||||
#[case::invalid_json(b"{", MalformedOutcome::Json)]
|
||||
#[case::entry_not_object(br#"{"a":1}"#, MalformedOutcome::EntryNotObject)]
|
||||
#[case::fallback_rules_missing(
|
||||
br#"{"fallback_generalizations":{},"a":{}}"#,
|
||||
MalformedOutcome::Json
|
||||
)]
|
||||
fn malformed_input_and_aliases_have_typed_outcomes(
|
||||
#[case] body: &[u8],
|
||||
#[case] expected: MalformedOutcome,
|
||||
) {
|
||||
let actual = Catalog::parse(body, Provenance::default());
|
||||
match expected {
|
||||
MalformedOutcome::Empty => assert!(matches!(actual, Err(Error::Empty))),
|
||||
MalformedOutcome::Json => assert!(matches!(actual, Err(Error::Json(_)))),
|
||||
MalformedOutcome::EntryNotObject => {
|
||||
assert!(matches!(actual, Err(Error::EntryNotObject { .. })))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn invalid_aliases_are_reported_not_fatal() {
|
||||
let catalog = Catalog::parse(
|
||||
br#"{"a":{"aliases":"bad"},"b":{"aliases":[9,"ok"]}}"#,
|
||||
Provenance::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
catalog.alias_issues(),
|
||||
&[
|
||||
AliasIssue::InvalidList { model: "a".into() },
|
||||
AliasIssue::InvalidName { model: "b".into() },
|
||||
]
|
||||
);
|
||||
assert_eq!(catalog.lookup("ok").unwrap().canonical_key, "b");
|
||||
assert!(catalog.lookup("missing").is_none());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn parses_current_and_packaged_catalogs_without_pinning_counts(
|
||||
current_catalog: Catalog,
|
||||
backup_catalog: Catalog,
|
||||
) {
|
||||
assert!(current_catalog.model_count() > 0);
|
||||
assert!(backup_catalog.model_count() > 0);
|
||||
assert!(current_catalog.sample_spec().is_some());
|
||||
assert!(backup_catalog.sample_spec().is_some());
|
||||
assert!(
|
||||
current_catalog
|
||||
.validate(IntegrityLimits::python_defaults(
|
||||
backup_catalog.model_count()
|
||||
))
|
||||
.is_ok()
|
||||
);
|
||||
for name in current_catalog.model_names() {
|
||||
let entry = current_catalog.lookup(name).unwrap().entry;
|
||||
assert_eq!(
|
||||
entry.info().litellm_provider.is_some(),
|
||||
entry.field("litellm_provider").is_some()
|
||||
);
|
||||
}
|
||||
}
|
||||
121
litellm-rust/crates/model-catalog/tests/spec_parity.rs
Normal file
121
litellm-rust/crates/model-catalog/tests/spec_parity.rs
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
use std::collections::{BTreeSet, HashSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use indexmap::IndexMap;
|
||||
use litellm_model_catalog::{
|
||||
Catalog, FallbackGeneralizations, ModelInfo, Provenance, model_entry_json_schema,
|
||||
};
|
||||
use rstest::{fixture, rstest};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
#[fixture]
|
||||
fn repo_root() -> PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..")
|
||||
}
|
||||
|
||||
fn json_eq(left: &Value, right: &Value) -> bool {
|
||||
match (left, right) {
|
||||
(Value::Number(left), Value::Number(right)) => left.as_f64() == right.as_f64(),
|
||||
(Value::Array(left), Value::Array(right)) => {
|
||||
left.len() == right.len() && left.iter().zip(right).all(|(a, b)| json_eq(a, b))
|
||||
}
|
||||
(Value::Object(left), Value::Object(right)) => {
|
||||
left.len() == right.len()
|
||||
&& left
|
||||
.iter()
|
||||
.all(|(key, value)| right.get(key).is_some_and(|other| json_eq(value, other)))
|
||||
}
|
||||
_ => left == right,
|
||||
}
|
||||
}
|
||||
|
||||
fn keys(value: &Map<String, Value>) -> BTreeSet<String> {
|
||||
value.keys().cloned().collect()
|
||||
}
|
||||
|
||||
fn symmetric_difference(left: &BTreeSet<String>, right: &BTreeSet<String>) -> BTreeSet<String> {
|
||||
left.symmetric_difference(right).cloned().collect()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("model_prices_and_context_window.json")]
|
||||
#[case("litellm/model_prices_and_context_window_backup.json")]
|
||||
fn every_entry_round_trips_through_model_info(repo_root: PathBuf, #[case] filename: &str) {
|
||||
let body = std::fs::read(repo_root.join(filename)).unwrap();
|
||||
let document: IndexMap<String, Value> = serde_json::from_slice(&body).unwrap();
|
||||
for (model_name, value) in document {
|
||||
if matches!(
|
||||
model_name.as_str(),
|
||||
"sample_spec" | "fallback_generalizations"
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
let object = value
|
||||
.as_object()
|
||||
.unwrap_or_else(|| panic!("{model_name} is not an object"));
|
||||
let info: ModelInfo = serde_json::from_value(value.clone())
|
||||
.unwrap_or_else(|error| panic!("{model_name} does not deserialize: {error}"));
|
||||
let serialized = serde_json::to_value(info).unwrap();
|
||||
let serialized_object = serialized
|
||||
.as_object()
|
||||
.unwrap_or_else(|| panic!("{model_name} did not serialize as an object"));
|
||||
let mut expected = object.clone();
|
||||
expected.remove("aliases");
|
||||
let expected_keys = keys(&expected);
|
||||
let serialized_keys = keys(serialized_object);
|
||||
assert_eq!(
|
||||
expected_keys,
|
||||
serialized_keys,
|
||||
"{model_name} key difference: {:?}",
|
||||
symmetric_difference(&expected_keys, &serialized_keys)
|
||||
);
|
||||
assert!(
|
||||
json_eq(&Value::Object(expected), &serialized),
|
||||
"{model_name} changed during ModelInfo round-trip"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn fallback_generalizations_are_typed(repo_root: PathBuf) {
|
||||
let body = std::fs::read(repo_root.join("model_prices_and_context_window.json")).unwrap();
|
||||
let document: Map<String, Value> = serde_json::from_slice(&body).unwrap();
|
||||
let Some(raw_rules) = document.get("fallback_generalizations") else {
|
||||
return;
|
||||
};
|
||||
let _: FallbackGeneralizations = serde_json::from_value(raw_rules.clone()).unwrap();
|
||||
let catalog = Catalog::parse(&body, Provenance::default()).unwrap();
|
||||
assert!(
|
||||
catalog
|
||||
.fallback_rules()
|
||||
.is_some_and(|rules| !rules.is_empty())
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn generated_schema_properties_match_repo_schema(repo_root: PathBuf) {
|
||||
let body =
|
||||
std::fs::read(repo_root.join("model_prices_and_context_window.schema.json")).unwrap();
|
||||
let document: Value = serde_json::from_slice(&body).unwrap();
|
||||
let repo_entry_properties = document["$defs"]["modelEntry"]["properties"]
|
||||
.as_object()
|
||||
.unwrap();
|
||||
let generated = serde_json::to_value(model_entry_json_schema()).unwrap();
|
||||
let generated_properties = generated["properties"].as_object().unwrap();
|
||||
let expected = keys(repo_entry_properties);
|
||||
let actual = keys(generated_properties);
|
||||
assert_eq!(
|
||||
expected,
|
||||
actual,
|
||||
"modelEntry property difference: {:?}",
|
||||
symmetric_difference(&expected, &actual)
|
||||
);
|
||||
|
||||
let repo_root_properties = document["properties"].as_object().unwrap();
|
||||
let actual_root: HashSet<String> = repo_root_properties.keys().cloned().collect();
|
||||
let expected_root: HashSet<String> = ["sample_spec", "fallback_generalizations"]
|
||||
.into_iter()
|
||||
.map(str::to_owned)
|
||||
.collect();
|
||||
assert_eq!(actual_root, expected_root);
|
||||
}
|
||||
|
|
@ -1402,6 +1402,7 @@ from .exceptions import (
|
|||
JSONSchemaValidationError,
|
||||
LITELLM_EXCEPTION_TYPES,
|
||||
MockException,
|
||||
ModelNotMappedError as ModelNotMappedError,
|
||||
)
|
||||
from .budget_manager import BudgetManager
|
||||
from .proxy.proxy_cli import run_server
|
||||
|
|
|
|||
|
|
@ -991,6 +991,10 @@ LITELLM_EXCEPTION_TYPES: Final = [
|
|||
]
|
||||
|
||||
|
||||
class ModelNotMappedError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class BudgetExceededError(Exception):
|
||||
def __init__(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -277,7 +277,7 @@ def get_s3_object_key(
|
|||
start_time: datetime,
|
||||
s3_file_name: str,
|
||||
) -> str:
|
||||
sanitized_s3_file_name: Final = s3_file_name.replace("/", "_")
|
||||
sanitized_s3_file_name: Final = s3_file_name.replace("/", "_").replace(":", "_")
|
||||
configured_prefix: Final = (s3_path.rstrip("/") + "/" if s3_path else "") + prefix
|
||||
date_segment: Final = start_time.strftime("%Y-%m-%d") + "/"
|
||||
# we need the s3 key to include the time, so we log cache hits too
|
||||
|
|
|
|||
195
litellm/litellm_core_utils/error_normalization.py
Normal file
195
litellm/litellm_core_utils/error_normalization.py
Normal file
|
|
@ -0,0 +1,195 @@
|
|||
"""
|
||||
Map any exception litellm logs to one stable ``normalized_error`` code so dashboards can cluster
|
||||
failures without parsing free-text messages that embed team names, token counts, model names, etc.
|
||||
"""
|
||||
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Protocol, runtime_checkable
|
||||
|
||||
from litellm.exceptions import (
|
||||
APIConnectionError,
|
||||
AuthenticationError,
|
||||
BadGatewayError,
|
||||
BadRequestError,
|
||||
BlockedPiiEntityError,
|
||||
BudgetExceededError,
|
||||
ContentPolicyViolationError,
|
||||
ContextWindowExceededError,
|
||||
GuardrailRaisedException,
|
||||
InternalServerError,
|
||||
MidStreamFallbackError,
|
||||
NotFoundError,
|
||||
PermissionDeniedError,
|
||||
RateLimitError,
|
||||
RateLimitType,
|
||||
ServiceUnavailableError,
|
||||
Timeout,
|
||||
UnprocessableEntityError,
|
||||
UnsupportedParamsError,
|
||||
)
|
||||
|
||||
RATE_LIMIT_EXCEEDED: Final = "429_RATE_LIMIT_EXCEEDED"
|
||||
BUDGET_EXCEEDED: Final = "429_BUDGET_EXCEEDED"
|
||||
NO_HEALTHY_DEPLOYMENTS: Final = "429_NO_HEALTHY_DEPLOYMENTS"
|
||||
AUTHENTICATION_FAILED: Final = "401_AUTHENTICATION_FAILED"
|
||||
MODEL_ACCESS_DENIED: Final = "403_MODEL_ACCESS_DENIED"
|
||||
PERMISSION_DENIED: Final = "403_PERMISSION_DENIED"
|
||||
MISSING_REQUIRED_PARAMETER: Final = "400_MISSING_REQUIRED_PARAMETER"
|
||||
INVALID_PARAMETER_VALUE: Final = "400_INVALID_PARAMETER_VALUE"
|
||||
CONTEXT_WINDOW_EXCEEDED: Final = "400_CONTEXT_WINDOW_EXCEEDED"
|
||||
CONTENT_POLICY_VIOLATION: Final = "400_CONTENT_POLICY_VIOLATION"
|
||||
INVALID_REQUEST: Final = "400_INVALID_REQUEST"
|
||||
RESOURCE_NOT_FOUND: Final = "404_RESOURCE_NOT_FOUND"
|
||||
UPSTREAM_TIMEOUT: Final = "408_UPSTREAM_TIMEOUT"
|
||||
PROVIDER_CONNECTION_ERROR: Final = "500_PROVIDER_CONNECTION_ERROR"
|
||||
PROVIDER_OVERLOADED: Final = "503_PROVIDER_OVERLOADED"
|
||||
PROVIDER_INTERNAL_ERROR: Final = "500_PROVIDER_INTERNAL_ERROR"
|
||||
ROUTER_NO_FALLBACK: Final = "500_ROUTER_NO_FALLBACK"
|
||||
ROUTER_FALLBACK_FAILURE: Final = "500_ROUTER_FALLBACK_FAILURE"
|
||||
UPSTREAM_PASSTHROUGH: Final = "500_UPSTREAM_PASSTHROUGH"
|
||||
UNSUPPORTED_OPERATION: Final = "500_UNSUPPORTED_OPERATION"
|
||||
INTERNAL_STATE_ERROR: Final = "500_INTERNAL_STATE_ERROR"
|
||||
UNCLASSIFIED: Final = "UNCLASSIFIED"
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _HasProxyErrorType(Protocol):
|
||||
type: str
|
||||
|
||||
|
||||
_MESSAGE_PATTERNS: Final[tuple[tuple[re.Pattern[str], str], ...]] = (
|
||||
(
|
||||
re.compile(r"budget has been exceeded|max budget|exceeded.*budget|crossed budget", re.IGNORECASE),
|
||||
BUDGET_EXCEEDED,
|
||||
),
|
||||
(re.compile(r"no healthy deployments?|no deployments available", re.IGNORECASE), NO_HEALTHY_DEPLOYMENTS),
|
||||
(re.compile(r"not allowed to access model due to tags configuration", re.IGNORECASE), MODEL_ACCESS_DENIED),
|
||||
(re.compile(r"upstream passthrough request failed", re.IGNORECASE), UPSTREAM_PASSTHROUGH),
|
||||
(re.compile(r"is not supported for provider|not implemented", re.IGNORECASE), UNSUPPORTED_OPERATION),
|
||||
(
|
||||
re.compile(r"context window|context length|(prompt|input) is too long|tokens? ?> ?\d+ ?maximum", re.IGNORECASE),
|
||||
CONTEXT_WINDOW_EXCEEDED,
|
||||
),
|
||||
(re.compile(r"missing required parameter|field required", re.IGNORECASE), MISSING_REQUIRED_PARAMETER),
|
||||
(re.compile(r"overloaded|unable to process your request", re.IGNORECASE), PROVIDER_OVERLOADED),
|
||||
(
|
||||
re.compile(
|
||||
r"connection error|APIConnectionError|TransferEncodingError|payload is not completed|connection reset"
|
||||
r"|peer closed connection|incomplete chunked read",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
PROVIDER_CONNECTION_ERROR,
|
||||
),
|
||||
(re.compile(r"timed? ?out", re.IGNORECASE), UPSTREAM_TIMEOUT),
|
||||
)
|
||||
|
||||
_ROUTER_WRAPPER_PATTERNS: Final[tuple[tuple[re.Pattern[str], str], ...]] = (
|
||||
(re.compile(r"no fallback model group found", re.IGNORECASE), ROUTER_NO_FALLBACK),
|
||||
(re.compile(r"error doing the fallback|MidStreamFallbackError", re.IGNORECASE), ROUTER_FALLBACK_FAILURE),
|
||||
)
|
||||
|
||||
_PROXY_ERROR_TYPE_MAP: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{
|
||||
"budget_exceeded": BUDGET_EXCEEDED,
|
||||
"auth_error": AUTHENTICATION_FAILED,
|
||||
"expired_key": AUTHENTICATION_FAILED,
|
||||
"token_not_found_in_db": AUTHENTICATION_FAILED,
|
||||
"auth_provider_unavailable": AUTHENTICATION_FAILED,
|
||||
"key_model_access_denied": MODEL_ACCESS_DENIED,
|
||||
"team_model_access_denied": MODEL_ACCESS_DENIED,
|
||||
"user_model_access_denied": MODEL_ACCESS_DENIED,
|
||||
"org_model_access_denied": MODEL_ACCESS_DENIED,
|
||||
"project_model_access_denied": MODEL_ACCESS_DENIED,
|
||||
"agent_model_access_denied": MODEL_ACCESS_DENIED,
|
||||
"key_vector_store_access_denied": PERMISSION_DENIED,
|
||||
"team_vector_store_access_denied": PERMISSION_DENIED,
|
||||
"org_vector_store_access_denied": PERMISSION_DENIED,
|
||||
"tool_access_denied": PERMISSION_DENIED,
|
||||
"team_member_permission_error": PERMISSION_DENIED,
|
||||
"not_found_error": RESOURCE_NOT_FOUND,
|
||||
}
|
||||
)
|
||||
|
||||
_STATUS_CODE_MAP: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{
|
||||
"400": INVALID_REQUEST,
|
||||
"401": AUTHENTICATION_FAILED,
|
||||
"403": PERMISSION_DENIED,
|
||||
"404": RESOURCE_NOT_FOUND,
|
||||
"408": UPSTREAM_TIMEOUT,
|
||||
"422": INVALID_PARAMETER_VALUE,
|
||||
"429": RATE_LIMIT_EXCEEDED,
|
||||
"500": PROVIDER_INTERNAL_ERROR,
|
||||
"502": PROVIDER_INTERNAL_ERROR,
|
||||
"503": PROVIDER_OVERLOADED,
|
||||
"504": UPSTREAM_TIMEOUT,
|
||||
}
|
||||
)
|
||||
|
||||
_INTERNAL_STATE_EXCEPTIONS: Final[tuple[type[BaseException], ...]] = (
|
||||
TypeError,
|
||||
KeyError,
|
||||
AttributeError,
|
||||
IndexError,
|
||||
RuntimeError,
|
||||
AssertionError,
|
||||
ZeroDivisionError,
|
||||
)
|
||||
|
||||
_CLASS_CODE_TABLE: Final[tuple[tuple[tuple[type[BaseException], ...], str], ...]] = (
|
||||
((AuthenticationError,), AUTHENTICATION_FAILED),
|
||||
((PermissionDeniedError,), PERMISSION_DENIED),
|
||||
((ContextWindowExceededError,), CONTEXT_WINDOW_EXCEEDED),
|
||||
((ContentPolicyViolationError, GuardrailRaisedException, BlockedPiiEntityError), CONTENT_POLICY_VIOLATION),
|
||||
((UnsupportedParamsError,), INVALID_PARAMETER_VALUE),
|
||||
((NotFoundError,), RESOURCE_NOT_FOUND),
|
||||
((Timeout,), UPSTREAM_TIMEOUT),
|
||||
((MidStreamFallbackError,), ROUTER_FALLBACK_FAILURE),
|
||||
((APIConnectionError,), PROVIDER_CONNECTION_ERROR),
|
||||
((ServiceUnavailableError,), PROVIDER_OVERLOADED),
|
||||
((InternalServerError, BadGatewayError), PROVIDER_INTERNAL_ERROR),
|
||||
((BadRequestError, UnprocessableEntityError), INVALID_REQUEST),
|
||||
((NotImplementedError,), UNSUPPORTED_OPERATION),
|
||||
)
|
||||
|
||||
|
||||
def _classify_by_message(message: str, patterns: tuple[tuple[re.Pattern[str], str], ...]) -> str | None:
|
||||
return next((code for pattern, code in patterns if pattern.search(message)), None)
|
||||
|
||||
|
||||
def _classify_by_class(exc: Exception) -> str | None:
|
||||
if isinstance(exc, BudgetExceededError):
|
||||
return BUDGET_EXCEEDED
|
||||
if isinstance(exc, RateLimitError):
|
||||
return BUDGET_EXCEEDED if exc.rate_limit_type == RateLimitType.BUDGET.value else RATE_LIMIT_EXCEEDED
|
||||
for exc_types, code in _CLASS_CODE_TABLE:
|
||||
if isinstance(exc, exc_types):
|
||||
return code
|
||||
if isinstance(exc, _INTERNAL_STATE_EXCEPTIONS):
|
||||
return INTERNAL_STATE_ERROR
|
||||
return None
|
||||
|
||||
|
||||
def normalize_error(exc: Exception | None, status_code: str, message: str) -> str | None:
|
||||
"""
|
||||
Return a stable cluster key for ``exc``. ``status_code`` and ``message`` are the values
|
||||
``get_error_information`` already extracted, so the same exception always yields the same code.
|
||||
"""
|
||||
if exc is None:
|
||||
return None
|
||||
proxy_type: Final = exc.type if isinstance(exc, _HasProxyErrorType) else None
|
||||
by_proxy_type: Final = _PROXY_ERROR_TYPE_MAP.get(proxy_type) if isinstance(proxy_type, str) else None
|
||||
if by_proxy_type is not None:
|
||||
return by_proxy_type
|
||||
by_message: Final = _classify_by_message(message, _MESSAGE_PATTERNS)
|
||||
if by_message is not None:
|
||||
return by_message
|
||||
by_class: Final = _classify_by_class(exc)
|
||||
if by_class is not None:
|
||||
return by_class
|
||||
by_router_wrapper: Final = _classify_by_message(message, _ROUTER_WRAPPER_PATTERNS)
|
||||
if by_router_wrapper is not None:
|
||||
return by_router_wrapper
|
||||
return _STATUS_CODE_MAP.get(status_code, UNCLASSIFIED)
|
||||
|
|
@ -76,6 +76,7 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
reconstruct_model_name,
|
||||
set_response_cost_in_hidden_params,
|
||||
)
|
||||
from litellm.litellm_core_utils.error_normalization import normalize_error
|
||||
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
|
||||
from litellm.litellm_core_utils.internal_call_metadata import (
|
||||
MODEL_ACCESS_GROUP_METADATA_KEY,
|
||||
|
|
@ -6100,6 +6101,7 @@ class StandardLoggingPayloadSetup:
|
|||
error_budget_entity_id=budget_error.entity_id if budget_error else None,
|
||||
error_budget_limit=budget_error.max_budget if budget_error else None,
|
||||
error_budget_spend=budget_error.current_cost if budget_error else None,
|
||||
normalized_error=normalize_error(original_exception, error_status, error_message),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -3,10 +3,30 @@ from typing import Final
|
|||
import litellm
|
||||
from litellm import verbose_logger
|
||||
|
||||
from ...litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from ...litellm_core_utils.get_llm_provider_logic import (
|
||||
declared_authenticating_provider,
|
||||
get_llm_provider,
|
||||
)
|
||||
from ...types.router import LiteLLM_Params
|
||||
|
||||
|
||||
def _api_base_without_login(provider: str) -> str | None:
|
||||
if provider == "github_copilot":
|
||||
return litellm.GithubCopilotConfig().api_base_without_login()
|
||||
if provider == "chatgpt":
|
||||
return litellm.ChatGPTConfig().api_base_without_login()
|
||||
return None
|
||||
|
||||
|
||||
def _provider_default_api_base(model: str, custom_llm_provider: str | None, stream: bool) -> str | None:
|
||||
if custom_llm_provider == "gemini":
|
||||
action: Final = "streamGenerateContent" if stream else "generateContent"
|
||||
return f"https://generativelanguage.googleapis.com/v1beta/models/{model}:{action}"
|
||||
if custom_llm_provider == "openai":
|
||||
return "https://api.openai.com"
|
||||
return None
|
||||
|
||||
|
||||
def get_api_base(model: str, optional_params: dict | LiteLLM_Params) -> str | None:
|
||||
"""
|
||||
Returns the api base used for calling the model.
|
||||
|
|
@ -42,6 +62,9 @@ def get_api_base(model: str, optional_params: dict | LiteLLM_Params) -> str | No
|
|||
|
||||
if litellm.model_alias_map and model in litellm.model_alias_map:
|
||||
model = litellm.model_alias_map[model]
|
||||
declared: Final = declared_authenticating_provider(model, _optional_params.custom_llm_provider)
|
||||
if declared is not None:
|
||||
return _api_base_without_login(declared)
|
||||
try:
|
||||
(
|
||||
model,
|
||||
|
|
@ -83,16 +106,4 @@ def get_api_base(model: str, optional_params: dict | LiteLLM_Params) -> str | No
|
|||
_api_base = f"{_optional_params.vertex_location}-aiplatform.googleapis.com/v1/projects/{_optional_params.vertex_project}/locations/{_optional_params.vertex_location}/publishers/google/models/{model}:generateContent"
|
||||
return _api_base
|
||||
|
||||
if custom_llm_provider is None:
|
||||
return None
|
||||
|
||||
if custom_llm_provider == "gemini":
|
||||
if stream:
|
||||
_api_base = f"https://generativelanguage.googleapis.com/v1beta/models/{model}:streamGenerateContent"
|
||||
else:
|
||||
_api_base = f"https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent"
|
||||
return _api_base
|
||||
elif custom_llm_provider == "openai":
|
||||
_api_base = "https://api.openai.com"
|
||||
return _api_base
|
||||
return None
|
||||
return _provider_default_api_base(model, custom_llm_provider, stream)
|
||||
|
|
|
|||
|
|
@ -1657,7 +1657,7 @@ def get_file_ids_from_messages(messages: list[AllMessageValues]) -> list[str]:
|
|||
if isinstance(content, str):
|
||||
continue
|
||||
for c in content:
|
||||
if c["type"] == "file":
|
||||
if isinstance(c, dict) and c["type"] == "file":
|
||||
file_object = cast(ChatCompletionFileObject, c)
|
||||
file_object_file_field = file_object.get("file")
|
||||
if not isinstance(file_object_file_field, dict):
|
||||
|
|
|
|||
|
|
@ -314,7 +314,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
_message_content = message.get("content")
|
||||
if _message_content is not None and isinstance(_message_content, list):
|
||||
for content in _message_content:
|
||||
if "cache_control" in content:
|
||||
if isinstance(content, dict) and "cache_control" in content:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
|
@ -359,7 +359,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
for message in messages:
|
||||
if "content" in message and message["content"] is not None and isinstance(message["content"], list):
|
||||
for content in message["content"]:
|
||||
if "type" in content and content["type"] != "text":
|
||||
if isinstance(content, dict) and "type" in content and content["type"] != "text":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
|
|
|||
|
|
@ -1593,7 +1593,7 @@ def _resolve_s3_setting(
|
|||
source.get(param_name) for source in (litellm_params, optional_params) if source is not None
|
||||
)
|
||||
explicit: Final = next((value for value in candidates if isinstance(value, str) and value), None)
|
||||
return explicit or get_secret_str(env_var)
|
||||
return explicit or get_secret_str(env_var) or None
|
||||
|
||||
|
||||
class CommonBatchFilesUtils:
|
||||
|
|
|
|||
|
|
@ -23,6 +23,9 @@ class ChatGPTConfig(OpenAIConfig):
|
|||
super().__init__()
|
||||
self.authenticator = Authenticator()
|
||||
|
||||
def api_base_without_login(self) -> str:
|
||||
return self.authenticator.get_api_base()
|
||||
|
||||
def _get_openai_compatible_provider_info(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -30,7 +33,7 @@ class ChatGPTConfig(OpenAIConfig):
|
|||
api_key: str | None,
|
||||
custom_llm_provider: str,
|
||||
) -> tuple[str | None, str | None, str]:
|
||||
dynamic_api_base: Final = self.authenticator.get_api_base()
|
||||
dynamic_api_base: Final = self.api_base_without_login()
|
||||
try:
|
||||
dynamic_api_key: Final = self.authenticator.get_access_token()
|
||||
except GetAccessTokenError as e:
|
||||
|
|
|
|||
|
|
@ -31,6 +31,14 @@ class GithubCopilotConfig(OpenAIConfig):
|
|||
super().__init__()
|
||||
self.authenticator = Authenticator()
|
||||
|
||||
def api_base_without_login(self, api_base: str | None = None) -> str:
|
||||
return (
|
||||
api_base
|
||||
or self.authenticator.get_api_base()
|
||||
or os.getenv("GITHUB_COPILOT_API_BASE")
|
||||
or DEFAULT_GITHUB_COPILOT_API_BASE
|
||||
)
|
||||
|
||||
def _get_openai_compatible_provider_info(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -38,12 +46,7 @@ class GithubCopilotConfig(OpenAIConfig):
|
|||
api_key: str | None,
|
||||
custom_llm_provider: str,
|
||||
) -> tuple[str | None, str | None, str]:
|
||||
dynamic_api_base: Final = (
|
||||
api_base
|
||||
or self.authenticator.get_api_base()
|
||||
or os.getenv("GITHUB_COPILOT_API_BASE")
|
||||
or DEFAULT_GITHUB_COPILOT_API_BASE
|
||||
)
|
||||
dynamic_api_base: Final = self.api_base_without_login(api_base)
|
||||
try:
|
||||
dynamic_api_key: Final = self.authenticator.get_api_key()
|
||||
except GetAPIKeyError as e:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import base64
|
||||
import io
|
||||
from typing import Any, Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -11,37 +13,35 @@ class OllamaError(BaseLLMException):
|
|||
super().__init__(status_code=status_code, message=message, headers=headers)
|
||||
|
||||
|
||||
def _convert_image(image):
|
||||
"""
|
||||
Convert image to base64 encoded image if not already in base64 format
|
||||
_JPEG_AND_PNG_SIGNATURES: Final = (b"\xff\xd8\xff", b"\x89PNG\r\n\x1a\n")
|
||||
|
||||
If image is already in base64 format AND is a jpeg/png, return it
|
||||
|
||||
If image is not JPEG/PNG, convert it to JPEG base64 format
|
||||
"""
|
||||
import base64
|
||||
import io
|
||||
|
||||
def _reencode_as_jpeg(raw_image: bytes, original: str) -> str:
|
||||
try:
|
||||
from PIL import Image
|
||||
except Exception:
|
||||
raise Exception("ollama image conversion failed please run `pip install Pillow`")
|
||||
|
||||
orig: Final = image
|
||||
if image.startswith("data:"):
|
||||
image = image.split(",")[-1]
|
||||
try:
|
||||
image_data: Final = Image.open(io.BytesIO(base64.b64decode(image)))
|
||||
if image_data.format in ["JPEG", "PNG"]:
|
||||
return image
|
||||
picture: Final = Image.open(io.BytesIO(raw_image))
|
||||
except Exception:
|
||||
return orig
|
||||
return original
|
||||
jpeg_image: Final = io.BytesIO()
|
||||
image_data.convert("RGB").save(jpeg_image, "JPEG")
|
||||
jpeg_image.seek(0)
|
||||
picture.convert("RGB").save(jpeg_image, "JPEG")
|
||||
return base64.b64encode(jpeg_image.getvalue()).decode("utf-8")
|
||||
|
||||
|
||||
def _convert_image(image: str) -> str:
|
||||
payload: Final = image.split(",")[-1] if image.startswith("data:") else image
|
||||
try:
|
||||
raw_image: Final = base64.b64decode(payload)
|
||||
except ValueError:
|
||||
return image
|
||||
if raw_image.startswith(_JPEG_AND_PNG_SIGNATURES):
|
||||
return payload
|
||||
return _reencode_as_jpeg(raw_image, original=image)
|
||||
|
||||
|
||||
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -25314,8 +25314,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"cache_read_input_token_cost_batches": 1e-07
|
||||
"web_search_billing_unit": "per_query"
|
||||
},
|
||||
"gemini-3-pro-image-preview": {
|
||||
"input_cost_per_image": 0.0011,
|
||||
|
|
@ -25401,8 +25400,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"cache_read_input_token_cost_batches": 2.5e-08
|
||||
"web_search_billing_unit": "per_query"
|
||||
},
|
||||
"gemini-3.1-flash-image-preview": {
|
||||
"input_cost_per_image": 0.00056,
|
||||
|
|
@ -25482,8 +25480,7 @@
|
|||
"supports_response_schema": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_video_input": true,
|
||||
"supports_vision": true,
|
||||
"cache_read_input_token_cost_batches": 1.25e-08
|
||||
"supports_vision": true
|
||||
},
|
||||
"gemini-3.1-flash-lite-preview": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
|
|
@ -25593,8 +25590,7 @@
|
|||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_audio_token_batches": 2.5e-07,
|
||||
"cache_read_input_token_cost_batches": 1.25e-08
|
||||
"input_cost_per_audio_token_batches": 2.5e-07
|
||||
},
|
||||
"gemini-3.5-flash-lite": {
|
||||
"deprecation_date": "2027-07-21",
|
||||
|
|
@ -25652,8 +25648,7 @@
|
|||
"search_context_size_high": 0.014
|
||||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"cache_read_input_token_cost_batches": 1.5e-08
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
"deep-research-pro-preview-12-2025": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
|
|
@ -25689,8 +25684,7 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"cache_read_input_token_cost_batches": 1e-07
|
||||
"supports_web_search": true
|
||||
},
|
||||
"gemini-2.5-flash-lite": {
|
||||
"cache_read_input_audio_token_cost": 3e-08,
|
||||
|
|
@ -26321,8 +26315,7 @@
|
|||
"output_cost_per_token_batches": 4.5e-06,
|
||||
"input_cost_per_token_flex": 7.5e-07,
|
||||
"output_cost_per_token_flex": 4.5e-06,
|
||||
"cache_read_input_token_cost_flex": 7.5e-08,
|
||||
"cache_read_input_token_cost_batches": 7.5e-08
|
||||
"cache_read_input_token_cost_flex": 7.5e-08
|
||||
},
|
||||
"vertex_ai/gemini-3.6-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
|
|
@ -26380,8 +26373,7 @@
|
|||
"search_context_size_high": 0.014
|
||||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
"vertex_ai/gemini-3.7-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
|
|
@ -26440,8 +26432,7 @@
|
|||
"search_context_size_high": 0.014
|
||||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
"vertex_ai/gemini-3.8-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
|
|
@ -26500,8 +26491,7 @@
|
|||
"search_context_size_high": 0.014
|
||||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
"vertex_ai/gemini-3.1-pro-preview": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
|
|
@ -28250,8 +28240,7 @@
|
|||
"output_cost_per_token_batches": 4.5e-06,
|
||||
"input_cost_per_token_flex": 7.5e-07,
|
||||
"output_cost_per_token_flex": 4.5e-06,
|
||||
"cache_read_input_token_cost_flex": 7.5e-08,
|
||||
"cache_read_input_token_cost_batches": 7.5e-08
|
||||
"cache_read_input_token_cost_flex": 7.5e-08
|
||||
},
|
||||
"gemini-3.6-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
|
|
@ -28309,8 +28298,7 @@
|
|||
"search_context_size_high": 0.014
|
||||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
"gemini-3.7-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
|
|
@ -28369,8 +28357,7 @@
|
|||
"search_context_size_high": 0.014
|
||||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
"gemini-3.8-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
|
|
@ -28429,8 +28416,7 @@
|
|||
"search_context_size_high": 0.014
|
||||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
"gemini/gemini-2.5-pro-preview-tts": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -33110,8 +33096,7 @@
|
|||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true,
|
||||
"cache_read_input_token_cost_batches": 6.25e-08
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-chat": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -33292,8 +33277,7 @@
|
|||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true,
|
||||
"cache_read_input_token_cost_batches": 1.25e-08
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-nano": {
|
||||
"cache_read_input_token_cost": 5e-09,
|
||||
|
|
@ -33392,8 +33376,7 @@
|
|||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true,
|
||||
"cache_read_input_token_cost_batches": 2.5e-09
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-image-1": {
|
||||
"cache_read_input_token_cost": 1.25e-06,
|
||||
|
|
@ -47531,8 +47514,7 @@
|
|||
"output_cost_per_token_flex": 6e-06,
|
||||
"output_cost_per_token_priority": 2.16e-05,
|
||||
"supports_reasoning": false,
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"cache_read_input_token_cost_batches": 1e-07
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing"
|
||||
},
|
||||
"vertex_ai/gemini-3-pro-image-preview": {
|
||||
"input_cost_per_image": 0.0011,
|
||||
|
|
@ -47570,8 +47552,7 @@
|
|||
"output_cost_per_token_batches": 1.5e-06,
|
||||
"output_cost_per_token_flex": 1.5e-06,
|
||||
"supports_reasoning": false,
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"cache_read_input_token_cost_batches": 2.5e-08
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing"
|
||||
},
|
||||
"vertex_ai/gemini-3.1-flash-image-preview": {
|
||||
"input_cost_per_image": 0.00056,
|
||||
|
|
@ -47627,8 +47608,7 @@
|
|||
"supports_response_schema": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_video_input": true,
|
||||
"supports_vision": true,
|
||||
"cache_read_input_token_cost_batches": 1.25e-08
|
||||
"supports_vision": true
|
||||
},
|
||||
"vertex_ai/gemini-3.1-flash-lite-preview": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
|
|
@ -47739,8 +47719,7 @@
|
|||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_audio_token_batches": 2.5e-07,
|
||||
"cache_read_input_token_cost_batches": 1.25e-08
|
||||
"input_cost_per_audio_token_batches": 2.5e-07
|
||||
},
|
||||
"vertex_ai/gemini-3.5-flash-lite": {
|
||||
"deprecation_date": "2027-07-21",
|
||||
|
|
@ -47799,8 +47778,7 @@
|
|||
"search_context_size_high": 0.014
|
||||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"cache_read_input_token_cost_batches": 1.5e-08
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
"vertex_ai/deep-research-pro-preview-12-2025": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
|
|
@ -47817,8 +47795,7 @@
|
|||
"output_cost_per_image_token": 0.00012,
|
||||
"output_cost_per_token": 1.2e-05,
|
||||
"output_cost_per_token_batches": 6e-06,
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"cache_read_input_token_cost_batches": 1e-07
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing"
|
||||
},
|
||||
"vertex_ai/jamba-1.5": {
|
||||
"input_cost_per_token": 2e-07,
|
||||
|
|
|
|||
|
|
@ -937,6 +937,7 @@ class LiteLLMRoutes(enum.Enum):
|
|||
# proxy admin, or team admin naming their own team via team_id
|
||||
"/auto_router/test_routing",
|
||||
"/auto_router/validate_complexity_router_config",
|
||||
"/auto_router/availability",
|
||||
# Per-session auto-router read - the endpoint scopes the row to the caller's own key hash
|
||||
"/auto_router/session",
|
||||
"/cost/predict-cache",
|
||||
|
|
|
|||
|
|
@ -1606,13 +1606,21 @@ class DBSpendUpdateWriter:
|
|||
proxy_logging_obj: ProxyLogging,
|
||||
) -> None:
|
||||
transactions: Final = await queue.flush_and_get_aggregated_daily_spend_update_transactions()
|
||||
try:
|
||||
await commit(
|
||||
commit_task: Final = asyncio.ensure_future(
|
||||
commit(
|
||||
n_retry_times=n_retry_times,
|
||||
prisma_client=prisma_client,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
daily_spend_transactions=cast(dict[str, _DailySpendTransactionT], transactions),
|
||||
)
|
||||
)
|
||||
try:
|
||||
await asyncio.shield(commit_task)
|
||||
except asyncio.CancelledError:
|
||||
commit_task.cancel()
|
||||
if transactions:
|
||||
await queue.add_update(transactions)
|
||||
raise
|
||||
except Exception as e: # noqa: BLE001 # whatever failed here, the other tables must still flush
|
||||
if not transactions:
|
||||
return
|
||||
|
|
@ -1839,14 +1847,18 @@ class DBSpendUpdateWriter:
|
|||
if not daily_tag_spend_update_transactions:
|
||||
return
|
||||
|
||||
try:
|
||||
await DBSpendUpdateWriter.update_daily_tag_spend(
|
||||
commit_task: Final = asyncio.ensure_future(
|
||||
DBSpendUpdateWriter.update_daily_tag_spend(
|
||||
n_retry_times=n_retry_times,
|
||||
prisma_client=prisma_client,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
daily_spend_transactions=daily_tag_spend_update_transactions,
|
||||
)
|
||||
except Exception:
|
||||
)
|
||||
try:
|
||||
await asyncio.shield(commit_task)
|
||||
except BaseException: # noqa: BLE001 # a cancel must restore the drained rows before its rollback returns
|
||||
commit_task.cancel()
|
||||
await self.redis_update_buffer.restore_transactions_to_redis(
|
||||
daily_tag_spend_update_transactions=daily_tag_spend_update_transactions,
|
||||
)
|
||||
|
|
@ -2368,7 +2380,8 @@ class DBSpendUpdateWriter:
|
|||
table=table, transactions=tuple(transactions_to_process.values())
|
||||
)
|
||||
sql, params = build_bulk_upsert(table=table, batch=merged_batch)
|
||||
await prisma_client.db.execute_raw(sql, *params)
|
||||
async with _spend_update_tx(prisma_client) as transaction:
|
||||
await transaction.execute_raw(sql, *params)
|
||||
except Exception as batch_error:
|
||||
if _spend_commit_failure_is_requeue_safe(batch_error):
|
||||
spend_log_error(
|
||||
|
|
|
|||
|
|
@ -58,6 +58,8 @@ from litellm.router_utils.auto_router_model_naming import (
|
|||
)
|
||||
from litellm.types.management_endpoints.auto_router_endpoints import (
|
||||
SHADOW_EVAL_TURN_VALVE,
|
||||
AutoRouterAvailabilityRequest,
|
||||
AutoRouterAvailabilityResponse,
|
||||
AutoRouterBenchmarkGroup,
|
||||
AutoRouterBenchmarksResponse,
|
||||
AutoRouterBenchmarkTotals,
|
||||
|
|
@ -391,6 +393,54 @@ async def validate_complexity_router_config(
|
|||
return ComplexityRouterConfigValidationResponse(valid=error is None, error=error)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/auto_router/availability",
|
||||
tags=["model management"], # mutable-ok: FastAPI requires a list
|
||||
response_model=AutoRouterAvailabilityResponse,
|
||||
)
|
||||
async def get_auto_router_availability(
|
||||
data: AutoRouterAvailabilityRequest,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
) -> AutoRouterAvailabilityResponse:
|
||||
from litellm.proxy.management_helpers.auto_router_availability import auto_router_availability
|
||||
from litellm.proxy.proxy_server import (
|
||||
_license_check, # pyright: ignore[reportPrivateUsage] # same entitlement owner as the model write gate
|
||||
heuristic_v1_tuning_baselines,
|
||||
llm_router,
|
||||
proxy_config,
|
||||
)
|
||||
|
||||
member_team: Final = await _authorize_router_dry_run(user_api_key_dict, data.team_id)
|
||||
rows: Final = proxy_config.auto_router_db_catalog
|
||||
if rows is None or llm_router is None:
|
||||
raise HTTPException(status_code=503, detail="Auto-router availability is unavailable")
|
||||
saved: Final = next((row for row in rows if row.model_id == data.saved_model_id), None)
|
||||
if data.saved_model_id is not None:
|
||||
if saved is None:
|
||||
raise HTTPException(status_code=404, detail="Saved auto router is unavailable")
|
||||
if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN and (
|
||||
saved.team_id != data.team_id or (member_team is not None and saved.created_by != user_api_key_dict.user_id)
|
||||
):
|
||||
raise HTTPException(status_code=403, detail="Cannot check another user's auto router")
|
||||
existing: Final = saved.deployment if saved is not None else None
|
||||
others: Final = tuple(row.deployment for row in rows if row is not saved) + tuple(llm_router.config_deployments())
|
||||
candidate: Final = MappingProxyType(
|
||||
{
|
||||
"litellm_params": MappingProxyType(
|
||||
{"model": "auto_router/complexity_router", "complexity_router_config": data.complexity_router_config}
|
||||
),
|
||||
"model_info": MappingProxyType({"id": data.saved_model_id or "availability-new-router", "db_model": True}),
|
||||
}
|
||||
)
|
||||
return auto_router_availability(
|
||||
others=others,
|
||||
existing=existing,
|
||||
candidate=candidate,
|
||||
baselines=heuristic_v1_tuning_baselines,
|
||||
limit=_license_check.auto_router_capability_limit(),
|
||||
)
|
||||
|
||||
|
||||
async def _resolve_saved_routing_test(
|
||||
data: AutoRouterRoutingTestRequest,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
|
|
|
|||
|
|
@ -2741,6 +2741,10 @@ async def _resolve_team_org_filter(
|
|||
async def ui_view_users(
|
||||
user_id: str | None = fastapi.Query(default=None, description="User ID in the request parameters"),
|
||||
user_email: str | None = fastapi.Query(default=None, description="User email in the request parameters"),
|
||||
search: str | None = fastapi.Query(
|
||||
default=None,
|
||||
description="Combined search: matches users whose 'user_id' or 'user_email' contains the value (case-insensitive).",
|
||||
),
|
||||
team_id: str | None = fastapi.Query(
|
||||
default=None,
|
||||
description="Team ID — used when a team admin searches for users to add to their team",
|
||||
|
|
@ -2750,7 +2754,7 @@ async def ui_view_users(
|
|||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
):
|
||||
"""
|
||||
Filter users based on partial match of user_id or email with pagination.
|
||||
Filter users based on partial match of user_id or email, or combined ``search``, with pagination.
|
||||
|
||||
Behaviour depends on the ``scope_user_search_to_org`` UI-setting flag
|
||||
(stored in the ``litellm_uisettings`` table):
|
||||
|
|
@ -2802,9 +2806,15 @@ async def ui_view_users(
|
|||
if org_filter_ids is not None:
|
||||
where_conditions["organization_memberships"] = {"some": {"organization_id": {"in": org_filter_ids}}}
|
||||
|
||||
where: Final[Mapping[str, object]] = { # mutable-ok: prisma serializes `where`, keep it a plain dict
|
||||
key: value
|
||||
for key, value in (*where_conditions.items(), *_user_search_where(search).items())
|
||||
if value is not None
|
||||
}
|
||||
|
||||
# Query users with pagination and filters
|
||||
users: Final = await _user_table(prisma_client).find_many(
|
||||
where=where_conditions,
|
||||
where=where,
|
||||
skip=skip,
|
||||
take=page_size,
|
||||
order={"created_at": "desc"},
|
||||
|
|
|
|||
|
|
@ -1782,10 +1782,11 @@ async def delete_team_models(
|
|||
# Under MODEL_RECONCILE_LOCK, for the same reason as delete_model: the rows are
|
||||
# gone, but a reconcile holding a pre-delete snapshot would upsert these ids back
|
||||
# onto this pod. The lock orders the eviction after any in-flight reconcile.
|
||||
if llm_router is not None:
|
||||
from litellm.proxy.proxy_server import MODEL_RECONCILE_LOCK
|
||||
from litellm.proxy.proxy_server import MODEL_RECONCILE_LOCK, proxy_config
|
||||
|
||||
async with MODEL_RECONCILE_LOCK:
|
||||
async with MODEL_RECONCILE_LOCK:
|
||||
proxy_config.remove_auto_router_catalog_entries(frozenset(deleted_model_ids))
|
||||
if llm_router is not None:
|
||||
for model_id in deleted_model_ids:
|
||||
llm_router.delete_deployment(id=model_id)
|
||||
|
||||
|
|
@ -2194,6 +2195,7 @@ async def delete_model(
|
|||
llm_router,
|
||||
premium_user,
|
||||
prisma_client,
|
||||
proxy_config,
|
||||
proxy_logging_obj,
|
||||
store_model_in_db,
|
||||
user_api_key_cache,
|
||||
|
|
@ -2245,8 +2247,9 @@ async def delete_model(
|
|||
# this pod serving a model the database no longer has, until the next
|
||||
# reconcile. Taking the lock orders this eviction after any such in-flight
|
||||
# reconcile's re-add, so the eviction is the last word.
|
||||
if llm_router is not None:
|
||||
async with MODEL_RECONCILE_LOCK:
|
||||
async with MODEL_RECONCILE_LOCK:
|
||||
proxy_config.remove_auto_router_catalog_entries(frozenset({model_info.id}))
|
||||
if llm_router is not None:
|
||||
llm_router.delete_deployment(id=model_info.id)
|
||||
|
||||
# Runs after the row delete so the sibling check sees post-delete state.
|
||||
|
|
|
|||
148
litellm/proxy/management_helpers/auto_router_availability.py
Normal file
148
litellm/proxy/management_helpers/auto_router_availability.py
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, Json, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
GATED_AUTO_ROUTER_CAPABILITIES,
|
||||
capability_limit_violation,
|
||||
classify_strategy_router_model,
|
||||
count_capability_routers,
|
||||
gated_capability_of,
|
||||
)
|
||||
from litellm.router_utils.auto_router_tuning_baseline import (
|
||||
is_mutable_tuned_candidate,
|
||||
mutable_tuned_identities,
|
||||
tuning_quota_violation,
|
||||
)
|
||||
from litellm.types.management_endpoints.auto_router_endpoints import (
|
||||
AutoRouterAllowance,
|
||||
AutoRouterAvailabilityResponse,
|
||||
)
|
||||
|
||||
|
||||
class _CatalogModelInfo(BaseModel):
|
||||
team_id: str | None = None
|
||||
|
||||
|
||||
class _CatalogSource(BaseModel):
|
||||
model_id: str
|
||||
created_by: str | None = None
|
||||
litellm_params: Json[dict[str, object]] | dict[str, object]
|
||||
model_info: Json[_CatalogModelInfo] | _CatalogModelInfo | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AutoRouterCatalogEntry:
|
||||
model_id: str
|
||||
team_id: str | None
|
||||
created_by: str | None
|
||||
deployment: Mapping[str, object]
|
||||
|
||||
|
||||
def _catalog_field(value: object, key: str) -> object:
|
||||
if not isinstance(value, str):
|
||||
return deepcopy(value)
|
||||
return decrypt_value_helper(value, key=key, exception_type="debug", return_original_value=True)
|
||||
|
||||
|
||||
def build_auto_router_catalog(rows: Sequence[object]) -> tuple[AutoRouterCatalogEntry, ...] | None:
|
||||
try:
|
||||
sources: Final = TypeAdapter(tuple[_CatalogSource, ...]).validate_python(rows, from_attributes=True)
|
||||
except ValidationError:
|
||||
return None
|
||||
return tuple(
|
||||
AutoRouterCatalogEntry(
|
||||
model_id=row.model_id,
|
||||
team_id=row.model_info.team_id if row.model_info is not None else None,
|
||||
created_by=row.created_by,
|
||||
deployment=MappingProxyType(
|
||||
{
|
||||
"litellm_params": MappingProxyType(
|
||||
{
|
||||
"model": model,
|
||||
"complexity_router_config": _catalog_field(
|
||||
row.litellm_params.get("complexity_router_config"), "complexity_router_config"
|
||||
),
|
||||
}
|
||||
),
|
||||
"model_info": MappingProxyType({"id": row.model_id, "db_model": True}),
|
||||
}
|
||||
),
|
||||
)
|
||||
for row in sources
|
||||
if isinstance(model := _catalog_field(row.litellm_params.get("model"), "model"), str)
|
||||
and classify_strategy_router_model(model) == "complexity"
|
||||
)
|
||||
|
||||
|
||||
def auto_router_availability(
|
||||
*,
|
||||
others: Sequence[Mapping[str, object]],
|
||||
existing: Mapping[str, object] | None,
|
||||
candidate: Mapping[str, object],
|
||||
baselines: Mapping[str, str] | None,
|
||||
limit: int | None,
|
||||
) -> AutoRouterAvailabilityResponse:
|
||||
existing_params: Final = None if existing is None else existing.get("litellm_params")
|
||||
candidate_params: Final = candidate.get("litellm_params")
|
||||
owned: Final = gated_capability_of(existing_params) if isinstance(existing_params, Mapping) else None
|
||||
claimed: Final = gated_capability_of(candidate_params) if isinstance(candidate_params, Mapping) else None
|
||||
counts: Final = tuple(
|
||||
(capability, count_capability_routers(others, capability=capability))
|
||||
for capability in GATED_AUTO_ROUTER_CAPABILITIES
|
||||
)
|
||||
tuned_count: Final = len(mutable_tuned_identities(others, baselines)) if baselines is not None else 0
|
||||
allowances: Final = tuple(
|
||||
AutoRouterAllowance(
|
||||
key=capability.key,
|
||||
limit=limit,
|
||||
remaining=None if limit is None else max(0, limit - held),
|
||||
used_by_this_router=owned is capability,
|
||||
)
|
||||
for capability, held in counts
|
||||
)
|
||||
capability_error: Final = next(
|
||||
(
|
||||
capability_limit_violation(capability=capability, held=held + 1, limit=limit)
|
||||
for capability, held in counts
|
||||
if capability is claimed
|
||||
),
|
||||
None,
|
||||
)
|
||||
tuning_error: Final = (
|
||||
tuning_quota_violation(candidate=candidate, others=others, baselines=baselines, limit=limit)
|
||||
if baselines is not None
|
||||
else None
|
||||
)
|
||||
capability_labels: Final = {
|
||||
"heuristic_v2": "Heuristic v2",
|
||||
"capability": "Capability",
|
||||
"llm_v2": "Fuse v2",
|
||||
"tier_or_classifier_prompt": "Custom tiers or classifier instructions",
|
||||
}
|
||||
return AutoRouterAvailabilityResponse(
|
||||
allowances=(
|
||||
*allowances,
|
||||
AutoRouterAllowance(
|
||||
key="heuristic_tuning",
|
||||
limit=limit,
|
||||
remaining=None if limit is None or baselines is None else max(0, limit - tuned_count),
|
||||
available=limit is None or baselines is not None,
|
||||
used_by_this_router=bool(
|
||||
existing is not None and baselines is not None and is_mutable_tuned_candidate(existing, baselines)
|
||||
),
|
||||
),
|
||||
),
|
||||
error=(
|
||||
f"{capability_labels[claimed.key]} has no available allowance. Choose another option or free an existing allowance."
|
||||
if capability_error is not None and claimed is not None
|
||||
else "These scoring rules need an available Rule-based tuning allowance. Check the weights, thresholds, keywords, and custom dimensions in Advanced settings. Model choices do not use this allowance."
|
||||
if tuning_error is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
|
@ -132,6 +132,7 @@ from litellm.proxy.common_utils.callback_utils import (
|
|||
strip_callback_config,
|
||||
)
|
||||
from litellm.proxy.common_utils.realtime_utils import _realtime_request_body
|
||||
from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog
|
||||
from litellm.router_utils.access_windows import access_windows_config_error
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
get_fallback_errors_from_headers,
|
||||
|
|
@ -5068,6 +5069,7 @@ class ProxyConfig:
|
|||
|
||||
def __init__(self) -> None:
|
||||
self.config: Mapping[str, object] = MappingProxyType({})
|
||||
self.auto_router_db_catalog: tuple[AutoRouterCatalogEntry, ...] | None = None
|
||||
self._last_semantic_filter_config: dict[str, object] | None = None
|
||||
self._last_websearch_interception_config: dict[str, object] | None = None
|
||||
self._last_hashicorp_vault_config: dict[str, object] | None = None
|
||||
|
|
@ -7691,6 +7693,12 @@ class ProxyConfig:
|
|||
def _should_load_db_object(self, object_type: str | SupportedDBObjectType) -> bool:
|
||||
return should_load_db_object(object_type=object_type)
|
||||
|
||||
def remove_auto_router_catalog_entries(self, model_ids: frozenset[str]) -> None:
|
||||
if self.auto_router_db_catalog is not None:
|
||||
self.auto_router_db_catalog = tuple(
|
||||
row for row in self.auto_router_db_catalog if row.model_id not in model_ids
|
||||
)
|
||||
|
||||
async def _get_models_from_db(self, prisma_client: PrismaClient) -> Sequence[_ProxyModelRow] | None:
|
||||
"""
|
||||
Fetch all model deployments from the DB.
|
||||
|
|
@ -7711,6 +7719,7 @@ class ProxyConfig:
|
|||
new_models: Final[Sequence[_ProxyModelRow]] = await ModelRepository(
|
||||
WriterPinnedClient(prisma_client.db)
|
||||
).table.find_many()
|
||||
self.auto_router_db_catalog = build_auto_router_catalog(new_models)
|
||||
return new_models
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception(
|
||||
|
|
|
|||
|
|
@ -1,18 +1,18 @@
|
|||
{
|
||||
"1m_context": {
|
||||
"label": "1M Context",
|
||||
"description": "Routes across models with 1M-token context windows: Luna for simple queries, Terra for medium, Sol for complex, Opus 5 at high thinking for reasoning.",
|
||||
"description": "Routes across models with 1M-token context windows: GPT-6 Luna for simple queries, GPT-5.6 Terra for medium, GPT-6 Sol for complex, Opus 5.5 at high thinking for reasoning.",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": ["gpt-5.6-luna"],
|
||||
"SIMPLE": ["gpt-6-luna"],
|
||||
"MEDIUM": ["gpt-5.6-terra"],
|
||||
"COMPLEX": ["gpt-5.6-sol"],
|
||||
"REASONING": ["claude-opus-5"]
|
||||
"COMPLEX": ["gpt-6-sol"],
|
||||
"REASONING": ["claude-opus-5-5"]
|
||||
},
|
||||
"tier_model_configs": {
|
||||
"REASONING": [
|
||||
{
|
||||
"model_name": "claude-opus-5",
|
||||
"model_name": "claude-opus-5-5",
|
||||
"litellm_params": { "reasoning_effort": "high" }
|
||||
}
|
||||
]
|
||||
|
|
@ -28,12 +28,12 @@
|
|||
},
|
||||
"anthropic_family": {
|
||||
"label": "Anthropic Family",
|
||||
"description": "Routes across the Claude model family: Haiku for simple queries, Sonnet for medium, Opus for complex, Fable 5.1 at high thinking for reasoning.",
|
||||
"description": "Routes across the Claude model family: Haiku for simple queries, Sonnet for medium, Opus 5.5 for complex, Fable 5.1 at high thinking for reasoning.",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": ["claude-haiku-4-5"],
|
||||
"MEDIUM": ["claude-sonnet-5"],
|
||||
"COMPLEX": ["claude-opus-5"],
|
||||
"COMPLEX": ["claude-opus-5-5"],
|
||||
"REASONING": ["claude-fable-5-1"]
|
||||
},
|
||||
"tier_model_configs": {
|
||||
|
|
@ -55,12 +55,12 @@
|
|||
},
|
||||
"gemini_family": {
|
||||
"label": "Gemini Family",
|
||||
"description": "Routes across the Gemini model family: Flash Lite 2.5 for simple queries, Flash Lite 3.1 for medium, Flash 3.7 for complex, Pro 3.1 for reasoning-heavy requests.",
|
||||
"description": "Routes across the Gemini model family: Flash Lite 3.5 for simple queries, Flash 3.8 for medium and complex queries, Pro 3.1 for reasoning-heavy requests.",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": ["gemini-2.5-flash-lite"],
|
||||
"MEDIUM": ["gemini-3.1-flash-lite"],
|
||||
"COMPLEX": ["gemini-3.7-flash"],
|
||||
"SIMPLE": ["gemini-3.5-flash-lite"],
|
||||
"MEDIUM": ["gemini-3.8-flash"],
|
||||
"COMPLEX": ["gemini-3.8-flash"],
|
||||
"REASONING": ["gemini-3.1-pro-preview"]
|
||||
},
|
||||
"classifier_type": "heuristic",
|
||||
|
|
@ -74,18 +74,18 @@
|
|||
},
|
||||
"lite": {
|
||||
"label": "Lite",
|
||||
"description": "Cost-optimized routing across providers: DeepSeek V4 Flash for simple queries, Muse Spark 1.2 at xhigh for medium, Kimi K3 at max for complex, Claude Opus 5 for reasoning. An LLM classifier with the agentic rubric assigns tiers.",
|
||||
"description": "Cost-optimized routing across providers: DeepSeek V4 Flash for simple queries, Muse Spark 1.3 at xhigh for medium, Kimi K3 at max for complex, Claude Opus 5.5 for reasoning. An LLM classifier with the agentic rubric assigns tiers.",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": ["deepseek-v4-flash"],
|
||||
"MEDIUM": ["muse-spark-1.2"],
|
||||
"MEDIUM": ["muse-spark-1.3"],
|
||||
"COMPLEX": ["kimi-k3"],
|
||||
"REASONING": ["claude-opus-5"]
|
||||
"REASONING": ["claude-opus-5-5"]
|
||||
},
|
||||
"tier_model_configs": {
|
||||
"MEDIUM": [
|
||||
{
|
||||
"model_name": "muse-spark-1.2",
|
||||
"model_name": "muse-spark-1.3",
|
||||
"litellm_params": { "reasoning_effort": "xhigh" }
|
||||
}
|
||||
],
|
||||
|
|
@ -113,12 +113,12 @@
|
|||
},
|
||||
"openai_family": {
|
||||
"label": "OpenAI Family",
|
||||
"description": "Routes across the GPT model family: Luna for simple queries, Terra for medium, Sol for complex, Astra at xhigh thinking for reasoning.",
|
||||
"description": "Routes across the GPT model family: GPT-6 Luna for simple queries, GPT-5.6 Terra for medium, GPT-6 Sol for complex, GPT-6 Astra at xhigh thinking for reasoning.",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": ["gpt-5.6-luna"],
|
||||
"SIMPLE": ["gpt-6-luna"],
|
||||
"MEDIUM": ["gpt-5.6-terra"],
|
||||
"COMPLEX": ["gpt-5.6-sol"],
|
||||
"COMPLEX": ["gpt-6-sol"],
|
||||
"REASONING": ["gpt-6-astra"]
|
||||
},
|
||||
"tier_model_configs": {
|
||||
|
|
|
|||
|
|
@ -2323,6 +2323,13 @@ async def calculate_spend(request: SpendCalculateRequest):
|
|||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
if isinstance(e, litellm.exceptions.ModelNotMappedError):
|
||||
raise ProxyException(
|
||||
message=str(e),
|
||||
type="invalid_request_error",
|
||||
param="model",
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
error_msg: Final = f"{e}"
|
||||
raise ProxyException(
|
||||
message=getattr(e, "message", error_msg),
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"version": "2026-09-17-v1",
|
||||
"version": "2026-09-22-v1",
|
||||
"models": [
|
||||
{
|
||||
"id": "gpt-6-astra-v1",
|
||||
|
|
@ -8,6 +8,20 @@
|
|||
"text": "OpenAI model for demanding end-to-end work, including reasoning, coding, research, and document tasks",
|
||||
"sources": ["https://developers.openai.com/api/docs/models/gpt-6-astra"]
|
||||
},
|
||||
{
|
||||
"id": "gpt-6-sol-v1",
|
||||
"label": "GPT-6 Sol",
|
||||
"model": "gpt-6-sol",
|
||||
"text": "OpenAI model for complex coding and agentic workflows, supporting reasoning and tool calling through the Responses API",
|
||||
"sources": ["https://developers.openai.com/api/docs/models/gpt-6-sol"]
|
||||
},
|
||||
{
|
||||
"id": "gpt-6-luna-v1",
|
||||
"label": "GPT-6 Luna",
|
||||
"model": "gpt-6-luna",
|
||||
"text": "OpenAI model for efficient, high-volume workloads, supporting reasoning and tool calling through the Responses API",
|
||||
"sources": ["https://developers.openai.com/api/docs/models/gpt-6-luna"]
|
||||
},
|
||||
{
|
||||
"id": "gpt-5.6-sol-v1",
|
||||
"label": "GPT-5.6 Sol",
|
||||
|
|
@ -63,6 +77,13 @@
|
|||
"model": "claude-fable-5-1",
|
||||
"text": "Anthropic model for demanding reasoning, long-running agentic coding, and multistep research, with always-on adaptive thinking",
|
||||
"sources": ["https://platform.claude.com/docs/en/models/fable-5-1/overview"]
|
||||
},
|
||||
{
|
||||
"id": "claude-opus-5-5-v1",
|
||||
"label": "Claude Opus 5.5",
|
||||
"model": "claude-opus-5-5",
|
||||
"text": "Anthropic model for complex reasoning and agentic work, supporting adaptive thinking and tool use",
|
||||
"sources": ["https://platform.claude.com/docs/en/models/opus-5-5/overview"]
|
||||
}
|
||||
],
|
||||
"harnesses": [
|
||||
|
|
|
|||
|
|
@ -10,12 +10,10 @@ from pydantic import ValidationError
|
|||
|
||||
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig
|
||||
|
||||
TUNING_BASELINE_PARAM_NAME: Final = "auto_router_tuning_baseline_v2"
|
||||
# v2 hashes combine models and scoring rules; a new snapshot is required to separate them.
|
||||
TUNING_BASELINE_PARAM_NAME: Final = "auto_router_tuning_baseline_v3"
|
||||
|
||||
HEURISTIC_V1_TUNING_FIELDS: Final = (
|
||||
"tiers",
|
||||
"tier_model_configs",
|
||||
"classifier_type",
|
||||
"tier_boundaries",
|
||||
"reasoning_override_min_score",
|
||||
"token_thresholds",
|
||||
|
|
@ -49,8 +47,10 @@ def tuning_fingerprint(complexity_router_config: object) -> str | None:
|
|||
validated: Final = ComplexityRouterConfig.model_validate(raw)
|
||||
except ValidationError:
|
||||
return None
|
||||
supplied: Final = ((_TUNING_FIELD_SET - frozenset(("tier_model_configs",))) & frozenset(raw)) | (
|
||||
frozenset(("tier_model_configs",)) if validated.tier_model_configs else frozenset()
|
||||
# The UI always writes this built-in marker. Freeze its spelling so future defaults cannot change recorded hashes.
|
||||
default_escalation: Final = validated.escalation_keywords in (None, ["LITELLM ESCALATE"])
|
||||
supplied: Final = (_TUNING_FIELD_SET & frozenset(raw)) - (
|
||||
frozenset(("escalation_keywords",)) if default_escalation else frozenset()
|
||||
)
|
||||
payload: Final = validated.model_dump(
|
||||
mode="json",
|
||||
|
|
@ -148,9 +148,9 @@ def tuning_limit_violation(*, held: int, limit: int | None) -> str | None:
|
|||
if limit is None or held <= limit:
|
||||
return None
|
||||
return (
|
||||
f"At most {limit} auto-router(s) with changed heuristic scorer settings or tier models can be modified "
|
||||
f"At most {limit} auto-router(s) with changed heuristic scoring rules can be modified "
|
||||
"without an auto-router license. Keep this router on its recorded settings, or revert the other changed "
|
||||
"router to its baseline, or remove one of them."
|
||||
"router to its baseline, or remove one of them. Selecting models does not use this allowance."
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -44,6 +44,25 @@ class ComplexityRouterConfigValidationResponse(BaseModel):
|
|||
error: str | None = None
|
||||
|
||||
|
||||
class AutoRouterAvailabilityRequest(BaseModel):
|
||||
team_id: str | None = None
|
||||
saved_model_id: str | None = None
|
||||
complexity_router_config: Mapping[str, object] | None = None
|
||||
|
||||
|
||||
class AutoRouterAllowance(BaseModel):
|
||||
key: str
|
||||
limit: int | None
|
||||
remaining: int | None
|
||||
used_by_this_router: bool = False
|
||||
available: bool = True
|
||||
|
||||
|
||||
class AutoRouterAvailabilityResponse(BaseModel):
|
||||
allowances: tuple[AutoRouterAllowance, ...]
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class AutoRouterRoutingTestRequest(BaseModel):
|
||||
"""A single request to classify against a complexity-router config that need not be saved yet.
|
||||
|
||||
|
|
|
|||
|
|
@ -3256,6 +3256,7 @@ class StandardLoggingPayloadErrorInformation(TypedDict, total=False):
|
|||
error_budget_entity_id: str | None
|
||||
error_budget_limit: float | None
|
||||
error_budget_spend: float | None
|
||||
normalized_error: ReadOnly[str | None]
|
||||
|
||||
|
||||
class GuardrailMode(TypedDict, total=False):
|
||||
|
|
|
|||
|
|
@ -472,6 +472,7 @@ from .exceptions import (
|
|||
BudgetExceededError,
|
||||
ContentPolicyViolationError,
|
||||
ContextWindowExceededError,
|
||||
ModelNotMappedError,
|
||||
NotFoundError,
|
||||
OpenAIError,
|
||||
PermissionDeniedError,
|
||||
|
|
@ -1439,11 +1440,22 @@ async def async_post_call_success_deployment_hook(
|
|||
modified_response = response
|
||||
|
||||
CustomLogger: Final = _get_cached_custom_logger()
|
||||
CustomGuardrail: Final = _get_cached_custom_guardrail()
|
||||
for callback in litellm.callbacks:
|
||||
if isinstance(callback, CustomLogger):
|
||||
result = await callback.async_post_call_success_deployment_hook(
|
||||
request_data, cast(LLMResponseTypes, modified_response), typed_call_type
|
||||
)
|
||||
try:
|
||||
result = await callback.async_post_call_success_deployment_hook(
|
||||
request_data, cast(LLMResponseTypes, modified_response), typed_call_type
|
||||
)
|
||||
except Exception: # noqa: BLE001 # a broken callback must not fail a completed request
|
||||
if isinstance(callback, CustomGuardrail):
|
||||
raise
|
||||
verbose_logger.exception(
|
||||
"async_post_call_success_deployment_hook error in %s for call_type=%s",
|
||||
type(callback).__name__,
|
||||
typed_call_type,
|
||||
)
|
||||
continue
|
||||
if result is not None:
|
||||
modified_response = result
|
||||
|
||||
|
|
@ -5835,6 +5847,13 @@ def _is_potential_model_name_in_model_cost(
|
|||
_ABOVE_THRESHOLD_COST_KEY: Final = ABOVE_THRESHOLD_COST_KEY_PATTERN
|
||||
|
||||
|
||||
def _model_not_mapped_message(model: str, custom_llm_provider: str | None) -> str:
|
||||
return (
|
||||
f"This model isn't mapped yet. model={model}, custom_llm_provider={custom_llm_provider}. "
|
||||
"Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json."
|
||||
)
|
||||
|
||||
|
||||
def _get_model_info_helper(
|
||||
model: str,
|
||||
custom_llm_provider: str | None = None,
|
||||
|
|
@ -6017,9 +6036,7 @@ def _get_model_info_helper(
|
|||
key, _model_info = generalization
|
||||
|
||||
if _model_info is None or key is None:
|
||||
raise ValueError(
|
||||
"This model isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"
|
||||
)
|
||||
raise ModelNotMappedError(_model_not_mapped_message(model, custom_llm_provider))
|
||||
_input_cost_per_token: float | None = _model_info.get("input_cost_per_token")
|
||||
if _input_cost_per_token is None:
|
||||
# default value to 0, be noisy about this
|
||||
|
|
@ -6254,11 +6271,11 @@ def _get_model_info_helper(
|
|||
if cost_key not in returned_model_info and _ABOVE_THRESHOLD_COST_KEY.search(cost_key) is not None:
|
||||
returned_model_info[cost_key] = cost_value
|
||||
return returned_model_info
|
||||
except ModelNotMappedError:
|
||||
raise
|
||||
except Exception as e:
|
||||
verbose_logger.debug("Error getting model info: %s", e)
|
||||
raise Exception(
|
||||
f"This model isn't mapped yet. model={model}, custom_llm_provider={custom_llm_provider}. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json."
|
||||
)
|
||||
raise Exception(_model_not_mapped_message(model, custom_llm_provider))
|
||||
|
||||
|
||||
def _build_model_info(
|
||||
|
|
|
|||
|
|
@ -25314,8 +25314,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"cache_read_input_token_cost_batches": 1e-07
|
||||
"web_search_billing_unit": "per_query"
|
||||
},
|
||||
"gemini-3-pro-image-preview": {
|
||||
"input_cost_per_image": 0.0011,
|
||||
|
|
@ -25401,8 +25400,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"cache_read_input_token_cost_batches": 2.5e-08
|
||||
"web_search_billing_unit": "per_query"
|
||||
},
|
||||
"gemini-3.1-flash-image-preview": {
|
||||
"input_cost_per_image": 0.00056,
|
||||
|
|
@ -25482,8 +25480,7 @@
|
|||
"supports_response_schema": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_video_input": true,
|
||||
"supports_vision": true,
|
||||
"cache_read_input_token_cost_batches": 1.25e-08
|
||||
"supports_vision": true
|
||||
},
|
||||
"gemini-3.1-flash-lite-preview": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
|
|
@ -25593,8 +25590,7 @@
|
|||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_audio_token_batches": 2.5e-07,
|
||||
"cache_read_input_token_cost_batches": 1.25e-08
|
||||
"input_cost_per_audio_token_batches": 2.5e-07
|
||||
},
|
||||
"gemini-3.5-flash-lite": {
|
||||
"deprecation_date": "2027-07-21",
|
||||
|
|
@ -25652,8 +25648,7 @@
|
|||
"search_context_size_high": 0.014
|
||||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"cache_read_input_token_cost_batches": 1.5e-08
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
"deep-research-pro-preview-12-2025": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
|
|
@ -25689,8 +25684,7 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"cache_read_input_token_cost_batches": 1e-07
|
||||
"supports_web_search": true
|
||||
},
|
||||
"gemini-2.5-flash-lite": {
|
||||
"cache_read_input_audio_token_cost": 3e-08,
|
||||
|
|
@ -26321,8 +26315,7 @@
|
|||
"output_cost_per_token_batches": 4.5e-06,
|
||||
"input_cost_per_token_flex": 7.5e-07,
|
||||
"output_cost_per_token_flex": 4.5e-06,
|
||||
"cache_read_input_token_cost_flex": 7.5e-08,
|
||||
"cache_read_input_token_cost_batches": 7.5e-08
|
||||
"cache_read_input_token_cost_flex": 7.5e-08
|
||||
},
|
||||
"vertex_ai/gemini-3.6-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
|
|
@ -26380,8 +26373,7 @@
|
|||
"search_context_size_high": 0.014
|
||||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
"vertex_ai/gemini-3.7-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
|
|
@ -26440,8 +26432,7 @@
|
|||
"search_context_size_high": 0.014
|
||||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
"vertex_ai/gemini-3.8-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
|
|
@ -26500,8 +26491,7 @@
|
|||
"search_context_size_high": 0.014
|
||||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
"vertex_ai/gemini-3.1-pro-preview": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
|
|
@ -28250,8 +28240,7 @@
|
|||
"output_cost_per_token_batches": 4.5e-06,
|
||||
"input_cost_per_token_flex": 7.5e-07,
|
||||
"output_cost_per_token_flex": 4.5e-06,
|
||||
"cache_read_input_token_cost_flex": 7.5e-08,
|
||||
"cache_read_input_token_cost_batches": 7.5e-08
|
||||
"cache_read_input_token_cost_flex": 7.5e-08
|
||||
},
|
||||
"gemini-3.6-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
|
|
@ -28309,8 +28298,7 @@
|
|||
"search_context_size_high": 0.014
|
||||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
"gemini-3.7-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
|
|
@ -28369,8 +28357,7 @@
|
|||
"search_context_size_high": 0.014
|
||||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
"gemini-3.8-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
|
|
@ -28429,8 +28416,7 @@
|
|||
"search_context_size_high": 0.014
|
||||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
"gemini/gemini-2.5-pro-preview-tts": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -33110,8 +33096,7 @@
|
|||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true,
|
||||
"cache_read_input_token_cost_batches": 6.25e-08
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-chat": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
|
|
@ -33292,8 +33277,7 @@
|
|||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true,
|
||||
"cache_read_input_token_cost_batches": 1.25e-08
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-5-nano": {
|
||||
"cache_read_input_token_cost": 5e-09,
|
||||
|
|
@ -33392,8 +33376,7 @@
|
|||
"supports_web_search": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_xhigh_reasoning_effort": false,
|
||||
"supports_minimal_reasoning_effort": true,
|
||||
"cache_read_input_token_cost_batches": 2.5e-09
|
||||
"supports_minimal_reasoning_effort": true
|
||||
},
|
||||
"gpt-image-1": {
|
||||
"cache_read_input_token_cost": 1.25e-06,
|
||||
|
|
@ -47531,8 +47514,7 @@
|
|||
"output_cost_per_token_flex": 6e-06,
|
||||
"output_cost_per_token_priority": 2.16e-05,
|
||||
"supports_reasoning": false,
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"cache_read_input_token_cost_batches": 1e-07
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing"
|
||||
},
|
||||
"vertex_ai/gemini-3-pro-image-preview": {
|
||||
"input_cost_per_image": 0.0011,
|
||||
|
|
@ -47570,8 +47552,7 @@
|
|||
"output_cost_per_token_batches": 1.5e-06,
|
||||
"output_cost_per_token_flex": 1.5e-06,
|
||||
"supports_reasoning": false,
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"cache_read_input_token_cost_batches": 2.5e-08
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing"
|
||||
},
|
||||
"vertex_ai/gemini-3.1-flash-image-preview": {
|
||||
"input_cost_per_image": 0.00056,
|
||||
|
|
@ -47627,8 +47608,7 @@
|
|||
"supports_response_schema": false,
|
||||
"supports_system_messages": true,
|
||||
"supports_video_input": true,
|
||||
"supports_vision": true,
|
||||
"cache_read_input_token_cost_batches": 1.25e-08
|
||||
"supports_vision": true
|
||||
},
|
||||
"vertex_ai/gemini-3.1-flash-lite-preview": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
|
|
@ -47739,8 +47719,7 @@
|
|||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_audio_token_batches": 2.5e-07,
|
||||
"cache_read_input_token_cost_batches": 1.25e-08
|
||||
"input_cost_per_audio_token_batches": 2.5e-07
|
||||
},
|
||||
"vertex_ai/gemini-3.5-flash-lite": {
|
||||
"deprecation_date": "2027-07-21",
|
||||
|
|
@ -47799,8 +47778,7 @@
|
|||
"search_context_size_high": 0.014
|
||||
},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"cache_read_input_token_cost_batches": 1.5e-08
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
"vertex_ai/deep-research-pro-preview-12-2025": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
|
|
@ -47817,8 +47795,7 @@
|
|||
"output_cost_per_image_token": 0.00012,
|
||||
"output_cost_per_token": 1.2e-05,
|
||||
"output_cost_per_token_batches": 6e-06,
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"cache_read_input_token_cost_batches": 1e-07
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing"
|
||||
},
|
||||
"vertex_ai/jamba-1.5": {
|
||||
"input_cost_per_token": 2e-07,
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family
|
|||
- `router/` - routing and reliability behavior (fallbacks, cooldowns) plus the memory tests (`test_reliability_memory_e2e.py`: every worker's RSS as read at collection time, before any test traffic, must sit under a fixed idle budget, the release-gate check for a DB-backed boot that idles near the pod limit the way v1.100.x did; and a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a fixed budget nor store a request snapshot past a fixed size, the release-gate check for the v1.100.0 retry-breadcrumb leak)
|
||||
- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set and excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic
|
||||
- `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite
|
||||
- `secret_manager/` - the gateway's `key_management_system` against a real secret manager: deployment keys resolved from it (`os.environ/<name>` where the name exists only in the manager) and virtual keys written to and deleted from it. The tests are backend-agnostic and each backend is its own lane, because the setting is global to the proxy: `E2E_SECRET_MANAGER=<system>` opts in and picks the backend from `secret_backends.BACKENDS`, the proxy is booted from `gateway/secret_manager_<system>_ci_config.yml` against the live manager, and the tests reach that manager through the backend's `SecretStore` (`secret_store_<system>.py`). A test needing something not every backend does carries `requires_capability(...)` and is deselected on lanes that lack it. `secret_manager/backend.sh up <system>` runs a backend in Docker and writes the proxy's and the tests' env. Marked `secret_manager`, deselected unless `E2E_SECRET_MANAGER` is set, and kept out of the per-PR selector. Backends today: `hashicorp_vault` and `cyberark` (CyberArk Conjur, which cannot delete, so the delete test is Vault-only)
|
||||
- `gateway/` - proxy configuration only (`litellm-config.yml`); no tests
|
||||
- `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke
|
||||
- `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ A couple of logging destinations are configured on the proxy rather than by the
|
|||
|
||||
### The pull request check
|
||||
|
||||
Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. `logging/test_otel_v2_langfuse_generation_output_e2e.py` is marked `otel_v2` and deselects itself unless `E2E_OTEL_V2` is set, because it needs a gateway booted with `LITELLM_OTEL_V2=true` and Langfuse credentials, neither of which this stack provides, so run it with `E2E_OTEL_V2=1` against a local OTel v2 proxy. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set
|
||||
Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, `load/`, and `secret_manager/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. `logging/test_otel_v2_langfuse_generation_output_e2e.py` is marked `otel_v2` and deselects itself unless `E2E_OTEL_V2` is set, because it needs a gateway booted with `LITELLM_OTEL_V2=true` and Langfuse credentials, neither of which this stack provides, so run it with `E2E_OTEL_V2=1` against a local OTel v2 proxy. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set. The `secret_manager/` lanes each need a proxy configured against their own secret manager (see Secret manager lanes below)
|
||||
|
||||
Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch
|
||||
|
||||
|
|
@ -117,6 +117,31 @@ Fetched values of eight characters or more are masked before use, while shorter
|
|||
|
||||
To reproduce the CI topology on a dedicated machine, `bash .github/e2e-stack/up.sh` reads `tests/e2e/.env`, writes `stack.env` under `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}`, and `bash .github/e2e-stack/down.sh` stops it. Keep this directory private and remove its credential files and logs after use
|
||||
|
||||
### Secret manager lanes
|
||||
|
||||
`key_management_system` is global to the proxy, so the `secret_manager/` tests run once per backend, each against its own proxy. The backends are `hashicorp_vault` and `cyberark` (CyberArk Conjur). `E2E_SECRET_MANAGER` opts in and names the backend (a key of `secret_backends.BACKENDS`). The proxy boots from `gateway/secret_manager_<system>_ci_config.yml`, and the tests reach the same manager through that backend's `SecretStore`. The managers are enterprise features, so the proxy needs a license. `secret_manager/backend.sh` runs any backend in Docker and writes its env, so every lane runs the same way locally:
|
||||
|
||||
```bash
|
||||
bash tests/e2e/secret_manager/backend.sh up cyberark
|
||||
(set -a; . ~/.cache/litellm-e2e-secret-manager/cyberark/proxy.env; set +a; env -u OPENAI_API_KEY LITELLM_LICENSE=... \
|
||||
LITELLM_MASTER_KEY=sk-1234 DATABASE_URL=... uv run litellm --config tests/e2e/gateway/secret_manager_cyberark_ci_config.yml --port 4000)
|
||||
(set -a; . ~/.cache/litellm-e2e-secret-manager/cyberark/tests.env; set +a; OPENAI_API_KEY=... \
|
||||
uv run --group e2e-dev pytest tests/e2e/secret_manager/ -v)
|
||||
bash tests/e2e/secret_manager/backend.sh down cyberark
|
||||
```
|
||||
|
||||
`E2E_SECRET_MANAGER_PORT` moves the manager off its usual port (8200 for Vault, 8080 for Conjur), and `E2E_SECRET_MANAGER_DIR` moves the env files. Keep that directory private, because both files hold a working admin credential. Keep `OPENAI_API_KEY` out of the proxy's environment. The tests copy the runner's key into the manager under a fresh name per test, so a passing call proves the key came through the manager rather than the `os.environ` fallback `get_secret` takes when the manager errors
|
||||
|
||||
A backend declares what it supports in its `SecretBackend.capabilities`, and a test that needs something not every backend does carries `@pytest.mark.requires_capability(...)`, so it is deselected, not failed or skipped, on the lanes that lack it. CyberArk has no `deletes_stored_keys`, because the proxy's delete answers `not_supported` and Conjur keeps the key, so the delete test runs only on the Vault lane
|
||||
|
||||
To add a backend, leave the tests and markers alone and add:
|
||||
|
||||
1. `secret_manager/secret_store_<system>.py`: a `SecretStore` (`write`, `read` returning None when absent, idempotent `destroy`) over the manager's own API through `e2e_http`'s external helpers, read from `E2E_<SYSTEM>_*` env vars, and a `SecretBackend` whose `system` is the litellm `KeyManagementSystem` value and whose `capabilities` lists what it supports
|
||||
2. its entry in `secret_backends.BACKENDS`
|
||||
3. `gateway/secret_manager_<system>_ci_config.yml`, a copy of an existing lane's with only `key_management_system` changed
|
||||
4. an `up_<system>` function in `secret_manager/backend.sh` that starts the manager and writes `proxy.env` and `tests.env`
|
||||
5. a CI step that runs `backend.sh up <system>` (or the same containers as sidecars), boots the proxy with `proxy.env` and a license, and runs pytest with `tests.env`
|
||||
|
||||
### Record and replay
|
||||
|
||||
Record/replay scopes to the proxy's provider-bound traffic only. In `E2E_FIXTURE_MODE=record` the harness boots a local provider-edge server, edge-wired tests register their deployments with an `api_base` pointing at it, and every provider call the proxy makes is forwarded verbatim and written to a fixture bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`). `E2E_FIXTURE_MODE=replay` runs the same tests against the same live proxy and database, but the edge answers the proxy's provider calls from the bundle instead of the provider, so the run makes zero provider calls and spends nothing while key auth, routing, cost calculation, and spend-log writes all still execute for real. Unset (or `live`) behaves exactly as before the knob existed. Both record and replay need the proxy up; only the provider is taken out of the loop
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ failures are hard test failures (see `tests/e2e/AGENTS.md`).
|
|||
| Bedrock | yes (unified only) | yes | yes | yes (unfiltered managed list) | yes (provider-transformed) | S3 (`s3_bucket_name` + `aws_*` + `AWS_BATCH_ROLE_ARN` on model) |
|
||||
| Bedrock GovCloud (`us-gov-west-1`) | yes (unified only) | yes | no | no | yes (provider-transformed) | S3 (`s3_bucket_name` + `aws_*` on model, resolved from `AWS_GOVCLOUD_ACCESS_KEY_ID` / `AWS_GOVCLOUD_SECRET_ACCESS_KEY` / `AWS_GOVCLOUD_BATCH_S3_BUCKET` / `AWS_GOVCLOUD_BATCH_ROLE_ARN`) |
|
||||
| Bedrock split S3 identity | no | no | no | no | yes (file upload, content, delete) | S3 signed with `s3_access_key_id` / `s3_secret_access_key` (`AWS_S3_ONLY_ACCESS_KEY_ID` / `AWS_S3_ONLY_SECRET_ACCESS_KEY`, object rights on `AWS_BATCH_S3_BUCKET` only) while `aws_*` is `AWS_BEDROCK_ONLY_ACCESS_KEY_ID` / `AWS_BEDROCK_ONLY_SECRET_ACCESS_KEY`, an identity with no S3 rights on that bucket |
|
||||
| Bedrock blank S3 env | yes (unified only, on an owned gateway exporting `AWS_S3_ENCRYPTION_KEY_ID` / `AWS_S3_BUCKET_OWNER` as empty strings) | no | no | no | no | S3 (`s3_bucket_name` + `aws_*` + `AWS_BATCH_ROLE_ARN` in the gateway config); blank env vars must be treated as unset, not serialized |
|
||||
|
||||
Bedrock cancel maps to `StopModelInvocationJob` and comes back `cancelling`; the
|
||||
lifecycle asserts it the same way it does for OpenAI (`_CANCEL_ASSERTED_PROVIDERS`).
|
||||
|
|
|
|||
145
tests/e2e/batches/bedrock_env_gateway.py
Normal file
145
tests/e2e/batches/bedrock_env_gateway.py
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
"""An owned, source-built proxy whose process env exports AWS_S3_* vars blank.
|
||||
|
||||
The shared fixture proxy inherits the harness env, which cannot reproduce a user
|
||||
shell that exports AWS_S3_ENCRYPTION_KEY_ID / AWS_S3_BUCKET_OWNER as empty
|
||||
strings. This gateway boots a second proxy with both vars present but blank, so
|
||||
a batch create through it proves blank means unset, not an empty string.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import NoBody
|
||||
from idp import stop_process_group
|
||||
from proxy_client import ProxyClient, build_proxy_client
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
STARTUP_TIMEOUT_SECONDS: Final = 240
|
||||
LOG_TAIL_BYTES: Final = 4000
|
||||
REPO_ROOT: Final = Path(__file__).resolve().parents[3]
|
||||
|
||||
_CONFIG_YAML: Final = """model_list:
|
||||
- model_name: bedrock-blank-s3-batch
|
||||
litellm_params:
|
||||
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
|
||||
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
|
||||
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
|
||||
aws_region_name: os.environ/AWS_REGION
|
||||
s3_region_name: os.environ/AWS_REGION
|
||||
s3_bucket_name: os.environ/AWS_BATCH_S3_BUCKET
|
||||
s3_access_key_id: os.environ/AWS_ACCESS_KEY_ID
|
||||
s3_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
|
||||
aws_batch_role_arn: os.environ/AWS_BATCH_ROLE_ARN
|
||||
|
||||
general_settings:
|
||||
master_key: os.environ/LITELLM_MASTER_KEY
|
||||
database_url: os.environ/DATABASE_URL
|
||||
"""
|
||||
|
||||
|
||||
def available_port() -> int:
|
||||
with socket.socket() as listener:
|
||||
listener.bind(("127.0.0.1", 0))
|
||||
return TypeAdapter(tuple[str, int]).validate_python(listener.getsockname())[1]
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class BedrockEnvGateway:
|
||||
base_url: str
|
||||
master_key: str
|
||||
proxy: ProxyClient
|
||||
_environment: Mapping[str, str] = field(repr=False)
|
||||
_command: tuple[str, ...] = field(repr=False)
|
||||
_log_path: Path
|
||||
_child: subprocess.Popen[bytes] | None = field(default=None, init=False, repr=False)
|
||||
|
||||
@classmethod
|
||||
def start(cls) -> BedrockEnvGateway:
|
||||
assert os.environ.get("DATABASE_URL"), "DATABASE_URL is required for the blank-S3-env gateway"
|
||||
port: Final = available_port()
|
||||
base_url: Final = f"http://127.0.0.1:{port}"
|
||||
master_key: Final = f"sk-e2e-blank-s3-{unique_marker()}"
|
||||
directory: Final = Path(tempfile.mkdtemp(prefix="litellm-e2e-blank-s3-"))
|
||||
config: Final = directory / "blank-s3-gateway.yaml"
|
||||
config.write_text(_CONFIG_YAML)
|
||||
environment: Final = {
|
||||
**{key: value for key, value in os.environ.items() if not key.startswith("REDIS_")},
|
||||
"DATABASE_URL": os.environ["DATABASE_URL"],
|
||||
"LITELLM_MASTER_KEY": master_key,
|
||||
"STORE_MODEL_IN_DB": "False",
|
||||
"PYTHONPATH": str(REPO_ROOT),
|
||||
"AWS_S3_ENCRYPTION_KEY_ID": "",
|
||||
"AWS_S3_BUCKET_OWNER": "",
|
||||
}
|
||||
gateway: Final = cls(
|
||||
base_url=base_url,
|
||||
master_key=master_key,
|
||||
proxy=build_proxy_client(
|
||||
base_url=base_url,
|
||||
control_plane_base_url=base_url,
|
||||
replica_urls=(base_url,),
|
||||
master_key=master_key,
|
||||
),
|
||||
_environment=environment,
|
||||
_command=(
|
||||
sys.executable,
|
||||
"-m",
|
||||
"litellm.proxy.proxy_cli",
|
||||
"--config",
|
||||
str(config),
|
||||
"--port",
|
||||
str(port),
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
),
|
||||
_log_path=directory / "blank-s3-gateway.log",
|
||||
)
|
||||
with gateway._log_path.open("ab") as log:
|
||||
gateway._child = subprocess.Popen(
|
||||
gateway._command,
|
||||
env=dict(gateway._environment),
|
||||
stdout=log,
|
||||
stderr=log,
|
||||
start_new_session=True,
|
||||
cwd=REPO_ROOT,
|
||||
)
|
||||
deadline: Final = time.monotonic() + STARTUP_TIMEOUT_SECONDS
|
||||
while time.monotonic() < deadline:
|
||||
assert gateway._child.poll() is None, (
|
||||
f"blank-S3-env gateway exited early; log tail:\n{gateway.log_tail()}"
|
||||
)
|
||||
result = gateway.proxy.transport.probe("/health/liveliness", params=NoBody())
|
||||
if result.status_code == 200:
|
||||
return gateway
|
||||
time.sleep(0.5)
|
||||
tail: Final = gateway.log_tail()
|
||||
gateway.stop()
|
||||
raise AssertionError(
|
||||
f"blank-S3-env gateway did not become ready in {STARTUP_TIMEOUT_SECONDS}s; log tail:\n{tail}"
|
||||
)
|
||||
|
||||
def log_tail(self) -> str:
|
||||
if not self._log_path.exists():
|
||||
return "<no log written>"
|
||||
with self._log_path.open("rb") as log:
|
||||
log.seek(0, 2)
|
||||
size: Final = log.tell()
|
||||
log.seek(max(0, size - LOG_TAIL_BYTES))
|
||||
return log.read().decode("utf-8", errors="replace")
|
||||
|
||||
def stop(self) -> None:
|
||||
if self._child is not None:
|
||||
stop_process_group(self._child)
|
||||
shutil.rmtree(self._log_path.parent, ignore_errors=True)
|
||||
109
tests/e2e/batches/test_bedrock_blank_s3_env_e2e.py
Normal file
109
tests/e2e/batches/test_bedrock_blank_s3_env_e2e.py
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
"""Live e2e pin for Bedrock batch create with blank AWS_S3_* env vars.
|
||||
|
||||
Owns its own file (not test_batches_e2e.py) so the PR changed-file e2e gate
|
||||
stays a single tiny file: this class boots its own gateway with
|
||||
AWS_S3_ENCRYPTION_KEY_ID and AWS_S3_BUCKET_OWNER exported empty, then runs the
|
||||
unified target_model_names upload + batch create lifecycle against real Bedrock.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from batch_cleanup import cleanup_batch, cleanup_file
|
||||
from batch_client import BatchClient, BatchCreateBody, BatchObject, FileObject
|
||||
from bedrock_env_gateway import BedrockEnvGateway
|
||||
from capabilities import is_managed_id
|
||||
from e2e_http import FileUploadForm, require_successful_call, unwrap
|
||||
from lifecycle import ResourceManager
|
||||
from models import KeyGenerateBody
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
CREATED_BATCH_STATUSES = {"validating", "in_progress", "finalizing"}
|
||||
BLANK_S3_RAW_MODEL: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
|
||||
|
||||
|
||||
def render_jsonl(model: str) -> bytes:
|
||||
line = {
|
||||
"custom_id": "req-1",
|
||||
"method": "POST",
|
||||
"url": "/v1/chat/completions",
|
||||
"body": {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "ping"}],
|
||||
"max_tokens": 8,
|
||||
},
|
||||
}
|
||||
return (json.dumps(line) + "\n").encode()
|
||||
|
||||
|
||||
def assert_file_object(file: FileObject, *, provider: str) -> None:
|
||||
assert file.object == "file", f"file.object={file.object!r}"
|
||||
assert file.purpose == "batch", f"file.purpose={file.purpose!r}"
|
||||
assert file.bytes is not None, f"file.bytes={file.bytes!r}"
|
||||
if provider != "bedrock":
|
||||
assert file.bytes > 0, f"file.bytes={file.bytes!r}"
|
||||
assert file.status, "file.status missing"
|
||||
assert file.created_at is not None and file.created_at > 0, "file.created_at missing"
|
||||
|
||||
|
||||
def assert_batch_object(batch: BatchObject) -> None:
|
||||
assert batch.object == "batch", f"batch.object={batch.object!r}"
|
||||
if batch.endpoint:
|
||||
assert batch.endpoint == "/v1/chat/completions", f"batch.endpoint={batch.endpoint!r}"
|
||||
assert batch.completion_window == "24h", f"window={batch.completion_window!r}"
|
||||
assert batch.input_file_id, "batch.input_file_id missing"
|
||||
assert batch.created_at is not None and batch.created_at > 0, "batch.created_at missing"
|
||||
|
||||
|
||||
class TestBedrockBatchBlankS3EnvVars:
|
||||
"""Bedrock batch create with AWS_S3_* env vars exported but blank.
|
||||
|
||||
Regression: a blank AWS_S3_ENCRYPTION_KEY_ID or AWS_S3_BUCKET_OWNER env var
|
||||
resolved to "" and was serialized into the create-job request, which Bedrock
|
||||
rejects. The owned gateway exports both vars empty, so the unified lifecycle
|
||||
only passes when blank is treated as unset.
|
||||
"""
|
||||
|
||||
@pytest.mark.covers(
|
||||
"llm.batches.bedrock.blank_s3_env.nonstream.works",
|
||||
"llm.files.bedrock.upload.nonstream.works",
|
||||
exercised_on=["batches", "files"],
|
||||
)
|
||||
def test_unified_batch_create_ignores_blank_s3_env_vars(self, resources: ResourceManager) -> None:
|
||||
gateway: Final = BedrockEnvGateway.start()
|
||||
resources.defer(gateway.stop)
|
||||
client: Final = BatchClient(proxy=gateway.proxy)
|
||||
|
||||
key: Final = client.proxy.generate_key(KeyGenerateBody(models=[], user_id="e2e-test-user"))
|
||||
resources.defer(lambda: client.proxy.delete_key(key))
|
||||
|
||||
file: Final = unwrap(
|
||||
client.upload_file(
|
||||
content=render_jsonl(BLANK_S3_RAW_MODEL),
|
||||
form=FileUploadForm(purpose="batch", target_model_names="bedrock-blank-s3-batch"),
|
||||
key=key,
|
||||
)
|
||||
)
|
||||
resources.defer(lambda: cleanup_file(client, file.id, key=key))
|
||||
assert_file_object(file, provider="bedrock")
|
||||
|
||||
created: Final = client.create_batch(body=BatchCreateBody(input_file_id=file.id), key=key)
|
||||
assert created.status_code < 400, (
|
||||
f"blank AWS_S3_ENCRYPTION_KEY_ID / AWS_S3_BUCKET_OWNER must be treated as "
|
||||
f"unset; Bedrock rejected the job: {created.body[:400]}"
|
||||
)
|
||||
require_successful_call(created)
|
||||
batch: Final = BatchObject.model_validate_json(created.body)
|
||||
resources.defer(lambda: cleanup_batch(client, batch.id, key=key))
|
||||
|
||||
assert is_managed_id(batch.id), (
|
||||
f"blank-S3-env create via target_model_names must return a managed batch id, got {batch.id!r}"
|
||||
)
|
||||
assert batch.status in CREATED_BATCH_STATUSES, (
|
||||
f"blank-S3-env batch has non-transitional status {batch.status!r}"
|
||||
)
|
||||
assert_batch_object(batch)
|
||||
|
|
@ -36,6 +36,7 @@ from e2e_config import (
|
|||
PROVIDER_EDGE_HOST_OPT_IN_ENV,
|
||||
PROXY_BASE_URL,
|
||||
REDIS_CHAOS_OPT_IN_ENV,
|
||||
SECRET_MANAGER_OPT_IN_ENV,
|
||||
WEEKLY_ANOMALY_OPT_IN_ENV,
|
||||
unique_marker,
|
||||
)
|
||||
|
|
@ -70,6 +71,7 @@ OPT_IN_MARKERS: Final = MappingProxyType(
|
|||
"provider_edge_host": PROVIDER_EDGE_HOST_OPT_IN_ENV,
|
||||
"otel_v2": OTEL_V2_OPT_IN_ENV,
|
||||
"otel_tls": OTEL_TLS_OPT_IN_ENV,
|
||||
"secret_manager": SECRET_MANAGER_OPT_IN_ENV,
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -172,6 +174,11 @@ def pytest_configure(config: pytest.Config) -> None:
|
|||
"markers",
|
||||
"otel_tls: needs a stack whose gateway exports OTLP over TLS signed by the CA in SSL_CERT_FILE; deselected unless E2E_OTEL_EXPORTER_ENDPOINT is set",
|
||||
)
|
||||
config.addinivalue_line(
|
||||
"markers",
|
||||
"secret_manager: needs a proxy booted from gateway/secret_manager_<system>_ci_config.yml against that live "
|
||||
"secret manager; deselected unless E2E_SECRET_MANAGER names the backend (see secret_manager/secret_backends.py)",
|
||||
)
|
||||
|
||||
|
||||
def pytest_sessionstart(session: pytest.Session) -> None:
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@
|
|||
- {id: llm.batches.bedrock.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:98", rationale: "Bedrock batches (encoded/unified only)"}
|
||||
- {id: llm.batches.bedrock.assume_role.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: assume_role, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Bedrock batch create under STS assume-role credentials"}
|
||||
- {id: llm.batches.bedrock.govcloud_partition.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: govcloud_partition, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Bedrock batch create in the us-gov-west-1 partition"}
|
||||
- {id: llm.batches.bedrock.blank_s3_env.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: blank_s3_env, streaming: nonstream, assertions: [works], source: "test_bedrock_blank_s3_env_e2e.py", rationale: "Bedrock batch create treats blank AWS_S3_ENCRYPTION_KEY_ID / AWS_S3_BUCKET_OWNER env vars as unset instead of serializing empty strings"}
|
||||
- {id: llm.batches.bedrock.cancel.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Bedrock batch cancel (StopModelInvocationJob) returns the same id with a cancelling/cancelled status"}
|
||||
- {id: llm.batches.bedrock.list.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "A Bedrock managed batch is present in the GET /v1/batches list envelope"}
|
||||
- {id: llm.batches.hosted_vllm.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: batches, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "hosted_vllm OpenAI-compatible batch create"}
|
||||
|
|
|
|||
|
|
@ -40,7 +40,10 @@
|
|||
- {id: other.config.runtime_update.applies_at_runtime, module: other, tier: P0, area: config, assertions: [applies_at_runtime], source: "proxy_server.py:14014-14060", rationale: "/config/update persists to DB + invalidates cache"}
|
||||
- {id: other.config.passthrough.headers_forwarded, module: other, tier: P0, area: config, assertions: [headers_forwarded], source: "passthrough/utils.py forward_headers_from_request", rationale: "Custom pass-through static headers and x-pass-* client headers reach the upstream"}
|
||||
- {id: other.config.general_settings.alert_webhook_side_effect, module: other, tier: P1, area: config, assertions: [alert_webhook_side_effect], source: "proxy_server.py:14215", rationale: "alert_to_webhook_url auto-enables slack alerting"}
|
||||
- {id: other.config.secret_resolution.kms_integration, module: other, tier: P1, area: config, assertions: [kms_integration], source: "proxy_server.py:3984-4010", rationale: "Resolves secrets from Vault/KMS at startup"}
|
||||
- {id: other.config.secret_resolution.kms_integration, module: other, tier: P1, area: config, assertions: [kms_integration], source: "secret_managers/main.py get_secret / secret_manager/test_secret_manager_e2e.py", rationale: "A deployment whose api_key is os.environ/<name> gets its key from the configured secret manager when that name exists only in the manager"}
|
||||
- {id: other.config.secret_resolution.manager_value_used, module: other, tier: P1, area: config, assertions: [manager_value_used], source: "secret_managers/main.py get_secret / secret_manager/test_secret_manager_e2e.py", rationale: "The value the manager holds is what reaches the provider: a bogus key in the manager is rejected by the provider with 401, so a passing resolution test cannot be an env fallback"}
|
||||
- {id: other.config.secret_manager.virtual_key_stored, module: other, tier: P1, area: config, assertions: [virtual_key_stored], source: "key_management_event_hooks.py _store_virtual_key_in_secret_manager", rationale: "With store_virtual_keys, /key/generate writes the new key under prefix_for_stored_virtual_keys + key_alias in the manager"}
|
||||
- {id: other.config.secret_manager.virtual_key_deleted, module: other, tier: P1, area: config, assertions: [virtual_key_deleted], source: "key_management_event_hooks.py _delete_virtual_keys_from_secret_manager", rationale: "/key/delete removes the stored key from the manager, so a revoked key does not linger there"}
|
||||
- {id: other.config.overrides.audit_logged, module: other, tier: P1, area: config, assertions: [audit_logged], source: "config_override_endpoints.py:67-100", rationale: "Config override mutations audit-logged, values redacted"}
|
||||
- {id: other.key_mgmt.regenerate.grace_period_honored, module: other, tier: P1, area: auth, assertions: [grace_period_honored], source: "key_management_endpoints.py:4503-4560", rationale: "Old key valid during grace_period then revoked"}
|
||||
- {id: other.key_mgmt.spend_reset.resets_to_value, module: other, tier: P1, area: auth, assertions: [resets_to_value], source: "key_management_endpoints.py:4841", rationale: "reset_spend resets accumulated spend"}
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@
|
|||
- {id: quota_management.spend_tracking.end_user.attributes_spend, module: quota_management, tier: P1, behavior: spend_tracking, variant: end_user, assertions: [attributes_spend], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "user= attribution lands the end-user id on the spend row"}
|
||||
- {id: quota_management.spend_tracking.per_model.writes_own_rows, module: quota_management, tier: P2, behavior: spend_tracking, variant: per_model, assertions: [writes_own_rows], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Each model on a shared key gets its own spend row"}
|
||||
- {id: quota_management.spend_tracking.failure.writes_failure_row, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [writes_failure_row], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_log_error_logger.py", rationale: "A failed call writes a failure-status spend row"}
|
||||
- {id: quota_management.spend_tracking.failure.writes_normalized_error, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [writes_normalized_error], exercised_on: [chat_completions], source: "litellm_core_utils/error_normalization.py", rationale: "Failure rows carry a stable metadata.error_information.normalized_error key next to the unchanged error_message, so two upstream auth failures with different provider wording share one cluster key a dashboard can group by"}
|
||||
- {id: quota_management.spend_tracking.failure.attributes_provider, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [attributes_provider], exercised_on: [chat_completions], source: "proxy/utils.py", rationale: "A request rejected in pre_call_hook (rate limit, guardrail) still lands its single deployment's provider and model_id on the failure spend row"}
|
||||
- {id: quota_management.spend_tracking.spend_calculate.returns_cost, module: quota_management, tier: P2, behavior: spend_tracking, variant: spend_calculate, assertions: [returns_cost], exercised_on: [spend_calculate], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "/spend/calculate prices a hypothetical request at nonzero cost"}
|
||||
- {id: quota_management.spend_tracking.pagination.keeps_total, module: quota_management, tier: P2, behavior: spend_tracking, variant: pagination, assertions: [keeps_total], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "Spend-logs v2 pagination caps page size without losing the total"}
|
||||
|
|
|
|||
|
|
@ -65,6 +65,7 @@ LlmCapability = Literal[
|
|||
"assume_role",
|
||||
"basic",
|
||||
"batch_deployment",
|
||||
"blank_s3_env",
|
||||
"count_tokens",
|
||||
"govcloud_partition",
|
||||
"split_s3_credentials",
|
||||
|
|
|
|||
|
|
@ -150,6 +150,7 @@ MCP_OAUTH_LIVE_OPT_IN_ENV: Final = "E2E_MCP_OAUTH_LIVE"
|
|||
PROVIDER_EDGE_HOST_OPT_IN_ENV: Final = "E2E_PROVIDER_EDGE_HOST_REACHABLE"
|
||||
OTEL_V2_OPT_IN_ENV: Final = "E2E_OTEL_V2"
|
||||
OTEL_TLS_OPT_IN_ENV: Final = "E2E_OTEL_EXPORTER_ENDPOINT"
|
||||
SECRET_MANAGER_OPT_IN_ENV: Final = "E2E_SECRET_MANAGER"
|
||||
ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6"))
|
||||
ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6"))
|
||||
ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3"))
|
||||
|
|
|
|||
|
|
@ -129,9 +129,9 @@ class ProbeResult(BaseModel):
|
|||
|
||||
|
||||
class ExternalWrite(BaseModel):
|
||||
"""Outcome of a write to a non-proxy API (an identity provider's admin API)
|
||||
that answers with a status and, on create, a Location header naming the new
|
||||
resource rather than a JSON body."""
|
||||
"""Outcome of a call to a non-proxy API (an identity provider's admin API, a
|
||||
secret manager) that answers with a status, on create a Location header naming
|
||||
the new resource, and a body kept as text rather than parsed as JSON."""
|
||||
|
||||
status_code: int
|
||||
location: str = ""
|
||||
|
|
@ -491,6 +491,30 @@ def post_json_external(
|
|||
)
|
||||
|
||||
|
||||
def send_text_external(
|
||||
method: Literal["GET", "POST", "PATCH"],
|
||||
url: str,
|
||||
*,
|
||||
headers: BaseModel,
|
||||
content: str | None = None,
|
||||
timeout: float = 30.0,
|
||||
) -> ExternalWrite:
|
||||
"""Send an absolute URL outside the proxy a raw text body (or none) and keep the
|
||||
answer as text, for an API that takes and returns neither JSON nor forms: CyberArk
|
||||
Conjur takes a secret value or a YAML policy and returns a secret as its raw value."""
|
||||
try:
|
||||
resp = requests.request(
|
||||
method,
|
||||
url,
|
||||
headers=_headers(headers),
|
||||
data=content.encode() if content is not None else None,
|
||||
timeout=timeout,
|
||||
)
|
||||
except requests.RequestException as exc:
|
||||
return ExternalWrite(status_code=-1, body=str(exc))
|
||||
return ExternalWrite(status_code=resp.status_code, body=resp.text)
|
||||
|
||||
|
||||
def delete_external(url: str, *, headers: BaseModel, timeout: float = 30.0) -> ExternalWrite:
|
||||
try:
|
||||
resp = requests.delete(url, headers=_headers(headers), timeout=timeout)
|
||||
|
|
|
|||
8
tests/e2e/gateway/secret_manager_cyberark_ci_config.yml
Normal file
8
tests/e2e/gateway/secret_manager_cyberark_ci_config.yml
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
general_settings:
|
||||
master_key: os.environ/LITELLM_MASTER_KEY
|
||||
store_model_in_db: true
|
||||
key_management_system: cyberark
|
||||
key_management_settings:
|
||||
access_mode: read_and_write
|
||||
store_virtual_keys: true
|
||||
prefix_for_stored_virtual_keys: litellm-e2e/virtual-keys/
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
general_settings:
|
||||
master_key: os.environ/LITELLM_MASTER_KEY
|
||||
store_model_in_db: true
|
||||
key_management_system: hashicorp_vault
|
||||
key_management_settings:
|
||||
access_mode: read_and_write
|
||||
store_virtual_keys: true
|
||||
prefix_for_stored_virtual_keys: litellm-e2e/virtual-keys/
|
||||
|
|
@ -937,10 +937,18 @@ class GuardrailRunRecord(BaseModel):
|
|||
guardrail_response: object | None = None
|
||||
|
||||
|
||||
class SpendLogErrorInformation(BaseModel):
|
||||
error_code: str | None = None
|
||||
error_class: str | None = None
|
||||
error_message: str | None = None
|
||||
normalized_error: str | None = None
|
||||
|
||||
|
||||
class SpendLogMetadata(BaseModel):
|
||||
user_api_key_alias: str | None = None
|
||||
applied_guardrails: list[str] | None = None
|
||||
guardrail_information: list[GuardrailRunRecord] | None = None
|
||||
error_information: SpendLogErrorInformation | None = None
|
||||
|
||||
|
||||
class SpendLogRow(BaseModel):
|
||||
|
|
|
|||
|
|
@ -17,3 +17,4 @@ markers =
|
|||
provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set
|
||||
otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set
|
||||
otel_tls: needs a stack whose gateway exports OTLP over TLS signed by the CA in SSL_CERT_FILE; deselected unless E2E_OTEL_EXPORTER_ENDPOINT is set
|
||||
secret_manager: needs a proxy booted from gateway/secret_manager_<system>_ci_config.yml against that live secret manager; deselected unless E2E_SECRET_MANAGER names the backend (see secret_manager/secret_backends.py)
|
||||
|
|
|
|||
|
|
@ -499,6 +499,47 @@ def test_failure_call_writes_failure_status_row(
|
|||
assert (failure_row.spend or 0) == 0.0, "failed call must not be charged"
|
||||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.failure.writes_normalized_error")
|
||||
def test_failure_rows_share_normalized_error_across_provider_wording(
|
||||
client: SpendClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
"""Two upstream auth failures with different provider wording land as failure rows
|
||||
whose metadata.error_information keeps each provider's own error_message and
|
||||
carries the same stable normalized_error cluster key."""
|
||||
marker = unique_marker()
|
||||
deployments: Final = (
|
||||
(f"e2e-norm-openai-{marker}", "openai/gpt-5.5"),
|
||||
(f"e2e-norm-anthropic-{marker}", "anthropic/claude-haiku-4-5"),
|
||||
)
|
||||
for name, provider_model in deployments:
|
||||
model_id = client.proxy.create_model(
|
||||
name, LiteLLMParamsBody(model=provider_model, api_key=f"sk-invalid-{marker}")
|
||||
)
|
||||
resources.defer(lambda model_id=model_id: client.proxy.delete_model(model_id))
|
||||
result = client.chat(scoped_key, name, f"normalize failure {marker}", max_tokens=1)
|
||||
assert not is_ok(result), f"{name}: invalid upstream key must fail the call, got {result}"
|
||||
|
||||
rows = client.poll_logs_for_key(
|
||||
scoped_key,
|
||||
min_rows=2,
|
||||
predicate=lambda rs: sum(1 for r in rs if r.status == "failure") >= 2,
|
||||
)
|
||||
failure_rows = [r for r in rows if r.status == "failure"]
|
||||
assert len(failure_rows) == 2, f"expected one failure row per deployment: {_summarize(rows)}"
|
||||
|
||||
infos = [r.metadata.error_information if r.metadata else None for r in failure_rows]
|
||||
assert all(info is not None for info in infos), (
|
||||
f"failure rows must carry metadata.error_information: {[r.model_dump() for r in failure_rows]}"
|
||||
)
|
||||
messages = {info.error_message for info in infos if info is not None}
|
||||
assert len(messages) == 2, f"provider wording must stay distinct in error_message: {messages}"
|
||||
normalized = {info.normalized_error for info in infos if info is not None}
|
||||
assert normalized == {"401_AUTHENTICATION_FAILED"}, (
|
||||
f"both auth failures must share one normalized_error cluster key; saw {normalized} "
|
||||
f"for messages {messages}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.failure.attributes_provider")
|
||||
def test_pre_call_rejection_row_attributes_provider_and_model_id(
|
||||
client: SpendClient, resources: ResourceManager
|
||||
|
|
|
|||
76
tests/e2e/secret_manager/backend.sh
Executable file
76
tests/e2e/secret_manager/backend.sh
Executable file
|
|
@ -0,0 +1,76 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
usage() {
|
||||
local systems
|
||||
systems=$(declare -F | sed -n 's/^declare -f up_//p' | paste -sd '|' -)
|
||||
echo "usage: $0 up|down $systems" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
action=${1:-}
|
||||
system=${2:-}
|
||||
dir=${E2E_SECRET_MANAGER_DIR:-$HOME/.cache/litellm-e2e-secret-manager}/$system
|
||||
name=litellm-e2e-$system
|
||||
|
||||
wait_for() {
|
||||
local url=$1
|
||||
for _ in $(seq 1 90); do
|
||||
if curl -sf -o /dev/null "$url"; then
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "$system did not answer at $url" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
down() {
|
||||
docker rm -f "$name" "$name-db" >/dev/null 2>&1 || true
|
||||
docker network rm "$name" >/dev/null 2>&1 || true
|
||||
rm -rf "$dir"
|
||||
}
|
||||
|
||||
up_hashicorp_vault() {
|
||||
local port=${E2E_SECRET_MANAGER_PORT:-8200}
|
||||
local token
|
||||
token=e2e-$(openssl rand -hex 16)
|
||||
docker run -d --name "$name" -p "127.0.0.1:$port:8200" --cap-add IPC_LOCK \
|
||||
-e VAULT_DEV_ROOT_TOKEN_ID="$token" hashicorp/vault:1.20 >/dev/null
|
||||
wait_for "http://127.0.0.1:$port/v1/sys/health"
|
||||
printf 'HCP_VAULT_ADDR=http://127.0.0.1:%s\nHCP_VAULT_TOKEN=%s\n' "$port" "$token" >"$dir/proxy.env"
|
||||
printf 'E2E_VAULT_ADDR=http://127.0.0.1:%s\nE2E_VAULT_TOKEN=%s\n' "$port" "$token" >"$dir/tests.env"
|
||||
}
|
||||
|
||||
up_cyberark() {
|
||||
local port=${E2E_SECRET_MANAGER_PORT:-8080}
|
||||
local data_key api_key
|
||||
docker network create "$name" >/dev/null
|
||||
docker run -d --name "$name-db" --network "$name" -e POSTGRES_HOST_AUTH_METHOD=trust postgres:15 >/dev/null
|
||||
data_key=$(docker run --rm cyberark/conjur:1.24 data-key generate)
|
||||
docker run -d --name "$name" --network "$name" -p "127.0.0.1:$port:80" \
|
||||
-e DATABASE_URL="postgres://postgres@$name-db/postgres" -e CONJUR_DATA_KEY="$data_key" \
|
||||
-e CONJUR_AUTHENTICATORS=authn cyberark/conjur:1.24 server >/dev/null
|
||||
wait_for "http://127.0.0.1:$port/"
|
||||
docker exec "$name" conjurctl account create --name default >/dev/null
|
||||
api_key=$(docker exec "$name" conjurctl role retrieve-key default:user:admin | tr -d '\r\n')
|
||||
printf 'CYBERARK_API_BASE=http://127.0.0.1:%s\nCYBERARK_ACCOUNT=default\nCYBERARK_USERNAME=admin\nCYBERARK_API_KEY=%s\n' \
|
||||
"$port" "$api_key" >"$dir/proxy.env"
|
||||
printf 'E2E_CYBERARK_API_BASE=http://127.0.0.1:%s\nE2E_CYBERARK_ACCOUNT=default\nE2E_CYBERARK_USERNAME=admin\nE2E_CYBERARK_API_KEY=%s\n' \
|
||||
"$port" "$api_key" >"$dir/tests.env"
|
||||
}
|
||||
|
||||
[[ $# -eq 2 && -n $system ]] && declare -F "up_$system" >/dev/null || usage
|
||||
|
||||
case $action in
|
||||
up)
|
||||
down
|
||||
mkdir -p "$dir"
|
||||
"up_$system"
|
||||
echo "E2E_SECRET_MANAGER=$system" >>"$dir/tests.env"
|
||||
echo "$system is up; env in $dir/proxy.env (proxy) and $dir/tests.env (pytest)"
|
||||
;;
|
||||
down) down ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
57
tests/e2e/secret_manager/conftest.py
Normal file
57
tests/e2e/secret_manager/conftest.py
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from e2e_config import SECRET_MANAGER_OPT_IN_ENV
|
||||
from proxy_client import ProxyClient
|
||||
from secret_backends import BACKENDS, selected_backend
|
||||
from secret_store import SecretBackend, SecretStore
|
||||
|
||||
REQUIRES_CAPABILITY: Final = "requires_capability"
|
||||
|
||||
|
||||
def pytest_configure(config: pytest.Config) -> None:
|
||||
config.addinivalue_line(
|
||||
"markers",
|
||||
f"{REQUIRES_CAPABILITY}(capability): secret_manager test deselected when the backend "
|
||||
f"{SECRET_MANAGER_OPT_IN_ENV} names lacks the capability (secret_store.Capability)",
|
||||
)
|
||||
|
||||
|
||||
def _lacks_capability(item: pytest.Item, backend: SecretBackend) -> bool:
|
||||
marker: Final = item.get_closest_marker(REQUIRES_CAPABILITY)
|
||||
return marker is not None and marker.args[0] not in backend.capabilities
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
|
||||
backend: Final = BACKENDS.get(os.environ.get(SECRET_MANAGER_OPT_IN_ENV, "").strip())
|
||||
if backend is None:
|
||||
return
|
||||
deselected: Final = [item for item in items if _lacks_capability(item, backend)]
|
||||
if deselected:
|
||||
config.hook.pytest_deselected(items=deselected)
|
||||
items[:] = [item for item in items if not _lacks_capability(item, backend)]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SecretManagerClient:
|
||||
proxy: ProxyClient
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def client(proxy: ProxyClient) -> SecretManagerClient:
|
||||
return SecretManagerClient(proxy)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def backend() -> SecretBackend:
|
||||
return selected_backend()
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def store(backend: SecretBackend) -> SecretStore:
|
||||
return backend.from_env()
|
||||
25
tests/e2e/secret_manager/secret_backends.py
Normal file
25
tests/e2e/secret_manager/secret_backends.py
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from e2e_config import SECRET_MANAGER_OPT_IN_ENV
|
||||
from secret_store import SecretBackend
|
||||
from secret_store_cyberark import CYBERARK
|
||||
from secret_store_hashicorp_vault import HASHICORP_VAULT
|
||||
|
||||
BACKENDS: Final = MappingProxyType({backend.system: backend for backend in (HASHICORP_VAULT, CYBERARK)})
|
||||
|
||||
|
||||
def selected_backend() -> SecretBackend:
|
||||
system: Final = os.environ.get(SECRET_MANAGER_OPT_IN_ENV, "").strip()
|
||||
backend: Final = BACKENDS.get(system)
|
||||
if backend is None:
|
||||
pytest.fail(
|
||||
f"{SECRET_MANAGER_OPT_IN_ENV}={system!r} names no secret manager backend; "
|
||||
f"set it to one of {sorted(BACKENDS)}"
|
||||
)
|
||||
return backend
|
||||
29
tests/e2e/secret_manager/secret_store.py
Normal file
29
tests/e2e/secret_manager/secret_store.py
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Literal, Protocol
|
||||
|
||||
SECRET_MANAGER_CONFIG_DIR: Final = "gateway"
|
||||
|
||||
|
||||
class SecretStore(Protocol):
|
||||
def write(self, name: str, value: str) -> None: ...
|
||||
|
||||
def read(self, name: str) -> str | None: ...
|
||||
|
||||
def destroy(self, name: str) -> None: ...
|
||||
|
||||
|
||||
Capability = Literal["deletes_stored_keys"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SecretBackend:
|
||||
system: str
|
||||
from_env: Callable[[], SecretStore]
|
||||
capabilities: frozenset[Capability]
|
||||
|
||||
@property
|
||||
def proxy_config(self) -> str:
|
||||
return f"{SECRET_MANAGER_CONFIG_DIR}/secret_manager_{self.system}_ci_config.yml"
|
||||
118
tests/e2e/secret_manager/secret_store_cyberark.py
Normal file
118
tests/e2e/secret_manager/secret_store_cyberark.py
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Final, Literal
|
||||
from urllib.parse import quote
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from e2e_http import ExternalWrite, Headers, send_text_external
|
||||
from pydantic import Field
|
||||
|
||||
from secret_store import SecretBackend
|
||||
|
||||
CYBERARK_API_BASE_ENV: Final = "E2E_CYBERARK_API_BASE"
|
||||
CYBERARK_ACCOUNT_ENV: Final = "E2E_CYBERARK_ACCOUNT"
|
||||
CYBERARK_USERNAME_ENV: Final = "E2E_CYBERARK_USERNAME"
|
||||
CYBERARK_API_KEY_ENV: Final = "E2E_CYBERARK_API_KEY"
|
||||
|
||||
# The same defaults CyberArkSecretManager falls back to for CYBERARK_*.
|
||||
DEFAULT_API_BASE: Final = "http://127.0.0.1:8080"
|
||||
DEFAULT_ACCOUNT: Final = "default"
|
||||
DEFAULT_USERNAME: Final = "admin"
|
||||
|
||||
SYSTEM: Final = "cyberark"
|
||||
|
||||
_START_HINT: Final = (
|
||||
f"Start one with `bash tests/e2e/secret_manager/backend.sh up {SYSTEM}`, which writes the env for "
|
||||
f"the proxy (booted from gateway/secret_manager_{SYSTEM}_ci_config.yml) and for the tests"
|
||||
)
|
||||
|
||||
|
||||
class ConjurHeaders(Headers):
|
||||
authorization: str = Field(repr=False)
|
||||
content_type: str | None = Field(default=None, serialization_alias="Content-Type")
|
||||
|
||||
|
||||
def _policy_scalar(name: str) -> str:
|
||||
# Quoted the way CyberArkSecretManager._ensure_variable_exists quotes it.
|
||||
return yaml.safe_dump(name, default_style='"').strip()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Conjur:
|
||||
base_url: str
|
||||
account: str
|
||||
username: str
|
||||
api_key: str = field(repr=False)
|
||||
|
||||
def _fail_unless_reached(self, result: ExternalWrite, action: str) -> None:
|
||||
if result.status_code == -1:
|
||||
pytest.fail(f"No live Conjur at {self.base_url}: {result.body}. {_START_HINT}")
|
||||
if result.status_code == 401:
|
||||
pytest.fail(f"Conjur rejected {self.username}'s credentials while trying to {action}. {_START_HINT}")
|
||||
|
||||
def _headers(self, content_type: str | None = None) -> ConjurHeaders:
|
||||
# Tokens last about eight minutes, so each call authenticates afresh rather than
|
||||
# letting a long session outlive a cached one.
|
||||
auth: Final = send_text_external(
|
||||
"POST",
|
||||
f"{self.base_url}/authn/{self.account}/{quote(self.username, safe='')}/authenticate",
|
||||
headers=Headers(),
|
||||
content=self.api_key,
|
||||
)
|
||||
self._fail_unless_reached(auth, "authenticate")
|
||||
if not auth.ok:
|
||||
pytest.fail(f"Conjur refused to authenticate {self.username}: HTTP {auth.status_code} {auth.body[:300]}")
|
||||
token: Final = base64.b64encode(auth.body.encode()).decode()
|
||||
return ConjurHeaders(authorization=f'Token token="{token}"', content_type=content_type)
|
||||
|
||||
def _secret_url(self, name: str) -> str:
|
||||
return f"{self.base_url}/secrets/{self.account}/variable/{quote(name, safe='')}"
|
||||
|
||||
def _update_root_policy(self, method: Literal["POST", "PATCH"], policy: str, action: str) -> None:
|
||||
result: Final = send_text_external(
|
||||
method,
|
||||
f"{self.base_url}/policies/{self.account}/policy/root",
|
||||
headers=self._headers(content_type="application/x-yaml"),
|
||||
content=policy,
|
||||
)
|
||||
self._fail_unless_reached(result, action)
|
||||
if not result.ok:
|
||||
pytest.fail(f"Conjur refused to {action}: HTTP {result.status_code} {result.body[:300]}")
|
||||
|
||||
def write(self, name: str, value: str) -> None:
|
||||
self._update_root_policy("POST", f"- !variable {_policy_scalar(name)}\n", f"declare {name}")
|
||||
result: Final = send_text_external("POST", self._secret_url(name), headers=self._headers(), content=value)
|
||||
self._fail_unless_reached(result, f"write {name}")
|
||||
if not result.ok:
|
||||
pytest.fail(f"Conjur refused to write {name}: HTTP {result.status_code} {result.body[:300]}")
|
||||
|
||||
def read(self, name: str) -> str | None:
|
||||
result: Final = send_text_external("GET", self._secret_url(name), headers=self._headers())
|
||||
self._fail_unless_reached(result, f"read {name}")
|
||||
if result.status_code == 404:
|
||||
return None
|
||||
if not result.ok:
|
||||
pytest.fail(f"Conjur refused to read {name}: HTTP {result.status_code} {result.body[:300]}")
|
||||
return result.body
|
||||
|
||||
def destroy(self, name: str) -> None:
|
||||
self._update_root_policy("PATCH", f"- !delete\n record: !variable {_policy_scalar(name)}\n", f"destroy {name}")
|
||||
|
||||
|
||||
def conjur_from_env() -> Conjur:
|
||||
api_key: Final = os.environ.get(CYBERARK_API_KEY_ENV, "").strip()
|
||||
if not api_key:
|
||||
pytest.fail(f"The {SYSTEM} lane needs {CYBERARK_API_KEY_ENV} to reach its Conjur. {_START_HINT}")
|
||||
return Conjur(
|
||||
base_url=os.environ.get(CYBERARK_API_BASE_ENV, "").strip().rstrip("/") or DEFAULT_API_BASE,
|
||||
account=os.environ.get(CYBERARK_ACCOUNT_ENV, "").strip() or DEFAULT_ACCOUNT,
|
||||
username=os.environ.get(CYBERARK_USERNAME_ENV, "").strip() or DEFAULT_USERNAME,
|
||||
api_key=api_key,
|
||||
)
|
||||
|
||||
|
||||
CYBERARK: Final = SecretBackend(system=SYSTEM, from_env=conjur_from_env, capabilities=frozenset())
|
||||
113
tests/e2e/secret_manager/secret_store_hashicorp_vault.py
Normal file
113
tests/e2e/secret_manager/secret_store_hashicorp_vault.py
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from e2e_http import (
|
||||
Headers,
|
||||
NetworkError,
|
||||
Success,
|
||||
UnknownApiError,
|
||||
delete_external,
|
||||
get_external,
|
||||
post_json_external,
|
||||
)
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from secret_store import SecretBackend
|
||||
|
||||
VAULT_ADDR_ENV: Final = "E2E_VAULT_ADDR"
|
||||
VAULT_TOKEN_ENV: Final = "E2E_VAULT_TOKEN"
|
||||
VAULT_MOUNT_ENV: Final = "E2E_VAULT_MOUNT_NAME"
|
||||
|
||||
DEFAULT_VAULT_ADDR: Final = "http://127.0.0.1:8200"
|
||||
DEFAULT_MOUNT: Final = "secret"
|
||||
|
||||
SYSTEM: Final = "hashicorp_vault"
|
||||
|
||||
_START_HINT: Final = (
|
||||
f"Start one with `bash tests/e2e/secret_manager/backend.sh up {SYSTEM}`, which writes the env for "
|
||||
f"the proxy (booted from gateway/secret_manager_{SYSTEM}_ci_config.yml) and for the tests"
|
||||
)
|
||||
|
||||
|
||||
class VaultHeaders(Headers):
|
||||
x_vault_token: str = Field(serialization_alias="X-Vault-Token", repr=False)
|
||||
|
||||
|
||||
class KvData(BaseModel):
|
||||
key: str = Field(repr=False)
|
||||
|
||||
|
||||
class KvWriteBody(BaseModel):
|
||||
data: KvData
|
||||
|
||||
|
||||
class KvReadData(BaseModel):
|
||||
data: KvData
|
||||
|
||||
|
||||
class KvReadResponse(BaseModel):
|
||||
data: KvReadData
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Vault:
|
||||
base_url: str
|
||||
token: str = field(repr=False)
|
||||
mount: str = DEFAULT_MOUNT
|
||||
|
||||
def _headers(self) -> VaultHeaders:
|
||||
return VaultHeaders(x_vault_token=self.token)
|
||||
|
||||
def _data_url(self, name: str) -> str:
|
||||
return f"{self.base_url}/v1/{self.mount}/data/{name}"
|
||||
|
||||
def _metadata_url(self, name: str) -> str:
|
||||
return f"{self.base_url}/v1/{self.mount}/metadata/{name}"
|
||||
|
||||
def write(self, name: str, value: str) -> None:
|
||||
write: Final = post_json_external(
|
||||
self._data_url(name), headers=self._headers(), json=KvWriteBody(data=KvData(key=value))
|
||||
)
|
||||
if write.status_code == -1:
|
||||
pytest.fail(f"No live Vault at {self.base_url}: {write.body}. {_START_HINT}")
|
||||
if not write.ok:
|
||||
pytest.fail(f"Vault refused to write {name}: HTTP {write.status_code} {write.body[:300]}")
|
||||
|
||||
def read(self, name: str) -> str | None:
|
||||
result: Final = get_external(self._data_url(name), headers=self._headers(), response_type=KvReadResponse)
|
||||
match result:
|
||||
case Success(data=body):
|
||||
return body.data.data.key
|
||||
case UnknownApiError(status_code=404):
|
||||
return None
|
||||
case NetworkError(message=message):
|
||||
return pytest.fail(f"No live Vault at {self.base_url}: {message}. {_START_HINT}")
|
||||
case _:
|
||||
return pytest.fail(f"Vault refused to read {name}: {result}")
|
||||
|
||||
def destroy(self, name: str) -> None:
|
||||
write: Final = delete_external(self._metadata_url(name), headers=self._headers())
|
||||
if not write.ok and write.status_code != 404:
|
||||
pytest.fail(f"Vault refused to destroy {name}: HTTP {write.status_code} {write.body[:300]}")
|
||||
|
||||
|
||||
def vault_from_env() -> Vault:
|
||||
token: Final = os.environ.get(VAULT_TOKEN_ENV, "").strip()
|
||||
if not token:
|
||||
pytest.fail(f"The hashicorp_vault lane needs {VAULT_TOKEN_ENV} to reach its Vault. {_START_HINT}")
|
||||
return Vault(
|
||||
base_url=os.environ.get(VAULT_ADDR_ENV, DEFAULT_VAULT_ADDR).rstrip("/"),
|
||||
token=token,
|
||||
mount=os.environ.get(VAULT_MOUNT_ENV, "").strip() or DEFAULT_MOUNT,
|
||||
)
|
||||
|
||||
|
||||
HASHICORP_VAULT: Final = SecretBackend(
|
||||
system=SYSTEM,
|
||||
from_env=vault_from_env,
|
||||
capabilities=frozenset({"deletes_stored_keys"}),
|
||||
)
|
||||
128
tests/e2e/secret_manager/test_secret_manager_e2e.py
Normal file
128
tests/e2e/secret_manager/test_secret_manager_e2e.py
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import Result, Success, UnauthorizedError, unwrap
|
||||
from lifecycle import ResourceManager
|
||||
from models import ChatBody, ChatMessage, ChatResponse, KeyGenerateBody, LiteLLMParamsBody
|
||||
from proxy_client import ProxyClient
|
||||
from secret_store import SecretStore
|
||||
|
||||
pytestmark = [pytest.mark.e2e, pytest.mark.secret_manager]
|
||||
|
||||
BACKEND_MODEL: Final = "openai/gpt-4o-mini"
|
||||
VIRTUAL_KEY_PREFIX: Final = "litellm-e2e/virtual-keys/"
|
||||
PROVIDER_KEY_ENV: Final = "OPENAI_API_KEY"
|
||||
|
||||
|
||||
# The proxy's env never holds OPENAI_API_KEY and each test seeds it under a fresh name, so a passing
|
||||
# call proves the key came from the manager and not get_secret's os.environ fallback.
|
||||
def _provider_key() -> str:
|
||||
key: Final = os.environ.get(PROVIDER_KEY_ENV, "").strip()
|
||||
if not key:
|
||||
pytest.fail(f"The secret manager suite seeds the manager with the runner's {PROVIDER_KEY_ENV}, which is unset")
|
||||
return key
|
||||
|
||||
|
||||
def _seed(store: SecretStore, resources: ResourceManager, value: str) -> str:
|
||||
name: Final = f"litellm-e2e-openai-{unique_marker()}"
|
||||
store.write(name, value)
|
||||
resources.defer(lambda: store.destroy(name))
|
||||
return name
|
||||
|
||||
|
||||
def _deploy(proxy: ProxyClient, resources: ResourceManager, secret_name: str) -> str:
|
||||
model_name: Final = f"secret-manager-backed-{unique_marker()}"
|
||||
model_id: Final = proxy.create_model(
|
||||
model_name,
|
||||
LiteLLMParamsBody(model=BACKEND_MODEL, api_key=f"os.environ/{secret_name}"),
|
||||
provider_live=True,
|
||||
)
|
||||
resources.defer(lambda: proxy.delete_model(model_id))
|
||||
return model_name
|
||||
|
||||
|
||||
def _chat(proxy: ProxyClient, key: str, model: str) -> Result[ChatResponse]:
|
||||
return proxy.chat(
|
||||
key,
|
||||
ChatBody(
|
||||
model=model,
|
||||
messages=[ChatMessage(role="user", content=f"reply with one word {unique_marker()}")],
|
||||
max_tokens=16,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _eventually(proxy: ProxyClient, read: Callable[[], str | None], expected: str | None, context: str) -> None:
|
||||
deadline: Final = time.monotonic() + proxy.poll_timeout
|
||||
last: str | None = read()
|
||||
while last != expected and time.monotonic() < deadline:
|
||||
time.sleep(proxy.poll_interval)
|
||||
last = read()
|
||||
if last != expected:
|
||||
pytest.fail(
|
||||
f"{context}: the secret manager still holds {'a value' if last is not None else 'nothing'} after the deadline"
|
||||
)
|
||||
|
||||
|
||||
class TestSecretManager:
|
||||
@pytest.mark.covers("other.config.secret_resolution.kms_integration")
|
||||
def test_deployment_key_resolves_from_the_manager(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, store: SecretStore, scoped_key: str
|
||||
) -> None:
|
||||
model: Final = _deploy(proxy, resources, _seed(store, resources, _provider_key()))
|
||||
|
||||
response: Final = unwrap(_chat(proxy, scoped_key, model))
|
||||
|
||||
assert response.choices, f"the manager-backed deployment answered with no choices: {response}"
|
||||
|
||||
@pytest.mark.covers("other.config.secret_resolution.manager_value_used")
|
||||
def test_deployment_uses_the_value_the_manager_holds(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, store: SecretStore, scoped_key: str
|
||||
) -> None:
|
||||
bogus: Final = f"sk-litellm-e2e-not-a-key-{unique_marker()}"
|
||||
model: Final = _deploy(proxy, resources, _seed(store, resources, bogus))
|
||||
|
||||
result: Final = _chat(proxy, scoped_key, model)
|
||||
|
||||
match result:
|
||||
case UnauthorizedError(body=body):
|
||||
assert "AuthenticationError" in body, f"the 401 did not come from the provider: {body[:300]}"
|
||||
case Success():
|
||||
pytest.fail("a deployment whose managed secret is not a real key still reached the provider")
|
||||
case _:
|
||||
pytest.fail(f"expected the provider to reject the manager-held key with 401, got {result}")
|
||||
|
||||
@pytest.mark.covers("other.config.secret_manager.virtual_key_stored")
|
||||
def test_generated_key_is_written_to_the_manager(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, store: SecretStore
|
||||
) -> None:
|
||||
alias: Final = f"litellm-e2e-vk-{unique_marker()}"
|
||||
secret_name: Final = f"{VIRTUAL_KEY_PREFIX}{alias}"
|
||||
resources.defer(lambda: store.destroy(secret_name))
|
||||
key: Final = proxy.generate_key(KeyGenerateBody(key_alias=alias))
|
||||
resources.defer(lambda: proxy.delete_key(key))
|
||||
|
||||
_eventually(proxy, lambda: store.read(secret_name), key, f"the generated key {alias}")
|
||||
|
||||
@pytest.mark.requires_capability("deletes_stored_keys")
|
||||
@pytest.mark.covers("other.config.secret_manager.virtual_key_deleted")
|
||||
def test_deleted_key_is_removed_from_the_manager(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, store: SecretStore
|
||||
) -> None:
|
||||
alias: Final = f"litellm-e2e-vk-{unique_marker()}"
|
||||
secret_name: Final = f"{VIRTUAL_KEY_PREFIX}{alias}"
|
||||
resources.defer(lambda: store.destroy(secret_name))
|
||||
key: Final = proxy.generate_key(KeyGenerateBody(key_alias=alias))
|
||||
resources.defer(lambda: proxy.delete_key(key))
|
||||
_eventually(proxy, lambda: store.read(secret_name), key, f"the generated key {alias}")
|
||||
|
||||
proxy.delete_key(key)
|
||||
|
||||
_eventually(proxy, lambda: store.read(secret_name), None, f"the deleted key {alias}")
|
||||
|
|
@ -7,6 +7,7 @@ import time
|
|||
import uuid
|
||||
from collections.abc import Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -45,8 +46,37 @@ def stop_root_process(process: subprocess.Popen[bytes]) -> bool:
|
|||
return True
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class OwnedProxy:
|
||||
gateway: Gateway
|
||||
process: subprocess.Popen[bytes]
|
||||
log: Path
|
||||
|
||||
|
||||
@contextmanager
|
||||
def owned_proxy(gateway: Gateway, directory: Path, overrides: Mapping[str, str], *, config: Path | None = None, remove_environment: tuple[str, ...] = ()) -> Iterator[Gateway]:
|
||||
def owned_proxy(
|
||||
gateway: Gateway,
|
||||
directory: Path,
|
||||
overrides: Mapping[str, str],
|
||||
*,
|
||||
config: Path | None = None,
|
||||
remove_environment: tuple[str, ...] = (),
|
||||
) -> Iterator[Gateway]:
|
||||
with owned_proxy_process(
|
||||
gateway, directory, overrides, config=config, remove_environment=remove_environment
|
||||
) as owned:
|
||||
yield owned.gateway
|
||||
|
||||
|
||||
@contextmanager
|
||||
def owned_proxy_process(
|
||||
gateway: Gateway,
|
||||
directory: Path,
|
||||
overrides: Mapping[str, str],
|
||||
*,
|
||||
config: Path | None = None,
|
||||
remove_environment: tuple[str, ...] = (),
|
||||
) -> Iterator[OwnedProxy]:
|
||||
with socket.socket() as reserve:
|
||||
reserve.bind(("127.0.0.1", 0))
|
||||
port: Final = reserve.getsockname()[1]
|
||||
|
|
@ -60,7 +90,8 @@ def owned_proxy(gateway: Gateway, directory: Path, overrides: Mapping[str, str],
|
|||
}
|
||||
output: Final = Path(os.environ.get("INTEGRATION_RESULTS_DIR", str(directory)))
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
with (output / f"owned-proxy-{uuid.uuid4().hex}.log").open("w") as log:
|
||||
log_path: Final = output / f"owned-proxy-{uuid.uuid4().hex}.log"
|
||||
with log_path.open("w") as log:
|
||||
process: Final = subprocess.Popen(
|
||||
[
|
||||
sys.executable,
|
||||
|
|
@ -95,7 +126,7 @@ def owned_proxy(gateway: Gateway, directory: Path, overrides: Mapping[str, str],
|
|||
pass
|
||||
assert time.monotonic() < deadline, "Owned proxy readiness deadline exceeded"
|
||||
time.sleep(0.1)
|
||||
yield Gateway(client, gateway.key, gateway.upstream_url)
|
||||
yield OwnedProxy(Gateway(client, gateway.key, gateway.upstream_url), process, log_path)
|
||||
finally:
|
||||
root_stopped: Final = stop_root_process(process)
|
||||
residual: Final = group_members(process.pid)
|
||||
|
|
|
|||
|
|
@ -162,6 +162,12 @@
|
|||
"tests/integration/routing/test_redis_recovery.py::test_owned_redis_outage_recovers_requests_and_real_response_cache": [
|
||||
"other.routing.redis.owned_outage_recovers_serving_and_response_cache"
|
||||
],
|
||||
"tests/integration/providers/test_anthropic_wire.py::test_anthropic_bare_string_content_item_is_rejected_as_client_error_before_the_wire[type_word]": [
|
||||
"other.provider_wire.anthropic.bare_string_content_item_is_client_error"
|
||||
],
|
||||
"tests/integration/providers/test_anthropic_wire.py::test_anthropic_bare_string_content_item_is_rejected_as_client_error_before_the_wire[plain]": [
|
||||
"other.provider_wire.anthropic.bare_string_content_item_is_client_error"
|
||||
],
|
||||
"tests/integration/providers/test_anthropic_wire.py::test_anthropic_tool_history_and_cache_tokens_keep_wire_and_accounting_contracts": [
|
||||
"other.provider_wire.anthropic.tool_history_system_cache_and_internal_fields",
|
||||
"quota_management.spend_tracking.cache_tokens.disjoint_classes_use_explicit_rates"
|
||||
|
|
@ -278,6 +284,15 @@
|
|||
"tests/integration/spend/test_filtered_ledger.py::test_rotated_keys_users_and_model_groups_preserve_success_failure_cache_ledger": [
|
||||
"quota_management.spend_tracking.filtered_ledger_preserves_owner_identity_and_totals"
|
||||
],
|
||||
"tests/integration/spend/test_shutdown_flush.py::test_daily_spend_batch_cancelled_while_waiting_for_a_pool_connection_is_written_by_the_final_flush": [
|
||||
"quota_management.spend_tracking.shutdown_cancel_keeps_in_flight_daily_batch"
|
||||
],
|
||||
"tests/integration/spend/test_shutdown_flush.py::test_daily_spend_batch_cancelled_while_waiting_for_a_row_lock_is_written_exactly_once": [
|
||||
"quota_management.spend_tracking.shutdown_cancel_keeps_in_flight_daily_batch"
|
||||
],
|
||||
"tests/integration/spend/test_spend_calculate.py::test_spend_calculate_rejects_unpriced_model_with_400": [
|
||||
"quota_management.spend_tracking.spend_calculate.rejects_unpriced_model"
|
||||
],
|
||||
"tests/integration/management/test_partial_update_sequences.py::test_restricted_actor_cannot_detach_key_from_project": [
|
||||
"mgmt.key.update.project_detach_denied_to_restricted_actor"
|
||||
],
|
||||
|
|
@ -1770,6 +1785,18 @@
|
|||
],
|
||||
"tests/integration/mcp/test_mcp_lifecycle.py::test_same_url_server_grants_scope_discovery_and_direct_or_virtual_execution[bearer]": [
|
||||
"other.mcp.permissions.same_url_servers_enforce_discovery_and_execution"
|
||||
],
|
||||
"tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[chat]": [
|
||||
"other.observability.callbacks.raising_success_deployment_hook_keeps_response"
|
||||
],
|
||||
"tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[embeddings]": [
|
||||
"other.observability.callbacks.raising_success_deployment_hook_keeps_response"
|
||||
],
|
||||
"tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[responses]": [
|
||||
"other.observability.callbacks.raising_success_deployment_hook_keeps_response"
|
||||
],
|
||||
"tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[videos]": [
|
||||
"other.observability.callbacks.raising_success_deployment_hook_keeps_response"
|
||||
]
|
||||
},
|
||||
"browser": {
|
||||
|
|
|
|||
|
|
@ -1,13 +1,15 @@
|
|||
import base64
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Callable, Mapping
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from integration._support.client import Gateway, eventually
|
||||
from integration._support.client import Gateway, JsonValue, eventually, object_value, string_value
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
|
@ -152,3 +154,163 @@ def test_concurrent_success_and_failure_join_callbacks_and_rows_without_credenti
|
|||
assert rows[0]["prompt_tokens"] == event["prompt_tokens"]
|
||||
else:
|
||||
assert event["prompt_tokens"] == event["completion_tokens"] == rows[0]["completion_tokens"] == 0
|
||||
|
||||
|
||||
_RAISING_HOOK: Final = """
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
||||
|
||||
class RaisingHook(CustomLogger):
|
||||
async def async_post_call_success_deployment_hook(self, request_data, response, call_type):
|
||||
raise RuntimeError(f"hook rejected {type(response).__name__} for {call_type}")
|
||||
|
||||
|
||||
instance = RaisingHook()
|
||||
"""
|
||||
|
||||
_VIDEO_JOB: Final = {
|
||||
"id": "video_hook_isolation",
|
||||
"object": "video",
|
||||
"status": "queued",
|
||||
"model": "sora-2",
|
||||
"seconds": "4",
|
||||
"size": "720x1280",
|
||||
}
|
||||
|
||||
_UPSTREAM_REPLIES: Final[Mapping[str, Mapping[str, JsonValue]]] = {
|
||||
"/v1/chat/completions": {
|
||||
"id": "chatcmpl_hook_isolation",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-5.6",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
"/v1/embeddings": {
|
||||
"object": "list",
|
||||
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}],
|
||||
"model": "text-embedding-3-small",
|
||||
"usage": {"prompt_tokens": 1, "total_tokens": 1},
|
||||
},
|
||||
"/v1/responses": {
|
||||
"id": "resp_hook_isolation",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "completed",
|
||||
"model": "gpt-5.6",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_hook_isolation",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "hi", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"parallel_tool_calls": False,
|
||||
"tool_choice": "auto",
|
||||
"tools": [],
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
"/v1/videos": _VIDEO_JOB,
|
||||
}
|
||||
|
||||
|
||||
def _item(value: JsonValue, index: int) -> JsonValue:
|
||||
assert isinstance(value, list), f"Expected a list, received {type(value).__name__}"
|
||||
return value[index]
|
||||
|
||||
|
||||
def _chat_text(body: dict[str, JsonValue]) -> str:
|
||||
return string_value(object_value(object_value(_item(body["choices"], 0))["message"])["content"])
|
||||
|
||||
|
||||
def _embedding_vector(body: dict[str, JsonValue]) -> JsonValue:
|
||||
return object_value(_item(body["data"], 0))["embedding"]
|
||||
|
||||
|
||||
def _responses_text(body: dict[str, JsonValue]) -> str:
|
||||
return string_value(object_value(_item(object_value(_item(body["output"], 0))["content"], 0))["text"])
|
||||
|
||||
|
||||
def _video_job(body: dict[str, JsonValue]) -> tuple[str, str]:
|
||||
encoded_id: Final = string_value(body["id"]).removeprefix("video_")
|
||||
decoded: Final = base64.b64decode(encoded_id).decode()
|
||||
return decoded.rsplit("video_id:", 1)[-1], string_value(body["status"])
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Surface:
|
||||
route: str
|
||||
upstream_model: str
|
||||
body: Callable[[str], dict[str, JsonValue]]
|
||||
observed: Callable[[dict[str, JsonValue]], JsonValue | tuple[str, str]]
|
||||
expected: JsonValue | tuple[str, str]
|
||||
|
||||
|
||||
_SURFACES: Final = (
|
||||
pytest.param(
|
||||
_Surface(
|
||||
"/v1/chat/completions",
|
||||
"openai/gpt-5.6",
|
||||
lambda model: {"model": model, "messages": [{"role": "user", "content": "hook isolation"}]},
|
||||
_chat_text,
|
||||
"hi",
|
||||
),
|
||||
id="chat",
|
||||
),
|
||||
pytest.param(
|
||||
_Surface(
|
||||
"/v1/embeddings",
|
||||
"openai/text-embedding-3-small",
|
||||
lambda model: {"model": model, "input": "hook isolation"},
|
||||
_embedding_vector,
|
||||
[0.1, 0.2],
|
||||
),
|
||||
id="embeddings",
|
||||
),
|
||||
pytest.param(
|
||||
_Surface(
|
||||
"/v1/responses",
|
||||
"openai/gpt-5.6",
|
||||
lambda model: {"model": model, "input": "hook isolation"},
|
||||
_responses_text,
|
||||
"hi",
|
||||
),
|
||||
id="responses",
|
||||
),
|
||||
pytest.param(
|
||||
_Surface(
|
||||
"/v1/videos",
|
||||
"openai/sora-2",
|
||||
lambda model: {"model": model, "prompt": "a cat"},
|
||||
_video_job,
|
||||
(_VIDEO_JOB["id"], _VIDEO_JOB["status"]),
|
||||
),
|
||||
id="videos",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.observability.callbacks.raising_success_deployment_hook_keeps_response")
|
||||
@pytest.mark.parametrize("surface", _SURFACES)
|
||||
def test_response_survives_raising_success_deployment_hook(gateway: Gateway, tmp_path: Path, surface: _Surface) -> None:
|
||||
def upstream(request: Request) -> Reply:
|
||||
assert request.target == surface.route, request.target
|
||||
assert b"hook isolation" in request.body or b"a cat" in request.body, request.body[:300]
|
||||
return Reply(body=json.dumps(_UPSTREAM_REPLIES[surface.route]).encode())
|
||||
|
||||
(tmp_path / "raising_hook.py").write_text(_RAISING_HOOK)
|
||||
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
config["litellm_settings"].update({"callbacks": ["raising_hook.instance"]})
|
||||
path: Final = tmp_path / "hook.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
with (
|
||||
wire_server(upstream) as provider,
|
||||
owned_proxy(gateway, tmp_path, {}, config=path) as candidate,
|
||||
candidate.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(model=surface.upstream_model, api_base=provider.url + "/v1")
|
||||
response: Final = candidate.request("POST", surface.route, surface.body(model))
|
||||
assert response.status_code == 200, response.text
|
||||
assert surface.observed(object_value(response.json())) == surface.expected, response.text
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ import uuid
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from integration._support.client import Gateway, eventually, object_value
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
|
@ -59,3 +58,26 @@ def test_anthropic_tool_history_and_cache_tokens_keep_wire_and_accounting_contra
|
|||
parsed: Final = json.loads(metadata) if isinstance(metadata, str) else object_value(metadata)
|
||||
assert parsed["cost_breakdown"]["input_cost"] == pytest.approx(0.0245)
|
||||
assert parsed["cost_breakdown"]["output_cost"] == pytest.approx(0.008)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.anthropic.bare_string_content_item_is_client_error")
|
||||
@pytest.mark.parametrize(
|
||||
"text", [pytest.param("what type of file is this?", id="type_word"), pytest.param("hello", id="plain")]
|
||||
)
|
||||
def test_anthropic_bare_string_content_item_is_rejected_as_client_error_before_the_wire(
|
||||
gateway: Gateway, text: str
|
||||
) -> None:
|
||||
def respond(request: Request) -> Reply:
|
||||
raise AssertionError(f"upstream must not be reached: {request.target}")
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="anthropic/claude-sonnet-4-5-20250929", api_base=wire.url, api_key="synthetic-anthropic-key"
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "max_tokens": 16, "timeout": 5, "messages": [{"role": "system", "content": [text]}]},
|
||||
)
|
||||
assert response.status_code == 400, response.text
|
||||
assert wire.drain() == ()
|
||||
|
|
|
|||
189
tests/integration/spend/test_shutdown_flush.py
Normal file
189
tests/integration/spend/test_shutdown_flush.py
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
import json
|
||||
import os
|
||||
import signal
|
||||
import threading
|
||||
import uuid
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import psycopg
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from integration._support.client import Gateway, delete_key_if_present, eventually, string_value
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.process import OwnedProxy, owned_proxy_process
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
REQUESTS_WHILE_BLOCKED: Final = 6
|
||||
CANCEL_LOG_LINE: Final = "in-flight scheduled job(s) for shutdown"
|
||||
BATCH_DRAINED_LOG_LINE: Final = f"flushed {REQUESTS_WHILE_BLOCKED} daily spend update items from in-memory queue"
|
||||
MODEL_INSERT_ARRIVED_LOG_LINE: Final = "path=/model/new"
|
||||
|
||||
|
||||
def _api_requests(table: str, column: str, identity: str) -> int:
|
||||
rows: Final = read_rows(
|
||||
f'SELECT coalesce(sum(api_requests), 0)::int AS total FROM "{table}" WHERE {column}=%s', (identity,)
|
||||
)
|
||||
total: Final = rows[0]["total"]
|
||||
assert isinstance(total, int)
|
||||
return total
|
||||
|
||||
|
||||
def _waiting_on(table: str) -> int:
|
||||
rows: Final = read_rows(
|
||||
"SELECT count(*)::int AS waiting FROM pg_stat_activity WHERE wait_event_type='Lock' AND query LIKE %s",
|
||||
(f'%"{table}"%',),
|
||||
)
|
||||
waiting: Final = rows[0]["waiting"]
|
||||
assert isinstance(waiting, int)
|
||||
return waiting
|
||||
|
||||
|
||||
def _provider(request: Request) -> Reply:
|
||||
if request.method != "POST":
|
||||
return Reply(status=404, body=b'{"error":"not scripted"}')
|
||||
assert request.target == "/v1/chat/completions"
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"id": "chatcmpl-" + uuid.uuid4().hex,
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Shutdown:
|
||||
owner: str
|
||||
team: str
|
||||
owned: OwnedProxy
|
||||
key: str
|
||||
model: str
|
||||
|
||||
def chat(self) -> None:
|
||||
body: Final = {"model": self.model, "messages": [{"role": "user", "content": f"spend {uuid.uuid4().hex}"}]}
|
||||
assert self.owned.gateway.request("POST", "/v1/chat/completions", body, key=self.key).status_code == 200
|
||||
|
||||
def daily_user_requests(self) -> int:
|
||||
return _api_requests("LiteLLM_DailyUserSpend", "user_id", self.owner)
|
||||
|
||||
def logged(self, line: str, times: int = 1) -> bool:
|
||||
return self.owned.log.read_text(errors="replace").count(line) >= times
|
||||
|
||||
def chat_while_spend_update_is_blocked(self, blocker: psycopg.Connection, table: str) -> None:
|
||||
blocker.execute(f'LOCK TABLE "{table}" IN EXCLUSIVE MODE')
|
||||
for _ in range(REQUESTS_WHILE_BLOCKED):
|
||||
self.chat()
|
||||
eventually(lambda: _waiting_on(table), lambda waiting: waiting == 1, seconds=30)
|
||||
|
||||
def start_blocked_model_insert(self) -> threading.Thread:
|
||||
body: Final = {
|
||||
"model_name": f"integration-blocked-{uuid.uuid4().hex}",
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "integration-provider-key"},
|
||||
"model_info": {},
|
||||
}
|
||||
|
||||
def insert() -> None:
|
||||
try:
|
||||
self.owned.gateway.request("POST", "/model/new", body)
|
||||
except httpx.TransportError:
|
||||
pass
|
||||
|
||||
thread: Final = threading.Thread(target=insert, daemon=True)
|
||||
thread.start()
|
||||
return thread
|
||||
|
||||
def terminate_once(self, blocked: Callable[[], bool], release: Callable[[], None]) -> None:
|
||||
eventually(blocked, lambda state: state, seconds=60)
|
||||
self.owned.process.send_signal(signal.SIGTERM)
|
||||
eventually(lambda: self.logged(CANCEL_LOG_LINE), lambda seen: seen, seconds=60)
|
||||
release()
|
||||
self.owned.process.wait(timeout=120)
|
||||
|
||||
|
||||
def _config_with_pool_limit(tmp_path: Path, pool_limit: int) -> Path:
|
||||
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
config["general_settings"]["database_connection_pool_limit"] = pool_limit
|
||||
config["general_settings"]["database_connection_pool_timeout"] = 60
|
||||
path: Final = tmp_path / f"pool-{pool_limit}.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
return path
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _proxy_with_one_seeded_row(gateway: Gateway, tmp_path: Path, pool_limit: int) -> Iterator[_Shutdown]:
|
||||
owner: Final = f"integration-owner-{uuid.uuid4().hex}"
|
||||
with gateway.scenario() as scenario, wire_server(_provider) as wire:
|
||||
model: Final = scenario.model(api_base=wire.url + "/v1", num_retries=0)
|
||||
team: Final = scenario.team(models=[model])
|
||||
with owned_proxy_process(
|
||||
gateway,
|
||||
tmp_path,
|
||||
{
|
||||
"LITELLM_LOG": "DEBUG",
|
||||
"GRACEFUL_SHUTDOWN_TIMEOUT": "1",
|
||||
"SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS": "1",
|
||||
"SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS": "5",
|
||||
},
|
||||
config=_config_with_pool_limit(tmp_path, pool_limit),
|
||||
) as owned:
|
||||
key: Final = string_value(
|
||||
owned.gateway.post("/key/generate", {"user_id": owner, "team_id": team, "models": [model]})["key"]
|
||||
)
|
||||
scenario.cleanups.callback(delete_key_if_present, gateway, key)
|
||||
shutdown: Final = _Shutdown(owner, team, owned, key, model)
|
||||
shutdown.chat()
|
||||
eventually(shutdown.daily_user_requests, lambda total: total == 1, seconds=60)
|
||||
yield shutdown
|
||||
assert _api_requests("LiteLLM_DailyUserSpend", "user_id", owner) == 1 + REQUESTS_WHILE_BLOCKED
|
||||
assert _api_requests("LiteLLM_DailyTeamSpend", "team_id", team) == 1 + REQUESTS_WHILE_BLOCKED
|
||||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.shutdown_cancel_keeps_in_flight_daily_batch")
|
||||
def test_daily_spend_batch_cancelled_while_waiting_for_a_pool_connection_is_written_by_the_final_flush(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
with (
|
||||
_proxy_with_one_seeded_row(gateway, tmp_path, pool_limit=2) as shutdown,
|
||||
psycopg.connect(os.environ["DATABASE_URL"]) as models,
|
||||
psycopg.connect(os.environ["DATABASE_URL"]) as memberships,
|
||||
):
|
||||
models.execute('LOCK TABLE "LiteLLM_ProxyModelTable" IN EXCLUSIVE MODE')
|
||||
first: Final = shutdown.start_blocked_model_insert()
|
||||
eventually(lambda: _waiting_on("LiteLLM_ProxyModelTable"), lambda waiting: waiting == 1, seconds=30)
|
||||
shutdown.chat_while_spend_update_is_blocked(memberships, "LiteLLM_TeamMembership")
|
||||
second: Final = shutdown.start_blocked_model_insert()
|
||||
eventually(lambda: shutdown.logged(MODEL_INSERT_ARRIVED_LOG_LINE, times=2), lambda seen: seen, seconds=30)
|
||||
memberships.rollback()
|
||||
eventually(lambda: _waiting_on("LiteLLM_ProxyModelTable"), lambda waiting: waiting == 2, seconds=30)
|
||||
shutdown.terminate_once(lambda: shutdown.logged(BATCH_DRAINED_LOG_LINE), models.rollback)
|
||||
first.join(timeout=30)
|
||||
second.join(timeout=30)
|
||||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.shutdown_cancel_keeps_in_flight_daily_batch")
|
||||
def test_daily_spend_batch_cancelled_while_waiting_for_a_row_lock_is_written_exactly_once(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
with (
|
||||
_proxy_with_one_seeded_row(gateway, tmp_path, pool_limit=10) as shutdown,
|
||||
psycopg.connect(os.environ["DATABASE_URL"]) as holder,
|
||||
psycopg.connect(os.environ["DATABASE_URL"]) as memberships,
|
||||
):
|
||||
holder.execute('SELECT 1 FROM "LiteLLM_DailyUserSpend" WHERE user_id=%s FOR UPDATE', (shutdown.owner,))
|
||||
shutdown.chat_while_spend_update_is_blocked(memberships, "LiteLLM_TeamMembership")
|
||||
memberships.rollback()
|
||||
shutdown.terminate_once(
|
||||
lambda: shutdown.logged(BATCH_DRAINED_LOG_LINE) and _waiting_on("LiteLLM_DailyUserSpend") == 1,
|
||||
holder.rollback,
|
||||
)
|
||||
20
tests/integration/spend/test_spend_calculate.py
Normal file
20
tests/integration/spend/test_spend_calculate.py
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
import uuid
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import JSON_OBJECT, Gateway, object_value, string_value
|
||||
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.spend_calculate.rejects_unpriced_model")
|
||||
def test_spend_calculate_rejects_unpriced_model_with_400(gateway: Gateway) -> None:
|
||||
model: Final = f"openrouter/integration-unpriced-{uuid.uuid4().hex}"
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/spend/calculate",
|
||||
{"model": model, "messages": [{"role": "user", "content": "price this request"}]},
|
||||
)
|
||||
assert response.status_code == 400, response.text
|
||||
error: Final = object_value(JSON_OBJECT.validate_json(response.text)["error"])
|
||||
assert error["type"] == "invalid_request_error", response.text
|
||||
assert error["param"] == "model", response.text
|
||||
assert model in string_value(error["message"]), response.text
|
||||
|
|
@ -263,7 +263,7 @@ def test_trimming_should_not_change_original_messages():
|
|||
assert messages == messages_copy
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["gpt-4-0125-preview", "claude-sonnet-4-6"])
|
||||
@pytest.mark.parametrize("model", ["gpt-5.4-mini", "claude-sonnet-4-6"])
|
||||
def test_trimming_with_model_cost_max_input_tokens(model):
|
||||
messages = [
|
||||
{"role": "system", "content": "This is a normal system message"},
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import litellm
|
|||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
from unittest.mock import Mock
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.bedrock.common_utils import BedrockModelInfo
|
||||
|
||||
|
||||
|
||||
|
|
@ -42,7 +43,7 @@ def test_bedrock_completion_with_region_name():
|
|||
|
||||
# Pass the client so that the HTTP call will be intercepted.
|
||||
response = litellm.completion(
|
||||
model="cohere.command-r-v1:0",
|
||||
model="bedrock/cohere.command-r-v1:0",
|
||||
messages=[{"role": "user", "content": "Hello, world!"}],
|
||||
aws_region_name="us-west-12",
|
||||
client=client,
|
||||
|
|
@ -98,7 +99,7 @@ def test_bedrock_completion_with_dynamic_authentication_params():
|
|||
|
||||
# Pass the client so that the HTTP call will be intercepted.
|
||||
response = litellm.completion(
|
||||
model="cohere.command-r-v1:0",
|
||||
model="bedrock/cohere.command-r-v1:0",
|
||||
messages=[{"role": "user", "content": "Hello, world!"}],
|
||||
aws_access_key_id="dynamically_generated_access_key_id",
|
||||
aws_secret_access_key="dynamically_generated_secret_access_key",
|
||||
|
|
@ -146,7 +147,7 @@ def test_bedrock_completion_with_dynamic_bedrock_runtime_endpoint():
|
|||
|
||||
# Pass the client so that the HTTP call will be intercepted.
|
||||
response = litellm.completion(
|
||||
model="cohere.command-r-v1:0",
|
||||
model="bedrock/cohere.command-r-v1:0",
|
||||
messages=[{"role": "user", "content": "Hello, world!"}],
|
||||
aws_bedrock_runtime_endpoint="https://my-fake-endpoint.com",
|
||||
client=client,
|
||||
|
|
@ -179,7 +180,7 @@ class DummyCredentials:
|
|||
"model",
|
||||
[
|
||||
"bedrock/converse/cohere.command-r-v1:0",
|
||||
"cohere.command-r-v1:0",
|
||||
"amazon.nova-2-lite-v1:0",
|
||||
"bedrock/cohere.command-r-v1:0",
|
||||
"bedrock/invoke/cohere.command-r-v1:0",
|
||||
],
|
||||
|
|
@ -250,7 +251,7 @@ def test_dynamic_aws_params_propagation(model, param_name, param_value, expected
|
|||
"finish_reason": "COMPLETE",
|
||||
}
|
||||
)
|
||||
if "converse" in model:
|
||||
if BedrockModelInfo.get_bedrock_route(model) == "converse":
|
||||
mock_response.text = json.dumps(
|
||||
{
|
||||
"output": {
|
||||
|
|
|
|||
|
|
@ -9,6 +9,12 @@ from litellm.llms.fireworks_ai.chat.transformation import FireworksAIConfig
|
|||
|
||||
fireworks = FireworksAIConfig()
|
||||
|
||||
VISION_MODEL = next(
|
||||
key.removeprefix("fireworks_ai/")
|
||||
for key, info in litellm.model_cost.items()
|
||||
if key.startswith("fireworks_ai/accounts/fireworks/models/") and info.get("supports_vision") is True
|
||||
)
|
||||
|
||||
|
||||
def test_map_openai_params_tool_choice():
|
||||
# Test case 1: tool_choice is "required"
|
||||
|
|
@ -97,7 +103,7 @@ def test_document_inlining_example(disable_add_transform_inline_image_block):
|
|||
with patch.object(client, "post") as mock_post:
|
||||
try:
|
||||
completion(
|
||||
model="fireworks_ai/accounts/fireworks/models/minimax-m3",
|
||||
model=f"fireworks_ai/{VISION_MODEL}",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
|
|
@ -157,7 +163,7 @@ def test_transform_inline_no_longer_added(content, expected_url):
|
|||
|
||||
result = litellm.FireworksAIConfig()._transform_messages_helper(
|
||||
messages=messages,
|
||||
model="accounts/fireworks/models/minimax-m3",
|
||||
model=VISION_MODEL,
|
||||
litellm_params={},
|
||||
)
|
||||
result_image_block = result[0]["content"][0]
|
||||
|
|
@ -182,7 +188,7 @@ def test_global_disable_flag_no_longer_adds_transform_inline(is_disabled):
|
|||
]
|
||||
result = litellm.FireworksAIConfig()._transform_messages_helper(
|
||||
messages=messages,
|
||||
model="accounts/fireworks/models/minimax-m3",
|
||||
model=VISION_MODEL,
|
||||
litellm_params={},
|
||||
)
|
||||
assert result[0]["content"][0]["image_url"] == url
|
||||
|
|
@ -204,7 +210,7 @@ def test_global_disable_flag_with_transform_messages_helper(monkeypatch):
|
|||
) as mock_post:
|
||||
try:
|
||||
completion(
|
||||
model="fireworks_ai/accounts/fireworks/models/minimax-m3",
|
||||
model=f"fireworks_ai/{VISION_MODEL}",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
|
|
|
|||
|
|
@ -238,7 +238,7 @@ def test_gemini_image_generation_accumulates_multiple_image_prompt_token_details
|
|||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
model = "gemini/gemini-3-pro-image-preview"
|
||||
model = "gemini/gemini-3-pro-image"
|
||||
config = GoogleImageGenConfig()
|
||||
|
||||
usage_metadata = {
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ class TestGroq(BaseLLMChatTest):
|
|||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["groq/qwen/qwen3-32b", "groq/openai/gpt-oss-20b", "groq/openai/gpt-oss-120b"],
|
||||
["groq/qwen/qwen3.8-27b", "groq/openai/gpt-oss-20b", "groq/openai/gpt-oss-120b"],
|
||||
)
|
||||
def test_reasoning_effort_in_supported_params(self, model):
|
||||
"""Test that reasoning_effort is in the list of supported parameters for Groq"""
|
||||
|
|
|
|||
|
|
@ -537,7 +537,7 @@ def test_dynamic_drop_params_e2e():
|
|||
) as mock_response:
|
||||
try:
|
||||
response = litellm.completion(
|
||||
model="command-r",
|
||||
model="command-r-08-2024",
|
||||
messages=[{"role": "user", "content": "Hey, how's it going?"}],
|
||||
response_format={"key": "value"},
|
||||
drop_params=True,
|
||||
|
|
@ -556,7 +556,7 @@ def test_dynamic_pass_additional_params():
|
|||
) as mock_response:
|
||||
try:
|
||||
response = litellm.completion(
|
||||
model="command-r",
|
||||
model="command-r-08-2024",
|
||||
messages=[{"role": "user", "content": "Hey, how's it going?"}],
|
||||
custom_param="test",
|
||||
api_key="my-custom-key",
|
||||
|
|
@ -606,7 +606,7 @@ def test_dynamic_drop_params_parallel_tool_calls():
|
|||
) as mock_response:
|
||||
try:
|
||||
response = litellm.completion(
|
||||
model="command-r",
|
||||
model="command-r-08-2024",
|
||||
messages=[{"role": "user", "content": "Hey, how's it going?"}],
|
||||
parallel_tool_calls=True,
|
||||
drop_params=True,
|
||||
|
|
@ -663,7 +663,7 @@ def test_dynamic_drop_additional_params_e2e():
|
|||
) as mock_response:
|
||||
try:
|
||||
response = litellm.completion(
|
||||
model="command-r",
|
||||
model="command-r-08-2024",
|
||||
messages=[{"role": "user", "content": "Hey, how's it going?"}],
|
||||
response_format={"key": "value"},
|
||||
additional_drop_params=["response_format"],
|
||||
|
|
|
|||
|
|
@ -164,7 +164,7 @@ def test_xai_message_name_filtering():
|
|||
class TestXAIReasoningEffort(BaseReasoningLLMTests):
|
||||
def get_base_completion_call_args(self):
|
||||
return {
|
||||
"model": "xai/grok-3-mini-beta",
|
||||
"model": "xai/grok-4.7",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2863,7 +2863,7 @@ def test_gemini_function_call_parameter_in_messages():
|
|||
mock_client.return_value = mock_response
|
||||
try:
|
||||
completion(
|
||||
model="vertex_ai/gemini-2.0-flash",
|
||||
model="vertex_ai/gemini-2.5-flash-preview-09-2025",
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
|
|
@ -3263,7 +3263,7 @@ def test_vertex_anthropic_completion():
|
|||
client, "post", side_effect=vertex_ai_anthropic_thinking_mock_response
|
||||
):
|
||||
response = completion(
|
||||
model="vertex_ai/claude-3-7-sonnet@20250219",
|
||||
model="vertex_ai/claude-sonnet-4-6@default",
|
||||
messages=[{"role": "user", "content": "Hello, world!"}],
|
||||
vertex_ai_location="us-east5",
|
||||
vertex_ai_project="test-project",
|
||||
|
|
@ -3271,7 +3271,7 @@ def test_vertex_anthropic_completion():
|
|||
client=client,
|
||||
)
|
||||
print(response)
|
||||
assert response.model == "claude-3-7-sonnet@20250219"
|
||||
assert response.model == "claude-sonnet-4-6@default"
|
||||
assert response._hidden_params["response_cost"] is not None
|
||||
assert response._hidden_params["response_cost"] > 0
|
||||
|
||||
|
|
|
|||
|
|
@ -445,7 +445,7 @@ def test_groq_response_cost_tracking(is_streaming):
|
|||
|
||||
response_cost = litellm.response_cost_calculator(
|
||||
response_object=response,
|
||||
model="groq/llama-3.3-70b-versatile",
|
||||
model="groq/openai/gpt-oss-120b",
|
||||
custom_llm_provider="groq",
|
||||
call_type=CallTypes.acompletion.value,
|
||||
optional_params={},
|
||||
|
|
@ -515,7 +515,7 @@ def test_gemini_completion_cost(provider):
|
|||
"""
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
model_name = "gemini-2.0-flash"
|
||||
model_name = "gemini-3.8-flash"
|
||||
prompt_tokens = 128.0
|
||||
output_tokens = 228.0
|
||||
## GET MODEL FROM LITELLM.MODEL_INFO
|
||||
|
|
@ -543,7 +543,7 @@ def test_vertex_ai_completion_cost():
|
|||
|
||||
prompt_tokens = 100
|
||||
|
||||
model_info = litellm.get_model_info(model="gemini-2.0-flash")
|
||||
model_info = litellm.get_model_info(model="gemini-3.8-flash")
|
||||
|
||||
print("\nExpected model info:\n{}\n\n".format(model_info))
|
||||
|
||||
|
|
@ -551,7 +551,7 @@ def test_vertex_ai_completion_cost():
|
|||
|
||||
## CALCULATED COST
|
||||
calculated_input_cost, calculated_output_cost = cost_per_token(
|
||||
model="gemini-2.0-flash",
|
||||
model="gemini-3.8-flash",
|
||||
custom_llm_provider="vertex_ai",
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=0,
|
||||
|
|
@ -676,7 +676,7 @@ async def test_completion_cost_hidden_params(sync_mode):
|
|||
|
||||
|
||||
def test_vertex_ai_gemini_predict_cost():
|
||||
model = "gemini-2.0-flash"
|
||||
model = "gemini-3.8-flash"
|
||||
messages = [{"role": "user", "content": "Hey, hows it going???"}]
|
||||
predictive_cost = completion_cost(model=model, messages=messages)
|
||||
|
||||
|
|
@ -757,24 +757,24 @@ def test_completion_cost_tts(model):
|
|||
|
||||
def test_completion_cost_anthropic():
|
||||
"""
|
||||
model_name: claude-3-haiku-20240307
|
||||
model_name: claude-haiku-4-5
|
||||
litellm_params:
|
||||
model: anthropic/claude-3-haiku-20240307
|
||||
model: anthropic/claude-haiku-4-5
|
||||
max_tokens: 4096
|
||||
"""
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "claude-3-haiku-20240307",
|
||||
"model_name": "claude-haiku-4-5",
|
||||
"litellm_params": {
|
||||
"model": "anthropic/claude-3-haiku-20240307",
|
||||
"model": "anthropic/claude-haiku-4-5",
|
||||
"max_tokens": 4096,
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
data = {
|
||||
"model": "claude-3-haiku-20240307",
|
||||
"model": "claude-haiku-4-5",
|
||||
"prompt_tokens": 21,
|
||||
"completion_tokens": 20,
|
||||
"response_time_ms": 871.7040000000001,
|
||||
|
|
@ -2068,14 +2068,14 @@ def test_completion_cost_params():
|
|||
"""
|
||||
litellm.set_verbose = True
|
||||
resp1_prompt_cost, resp1_completion_cost = cost_per_token(
|
||||
model="gemini-2.0-flash",
|
||||
model="gemini-3.8-flash",
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=1000,
|
||||
custom_llm_provider="vertex_ai_beta",
|
||||
)
|
||||
|
||||
resp2_prompt_cost, resp2_completion_cost = cost_per_token(
|
||||
model="gemini-2.0-flash", prompt_tokens=1000, completion_tokens=1000
|
||||
model="gemini-3.8-flash", prompt_tokens=1000, completion_tokens=1000
|
||||
)
|
||||
|
||||
assert resp2_prompt_cost > 0
|
||||
|
|
@ -2084,7 +2084,7 @@ def test_completion_cost_params():
|
|||
assert resp1_completion_cost == resp2_completion_cost
|
||||
|
||||
resp3_prompt_cost, resp3_completion_cost = cost_per_token(
|
||||
model="vertex_ai/gemini-2.0-flash", prompt_tokens=1000, completion_tokens=1000
|
||||
model="vertex_ai/gemini-3.8-flash", prompt_tokens=1000, completion_tokens=1000
|
||||
)
|
||||
|
||||
assert resp3_prompt_cost > 0
|
||||
|
|
@ -2102,14 +2102,14 @@ def test_completion_cost_params_2():
|
|||
prompt_tokens = 1000
|
||||
completion_tokens = 1000
|
||||
resp1_prompt_cost, resp1_completion_cost = cost_per_token(
|
||||
model="gemini-2.0-flash",
|
||||
model="gemini-3.8-flash",
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
)
|
||||
|
||||
print(resp1_prompt_cost, resp1_completion_cost)
|
||||
|
||||
model_info = litellm.get_model_info("gemini-2.0-flash")
|
||||
model_info = litellm.get_model_info("gemini-3.8-flash")
|
||||
input_cost_per_token = model_info["input_cost_per_token"]
|
||||
output_cost_per_token = model_info["output_cost_per_token"]
|
||||
|
||||
|
|
@ -2148,7 +2148,7 @@ def test_completion_cost_params_gemini_3():
|
|||
)
|
||||
],
|
||||
created=1728529259,
|
||||
model="gemini-2.0-flash",
|
||||
model="gemini-3.8-flash",
|
||||
object="chat.completion",
|
||||
system_fingerprint=None,
|
||||
usage=usage,
|
||||
|
|
@ -2172,7 +2172,7 @@ def test_completion_cost_params_gemini_3():
|
|||
|
||||
pc, cc = cost_per_character(
|
||||
**{
|
||||
"model": "gemini-2.0-flash",
|
||||
"model": "gemini-3.8-flash",
|
||||
"custom_llm_provider": "vertex_ai",
|
||||
"prompt_characters": None,
|
||||
"completion_characters": 3,
|
||||
|
|
@ -2180,9 +2180,9 @@ def test_completion_cost_params_gemini_3():
|
|||
}
|
||||
)
|
||||
|
||||
model_info = litellm.get_model_info("gemini-2.0-flash")
|
||||
model_info = litellm.get_model_info("gemini-3.8-flash")
|
||||
|
||||
# gemini-2.0-flash has no per-character pricing, so cost_per_character
|
||||
# gemini-3.8-flash has no per-character pricing, so cost_per_character
|
||||
# falls back to per-token pricing using usage.prompt_tokens / usage.completion_tokens
|
||||
assert round(pc, 10) == round(3771 * model_info["input_cost_per_token"], 10)
|
||||
assert round(cc, 10) == round(
|
||||
|
|
@ -2239,16 +2239,16 @@ async def test_test_completion_cost_gpt4o_audio_output_from_model(stream):
|
|||
)
|
||||
],
|
||||
created=1729282652,
|
||||
model="gpt-4o-audio-preview",
|
||||
model="gpt-audio-1.5",
|
||||
object="chat.completion",
|
||||
system_fingerprint="fp_4eafc16e9d",
|
||||
usage=usage_object,
|
||||
service_tier=None,
|
||||
)
|
||||
|
||||
cost = completion_cost(completion, model="gpt-4o-audio-preview")
|
||||
cost = completion_cost(completion, model="gpt-audio-1.5")
|
||||
|
||||
model_info = litellm.get_model_info("gpt-4o-audio-preview")
|
||||
model_info = litellm.get_model_info("gpt-audio-1.5")
|
||||
print(f"model_info: {model_info}")
|
||||
## input cost
|
||||
|
||||
|
|
@ -2517,7 +2517,7 @@ def test_cost_calculator_with_base_model():
|
|||
resp = litellm.completion(
|
||||
model="bedrock/random-model",
|
||||
messages=[{"role": "user", "content": "Hello, how are you?"}],
|
||||
base_model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
base_model="bedrock/anthropic.claude-sonnet-5",
|
||||
mock_response="Hello, how are you?",
|
||||
)
|
||||
assert resp.model == "random-model"
|
||||
|
|
@ -2551,10 +2551,10 @@ def test_cost_calculator_with_base_model_with_router(base_model_arg):
|
|||
if base_model_arg == "litellm_param":
|
||||
model_item["litellm_params"][
|
||||
"base_model"
|
||||
] = "bedrock/anthropic.claude-3-sonnet-20240229-v1:0"
|
||||
] = "bedrock/anthropic.claude-sonnet-5"
|
||||
elif base_model_arg == "model_info":
|
||||
model_item["model_info"] = {
|
||||
"base_model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
"base_model": "bedrock/anthropic.claude-sonnet-5",
|
||||
}
|
||||
|
||||
router = Router(model_list=[model_item])
|
||||
|
|
|
|||
|
|
@ -1148,7 +1148,7 @@ def test_openai_gateway_timeout_error():
|
|||
@pytest.mark.parametrize(
|
||||
"provider, model, call_type",
|
||||
[
|
||||
("anthropic", "claude-3-haiku-20240307", "chat_completion"),
|
||||
("anthropic", "claude-haiku-4-5-20251001", "chat_completion"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -136,7 +136,7 @@ def trade(model_name: str) -> List[Trade]: # type: ignore
|
|||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model", ["claude-haiku-4-5-20251001", "anthropic.claude-3-haiku-20240307-v1:0"]
|
||||
"model", ["claude-haiku-4-5-20251001", "us.anthropic.claude-haiku-4-5-20251001-v1:0"]
|
||||
)
|
||||
@pytest.mark.flaky(retries=6, delay=10)
|
||||
def test_function_call_parsing(model):
|
||||
|
|
|
|||
|
|
@ -67,7 +67,17 @@ def test_get_llm_provider_deepseek_custom_api_base():
|
|||
os.environ.pop("DEEPSEEK_API_BASE")
|
||||
|
||||
|
||||
def test_get_llm_provider_vertex_ai_image_models():
|
||||
def test_get_llm_provider_vertex_ai_image_models(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "vertex_ai_image_models", set())
|
||||
monkeypatch.setattr(litellm, "models_by_provider", dict(litellm.models_by_provider))
|
||||
litellm.add_known_models(
|
||||
model_cost_map={
|
||||
"vertex_ai/imagegeneration@006": {
|
||||
"litellm_provider": "vertex_ai-image-models",
|
||||
"mode": "image_generation",
|
||||
}
|
||||
}
|
||||
)
|
||||
model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider(
|
||||
model="imagegeneration@006", custom_llm_provider=None
|
||||
)
|
||||
|
|
@ -101,17 +111,17 @@ def test_get_llm_provider_ai21_chat_test2():
|
|||
|
||||
def test_get_llm_provider_cohere_chat_test2():
|
||||
"""
|
||||
if user prefix with cohere/ but calls command-r-plus then it should be cohere_chat provider
|
||||
if user prefix with cohere/ but calls command-r-plus-08-2024 then it should be cohere_chat provider
|
||||
"""
|
||||
model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider(
|
||||
model="cohere/command-r-plus",
|
||||
model="cohere/command-r-plus-08-2024",
|
||||
)
|
||||
|
||||
print("model=", model)
|
||||
print("custom_llm_provider=", custom_llm_provider)
|
||||
print("api_base=", api_base)
|
||||
assert custom_llm_provider == "cohere_chat"
|
||||
assert model == "command-r-plus"
|
||||
assert model == "command-r-plus-08-2024"
|
||||
|
||||
|
||||
def test_get_llm_provider_azure_o1():
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ def test_get_model_info_simple_model_name():
|
|||
"""
|
||||
tests if model name given, and model exists in model info - the object is returned
|
||||
"""
|
||||
model = "claude-3-opus-20240229"
|
||||
model = "claude-opus-5-5"
|
||||
litellm.get_model_info(model)
|
||||
|
||||
|
||||
|
|
@ -24,7 +24,7 @@ def test_get_model_info_custom_llm_with_model_name():
|
|||
"""
|
||||
Tests if {custom_llm_provider}/{model_name} name given, and model exists in model info, the object is returned
|
||||
"""
|
||||
model = "anthropic/claude-3-opus-20240229"
|
||||
model = "anthropic/claude-opus-5-5"
|
||||
litellm.get_model_info(model)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ async def test_get_available_deployments():
|
|||
},
|
||||
{
|
||||
"model_name": "gpt-3.5-turbo",
|
||||
"litellm_params": {"model": "groq/llama-3.1-8b-instant"},
|
||||
"litellm_params": {"model": "groq/openai/gpt-oss-20b"},
|
||||
"model_info": {"id": "groq-llama"},
|
||||
},
|
||||
]
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ async def test_openai_moderation_error_raising(monkeypatch):
|
|||
from unittest.mock import AsyncMock, MagicMock
|
||||
from litellm.types.llms.openai import OpenAIModerationResponse
|
||||
|
||||
litellm.openai_moderations_model_name = "text-moderation-latest"
|
||||
litellm.openai_moderations_model_name = "omni-moderation-latest"
|
||||
openai_mod = _ENTERPRISE_OpenAI_Moderation()
|
||||
_api_key = "sk-12345"
|
||||
_api_key = hash_token("sk-12345")
|
||||
|
|
@ -41,9 +41,9 @@ async def test_openai_moderation_error_raising(monkeypatch):
|
|||
llm_router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "text-moderation-latest",
|
||||
"model_name": "omni-moderation-latest",
|
||||
"litellm_params": {
|
||||
"model": "text-moderation-latest",
|
||||
"model": "omni-moderation-latest",
|
||||
"api_key": os.environ.get("OPENAI_API_KEY", "fake-key"),
|
||||
},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1284,15 +1284,18 @@ def test_model_group_info():
|
|||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "command-r-plus",
|
||||
"litellm_params": {"model": "cohere.command-r-plus-v1:0"},
|
||||
"model_name": "nova-2-lite",
|
||||
"litellm_params": {"model": "bedrock/amazon.nova-2-lite-v1:0"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
response = router.get_model_group_info(model_group="command-r-plus")
|
||||
response = router.get_model_group_info(model_group="nova-2-lite")
|
||||
|
||||
assert response is not None
|
||||
assert response.model_group == "nova-2-lite"
|
||||
assert response.providers == ["bedrock"]
|
||||
assert response.max_input_tokens is not None
|
||||
|
||||
|
||||
def test_consistent_model_id():
|
||||
|
|
|
|||
|
|
@ -188,7 +188,7 @@ def test_router_get_model_info_wildcard_routes():
|
|||
]
|
||||
)
|
||||
model_info = router.get_router_model_info(
|
||||
deployment=None, received_model_name="gemini/gemini-1.5-flash", id="1"
|
||||
deployment=None, received_model_name="gemini/gemini-2.5-flash", id="1"
|
||||
)
|
||||
print(model_info)
|
||||
assert model_info is not None
|
||||
|
|
@ -212,7 +212,7 @@ async def test_router_get_model_group_usage_wildcard_routes():
|
|||
)
|
||||
|
||||
resp = await router.acompletion(
|
||||
model="gemini/gemini-1.5-flash",
|
||||
model="gemini/gemini-2.5-flash",
|
||||
messages=[{"role": "user", "content": "Hello, how are you?"}],
|
||||
mock_response="Hello, I'm good.",
|
||||
)
|
||||
|
|
@ -220,7 +220,7 @@ async def test_router_get_model_group_usage_wildcard_routes():
|
|||
|
||||
await asyncio.sleep(2)
|
||||
|
||||
tpm, rpm = await router.get_model_group_usage(model_group="gemini/gemini-1.5-flash")
|
||||
tpm, rpm = await router.get_model_group_usage(model_group="gemini/gemini-2.5-flash")
|
||||
|
||||
assert tpm is not None, "tpm is None"
|
||||
assert rpm is not None, "rpm is None"
|
||||
|
|
@ -242,7 +242,7 @@ async def test_call_router_callbacks_on_success():
|
|||
router.cache, "async_increment_cache_pipeline", new=AsyncMock()
|
||||
) as mock_callback:
|
||||
await router.acompletion(
|
||||
model="gemini/gemini-1.5-flash",
|
||||
model="gemini/gemini-2.5-flash",
|
||||
messages=[{"role": "user", "content": "Hello, how are you?"}],
|
||||
mock_response="Hello, I'm good.",
|
||||
)
|
||||
|
|
@ -255,12 +255,12 @@ async def test_call_router_callbacks_on_success():
|
|||
for increment in increment_list:
|
||||
if "tpm" in increment["key"]:
|
||||
assert increment["key"].startswith(
|
||||
"global_router:1:gemini/gemini-1.5-flash:tpm"
|
||||
"global_router:1:gemini/gemini-2.5-flash:tpm"
|
||||
)
|
||||
assert increment["increment_value"] == 30
|
||||
elif "rpm" in increment["key"]:
|
||||
assert increment["key"].startswith(
|
||||
"global_router:1:gemini/gemini-1.5-flash:rpm"
|
||||
"global_router:1:gemini/gemini-2.5-flash:rpm"
|
||||
)
|
||||
assert increment["increment_value"] == 1
|
||||
|
||||
|
|
@ -283,7 +283,7 @@ async def test_call_router_callbacks_on_failure():
|
|||
) as mock_callback:
|
||||
with pytest.raises(litellm.RateLimitError):
|
||||
await router.acompletion(
|
||||
model="gemini/gemini-1.5-flash",
|
||||
model="gemini/gemini-2.5-flash",
|
||||
messages=[{"role": "user", "content": "Hello, how are you?"}],
|
||||
mock_response="litellm.RateLimitError",
|
||||
num_retries=0,
|
||||
|
|
@ -295,7 +295,7 @@ async def test_call_router_callbacks_on_failure():
|
|||
assert (
|
||||
mock_callback.call_args_list[0]
|
||||
.kwargs["key"]
|
||||
.startswith("global_router:1:gemini/gemini-1.5-flash:rpm")
|
||||
.startswith("global_router:1:gemini/gemini-2.5-flash:rpm")
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -317,7 +317,7 @@ async def test_router_model_group_headers():
|
|||
|
||||
for _ in range(2):
|
||||
resp = await router.acompletion(
|
||||
model="gemini/gemini-1.5-flash",
|
||||
model="gemini/gemini-2.5-flash",
|
||||
messages=[{"role": "user", "content": "Hello, how are you?"}],
|
||||
mock_response="Hello, I'm good.",
|
||||
)
|
||||
|
|
@ -325,7 +325,7 @@ async def test_router_model_group_headers():
|
|||
|
||||
assert (
|
||||
resp._hidden_params["additional_headers"]["x-litellm-model-group"]
|
||||
== "gemini/gemini-1.5-flash"
|
||||
== "gemini/gemini-2.5-flash"
|
||||
)
|
||||
|
||||
assert "x-ratelimit-remaining-requests" in resp._hidden_params["additional_headers"]
|
||||
|
|
@ -349,7 +349,7 @@ async def test_get_remaining_model_group_usage():
|
|||
)
|
||||
for _ in range(2):
|
||||
resp = await router.acompletion(
|
||||
model="gemini/gemini-1.5-flash",
|
||||
model="gemini/gemini-2.5-flash",
|
||||
messages=[{"role": "user", "content": "Hello, how are you?"}],
|
||||
mock_response="Hello, I'm good.",
|
||||
)
|
||||
|
|
@ -363,7 +363,7 @@ async def test_get_remaining_model_group_usage():
|
|||
await asyncio.sleep(1)
|
||||
|
||||
remaining_usage = await router.get_remaining_model_group_usage(
|
||||
model_group="gemini/gemini-1.5-flash"
|
||||
model_group="gemini/gemini-2.5-flash"
|
||||
)
|
||||
assert remaining_usage is not None
|
||||
assert "x-ratelimit-remaining-requests" in remaining_usage
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ async def test_spend_calc_model_on_router_messages():
|
|||
{
|
||||
"model_name": "special-llama-model",
|
||||
"litellm_params": {
|
||||
"model": "groq/llama-3.1-8b-instant",
|
||||
"model": "groq/openai/gpt-oss-20b",
|
||||
},
|
||||
}
|
||||
]
|
||||
|
|
@ -81,7 +81,7 @@ async def test_spend_calc_using_response():
|
|||
}
|
||||
],
|
||||
"created": "1677652288",
|
||||
"model": "groq/llama-3.1-8b-instant",
|
||||
"model": "groq/openai/gpt-oss-20b",
|
||||
"object": "chat.completion",
|
||||
"system_fingerprint": "fp_873a560973",
|
||||
"usage": {
|
||||
|
|
|
|||
|
|
@ -31,14 +31,14 @@
|
|||
"model_id": null,
|
||||
"cache_key": null,
|
||||
"api_base": null,
|
||||
"response_cost": 7.5e-06,
|
||||
"response_cost": 3.5e-05,
|
||||
"additional_headers": {},
|
||||
"litellm_overhead_time_ms": null,
|
||||
"batch_models": null,
|
||||
"litellm_model_name": "vertex_ai/gemini-2.0-flash-001",
|
||||
"litellm_model_name": "vertex_ai/gemini-3-flash-preview",
|
||||
"usage_object": null
|
||||
},
|
||||
"litellm_response_cost": 7.5e-06,
|
||||
"litellm_response_cost": 3.5e-05,
|
||||
"cache_hit": false,
|
||||
"requester_metadata": {}
|
||||
},
|
||||
|
|
@ -54,13 +54,13 @@
|
|||
"id": "time-14-15-40-349639_chatcmpl-59a988d0-7ef1-4dc4-bc18-d2e78961817f",
|
||||
"endTime": "2025-05-26T14:15:40.607266-07:00",
|
||||
"completionStartTime": "2025-05-26T14:15:40.607266-07:00",
|
||||
"model": "gemini-2.0-flash-001",
|
||||
"model": "gemini-3-flash-preview",
|
||||
"modelParameters": {},
|
||||
"usage": {
|
||||
"input": 10,
|
||||
"output": 10,
|
||||
"unit": "TOKENS",
|
||||
"totalCost": 7.5e-06
|
||||
"totalCost": 3.5e-05
|
||||
},
|
||||
"usageDetails": {
|
||||
"input": 10,
|
||||
|
|
|
|||
|
|
@ -582,7 +582,7 @@ async def test_webhook_alerting(alerting_type):
|
|||
None,
|
||||
None,
|
||||
),
|
||||
("gemini-2.0-flash", None, "vertex_ai", "hardy-device-38811", "us-central1"),
|
||||
("gemini-3.8-flash", None, "vertex_ai", "hardy-device-38811", "us-central1"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("error_code", [500, 408, 400])
|
||||
|
|
@ -688,7 +688,7 @@ async def test_outage_alerting_called(
|
|||
None,
|
||||
None,
|
||||
),
|
||||
("gemini-2.0-flash", None, "vertex_ai", "hardy-device-38811", "us-central1"),
|
||||
("gemini-3.8-flash", None, "vertex_ai", "hardy-device-38811", "us-central1"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("error_code", [500, 408, 400])
|
||||
|
|
@ -775,7 +775,7 @@ async def test_region_outage_alerting_called(
|
|||
await slack_alerting.region_outage_alerts(
|
||||
exception=error_to_raise, deployment_id=deployment_id # type: ignore
|
||||
)
|
||||
if model == "gemini-2.0-flash" and (error_code == 500 or error_code == 408):
|
||||
if model == "gemini-3.8-flash" and (error_code == 500 or error_code == 408):
|
||||
mock_send_alert.assert_called_once()
|
||||
else:
|
||||
mock_send_alert.assert_not_called()
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue