refactor(model-cost): fetch and reload the cost map through Rust

model-catalog-refresh fetches over HTTP, S3 and file with main's retry
rules, python-bridge drives it with the GIL released, and provenance is
one Rust snapshot recorded by install. The S3 client setup moves into a
shared s3 crate used by cache-s3.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yujong Lee 2026-10-28 00:00:42 -07:00
parent 713b69b8d4
commit 18d2c77042
38 changed files with 2451 additions and 1307 deletions

View file

@ -3618,6 +3618,7 @@ dependencies = [
"litellm-cache",
"litellm-cache-testing",
"litellm-http",
"litellm-s3",
"reqwest 0.12.28",
"rstest",
"serde_json",
@ -4255,8 +4256,22 @@ dependencies = [
name = "litellm-model-catalog-refresh"
version = "0.1.0"
dependencies = [
"aws-config",
"aws-credential-types",
"aws-sdk-s3",
"aws-smithy-runtime-api",
"litellm-http",
"litellm-model-catalog",
"litellm-s3",
"rand 0.8.7",
"reqwest 0.12.28",
"rstest",
"serde_json",
"sha1 0.10.7",
"thiserror 2.0.19",
"tokio",
"url",
"wiremock",
]
[[package]]
@ -4289,6 +4304,7 @@ dependencies = [
"litellm-llms",
"litellm-llms-types",
"litellm-model-catalog",
"litellm-model-catalog-refresh",
"litellm-router-types",
"litellm-secrets",
"litellm-secrets-aws",
@ -4306,7 +4322,6 @@ dependencies = [
"serde",
"serde_json",
"serde_with",
"sha1 0.10.7",
"sqlx",
"strum",
"thiserror 2.0.19",
@ -4376,6 +4391,19 @@ dependencies = [
"serde_with",
]
[[package]]
name = "litellm-s3"
version = "0.1.0"
dependencies = [
"aws-credential-types",
"aws-sdk-s3",
"aws-smithy-runtime-api",
"aws-smithy-types",
"http 1.4.2",
"litellm-http",
"reqwest 0.12.28",
]
[[package]]
name = "litellm-secrets"
version = "0.1.0"

View file

@ -75,6 +75,7 @@ litellm-redis-testing = { path = "crates/redis-testing" }
litellm-cost = { path = "crates/cost" }
litellm-model-catalog = { path = "crates/model-catalog" }
litellm-model-catalog-refresh = { path = "crates/model-catalog-refresh" }
litellm-s3 = { path = "crates/s3" }
askama = { version = "0.16.1", default-features = false, features = ["derive", "std"] }
tracing = "0.1"

View file

@ -9,6 +9,7 @@ repository.workspace = true
litellm-http.workspace = true
litellm-cache.workspace = true
litellm-auth-aws.workspace = true
litellm-s3.workspace = true
aws-sdk-s3 = { version = "1.146.1", default-features = false, features = ["rustls", "rt-tokio"] }
aws-credential-types = "1.3.0"
aws-smithy-runtime-api = { version = "1.16.2", features = ["client", "http-1x"] }

View file

@ -4,11 +4,7 @@ use std::{
time::{Duration, SystemTime},
};
use aws_sdk_s3::{
config::{BehaviorVersion, Region, RequestChecksumCalculation, ResponseChecksumValidation},
error::SdkError,
primitives::ByteStream,
};
use aws_sdk_s3::{error::SdkError, primitives::ByteStream};
use aws_smithy_types::{DateTime, date_time::Format};
use futures_util::future::try_join_all;
use litellm_auth_aws::AwsAuthConfig;
@ -17,7 +13,9 @@ use litellm_cache::{
};
use tokio::runtime::Handle;
use crate::{auth::S3Credentials, transport::ReqwestHttpClient};
use litellm_s3::S3ClientConfig;
use crate::auth::S3Credentials;
pub struct S3Endpoint {
pub url: String,
@ -49,19 +47,16 @@ impl<C: CacheCodec> S3Cache<C> {
runtime: Handle,
) -> Self {
let endpoint_url: Option<String> = config.endpoint.map(|endpoint| endpoint.url);
let base = aws_sdk_s3::Config::builder()
.behavior_version(BehaviorVersion::latest())
.region(Region::new(config.region.clone()))
.http_client(ReqwestHttpClient(http))
.credentials_provider(S3Credentials::new(config.auth))
.request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
.response_checksum_validation(ResponseChecksumValidation::WhenRequired);
let builder = match &endpoint_url {
Some(url) => base.endpoint_url(url).force_path_style(true),
None => base,
};
let client = litellm_s3::client(
S3ClientConfig {
region: config.region.clone(),
endpoint: endpoint_url.clone(),
},
http,
S3Credentials::new(config.auth),
);
Self {
client: aws_sdk_s3::Client::from_conf(builder.build()),
client,
codec,
runtime,
bucket: config.bucket.into(),

View file

@ -1,6 +1,5 @@
mod auth;
mod cache;
mod transport;
pub use auth::S3Credentials;
pub use cache::{S3Cache, S3CacheConfig, S3Endpoint};

View file

@ -6,5 +6,23 @@ license.workspace = true
repository.workspace = true
[dependencies]
aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"] }
aws-sdk-s3 = { version = "1.146.1", default-features = false, features = ["rustls", "rt-tokio"] }
aws-smithy-runtime-api = { version = "1.16.2", features = ["client", "http-1x"] }
litellm-model-catalog.workspace = true
litellm-s3.workspace = true
rand.workspace = true
litellm-http.workspace = true
reqwest.workspace = true
sha1.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["fs", "time"] }
url.workspace = true
[dev-dependencies]
litellm-http = { workspace = true, features = ["test-support"] }
aws-credential-types = "1.3.0"
rstest.workspace = true
serde_json.workspace = true
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
wiremock.workspace = true

View file

@ -0,0 +1,9 @@
use thiserror::Error;
#[derive(Clone, Debug, PartialEq, Eq, Error)]
pub enum SourceError {
#[error("unsupported model cost map URL scheme {scheme}")]
UnsupportedScheme { scheme: String },
#[error("invalid model cost map URL: {reason}")]
Invalid { reason: String },
}

View file

@ -0,0 +1,25 @@
use std::io::ErrorKind;
use std::path::Path;
use crate::step::Reply;
/// A file has no validator, so every read is a fresh 200
pub(crate) async fn fetch(path: &Path) -> Reply {
match tokio::fs::read(path).await {
Ok(body) => Reply::Response {
status: 200,
etag: None,
retry_after: None,
body: Some(body),
},
Err(error) if error.kind() == ErrorKind::NotFound => Reply::Response {
status: 404,
etag: None,
retry_after: None,
body: None,
},
Err(error) => Reply::Transport {
reason: format!("reading {} failed: {error}", path.display()),
},
}
}

View file

@ -0,0 +1,82 @@
use std::time::Duration;
use reqwest::header::{ETAG, IF_NONE_MATCH, RETRY_AFTER};
use url::Url;
use crate::step::{FetchRequest, Reply};
pub(crate) async fn fetch(
client: &litellm_http::Client,
url: &Url,
request: FetchRequest,
) -> Reply {
let builder = client.get(url.clone()).timeout(request.timeout);
let builder = match &request.if_none_match {
Some(etag) => builder.header(IF_NONE_MATCH, etag),
None => builder,
};
let response = match builder.send().await {
Ok(response) => response,
Err(error) => {
return Reply::Transport {
reason: format!("fetching {url} failed: {error}"),
};
}
};
let status = response.status().as_u16();
let etags: Vec<&str> = response
.headers()
.get_all(ETAG)
.iter()
.filter_map(|value| value.to_str().ok())
.collect();
let etag = (!etags.is_empty()).then(|| etags.join(", "));
let retry_after = response
.headers()
.get(RETRY_AFTER)
.and_then(|value| value.to_str().ok())
.and_then(retry_after_seconds);
if status != 200 {
return Reply::Response {
status,
etag,
retry_after,
body: None,
};
}
match response.bytes().await {
Ok(body) => Reply::Response {
status,
etag,
retry_after,
body: Some(body.to_vec()),
},
Err(error) => Reply::Transport {
reason: format!("reading {url} failed: {error}"),
},
}
}
/// Main reads `Retry-After` as a number of seconds and ignores an HTTP date or a negative number
fn retry_after_seconds(header: &str) -> Option<Duration> {
let seconds = header.trim().parse::<f64>().ok()?;
(seconds.is_finite() && seconds >= 0.0)
.then(|| Duration::try_from_secs_f64(seconds).ok())
.flatten()
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
#[rstest]
#[case::seconds("7", Some(Duration::from_secs(7)))]
#[case::fraction("0.5", Some(Duration::from_millis(500)))]
#[case::negative("-1", None)]
#[case::http_date("Wed, 21 Oct 2015 07:28:00 GMT", None)]
fn retry_after_is_seconds(#[case] header: &str, #[case] expected: Option<Duration>) {
assert_eq!(retry_after_seconds(header), expected);
}
}

View file

@ -1,7 +1,67 @@
mod error;
mod file;
mod http;
mod policy;
mod revalidate;
mod s3;
mod source;
mod step;
use std::future::Future;
use std::time::Duration;
pub use error::*;
pub use policy::*;
pub use revalidate::*;
pub use s3::*;
pub use source::*;
pub use step::*;
/// The transports a fetch can use. The HTTP client comes from the caller's pool, the S3 client is built per source
pub struct Transports<'a> {
pub http: &'a litellm_http::Client,
pub s3: Option<&'a S3Source>,
}
/// One attempt at `source`, never retried here
pub async fn fetch(source: &Source, request: FetchRequest, transports: &Transports<'_>) -> Reply {
match source {
Source::Http(url) => http::fetch(transports.http, url, request).await,
Source::File(path) => file::fetch(path).await,
Source::S3(object) => match transports.s3 {
Some(s3) => s3.fetch(request).await,
None => {
S3Source::from_environment(object.clone(), transports.http.clone())
.await
.fetch(request)
.await
}
},
}
}
/// Drives `Revalidation` to `Adopt` or `Done`, sleeping between attempts with the injected `sleep`
pub async fn fetch_with_retry<S, F>(
source: &Source,
installed: &Installed,
policy: RetryPolicy,
seed: Option<u64>,
transports: &Transports<'_>,
sleep: S,
) -> Step
where
S: Fn(Duration) -> F,
F: Future<Output = ()>,
{
let (mut revalidation, mut step) = Revalidation::new(source, installed, policy, seed);
loop {
step = match step {
Step::Fetch(request) => revalidation.on_reply(fetch(source, request, transports).await),
Step::Sleep(delay) => {
sleep(delay).await;
revalidation.on_slept()
}
done => return done,
};
}
}

View file

@ -18,3 +18,17 @@ pub struct RetryPolicy {
/// Main's default `LITELLM_MODEL_COST_MAP_URL`
pub const DEFAULT_REGISTRY_URL: &str =
"https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json";
impl Default for RetryPolicy {
fn default() -> Self {
Self {
max_attempts: 3,
max_wait: Duration::from_secs(30),
timeout: Duration::from_secs(5),
backoff_base_seconds: 2.0,
jitter_min_seconds: 0.0,
jitter_max_seconds: 1.0,
limits: IntegrityLimits::new(50, 0.5).expect("0.5 is a ratio"),
}
}
}

View file

@ -0,0 +1,410 @@
use std::sync::LazyLock;
use std::time::Duration;
use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog};
use rand::rngs::StdRng;
use rand::{Rng, SeedableRng};
use sha1::{Digest, Sha1};
use crate::policy::RetryPolicy;
use crate::source::Source;
use crate::step::{FetchRequest, Installed, Origin, Reply, Step};
/// Main's `RETRYABLE_FETCH_STATUS_CODES`
const RETRYABLE_STATUSES: [u16; 5] = [429, 500, 502, 503, 504];
static BUNDLED_MODEL_COUNT: LazyLock<usize> = LazyLock::new(|| {
Catalog::parse(BUNDLED_REGISTRY)
.expect("the bundled model registry parses")
.value
.model_count()
});
/// The fetch state machine. It asks its driver for an attempt or a sleep and never does IO itself
#[derive(Debug)]
pub struct Revalidation {
source: Source,
if_none_match: Option<String>,
policy: RetryPolicy,
attempts: u32,
rng: StdRng,
state: State,
}
#[derive(Debug)]
enum State {
AwaitingReply,
AwaitingSleep,
Done(String, Option<Duration>),
}
impl Revalidation {
pub fn new(
source: &Source,
installed: &Installed,
policy: RetryPolicy,
seed: Option<u64>,
) -> (Self, Step) {
let if_none_match = match &installed.origin {
Origin::Remote { source: from } if from == source => installed.etag.clone(),
Origin::Remote { .. } | Origin::Bundled => None,
};
let revalidation = Self {
source: source.clone(),
if_none_match,
policy,
attempts: 1,
rng: seed.map_or_else(StdRng::from_entropy, StdRng::seed_from_u64),
state: State::AwaitingReply,
};
let step = revalidation.fetch();
(revalidation, step)
}
pub fn on_reply(&mut self, reply: Reply) -> Step {
if !matches!(self.state, State::AwaitingReply) {
return self.invalid_state("revalidation is not awaiting a reply");
}
match reply {
Reply::Transport { reason } => self.retry_or_done(reason, None),
Reply::Response {
status: 200,
etag,
body,
..
} => self.adopt(body.unwrap_or_default(), etag),
Reply::Response { status: 304, .. } => {
self.finish(format!("{} is not modified", self.source))
}
Reply::Response {
status,
retry_after,
..
} if RETRYABLE_STATUSES.contains(&status) => {
self.retry_or_done(format!("HTTP {status} from {}", self.source), retry_after)
}
Reply::Response { status, .. } => {
self.finish(format!("HTTP {status} from {}", self.source))
}
}
}
pub fn on_slept(&mut self) -> Step {
if !matches!(self.state, State::AwaitingSleep) {
return self.invalid_state("revalidation is not awaiting a sleep");
}
self.attempts += 1;
self.state = State::AwaitingReply;
self.fetch()
}
fn fetch(&self) -> Step {
Step::Fetch(FetchRequest {
if_none_match: self.if_none_match.clone(),
timeout: self.policy.timeout,
})
}
fn adopt(&mut self, body: Vec<u8>, etag: Option<String>) -> Step {
let loaded = match Catalog::parse(&body) {
Ok(loaded) => loaded,
Err(error) => {
return self.finish(format!(
"model cost map from {} is unusable: {error}",
self.source
));
}
};
if let Err(error) = loaded
.value
.check_size(*BUNDLED_MODEL_COUNT, self.policy.limits)
{
return self.finish(format!(
"model cost map from {} is unusable: {error}",
self.source
));
}
let installed = Installed {
origin: Origin::Remote {
source: self.source.clone(),
},
etag,
revision: git_blob_id(&body),
model_count: loaded.value.model_count(),
};
self.state = State::Done("adopted".to_owned(), None);
Step::Adopt {
catalog: Box::new(loaded.value),
installed,
errors: loaded.errors,
}
}
fn retry_or_done(&mut self, reason: String, retry_after: Option<Duration>) -> Step {
let delay = retry_after
.unwrap_or_else(|| self.backoff())
.min(self.policy.max_wait);
if self.attempts >= self.policy.max_attempts {
let reason = if self.policy.max_attempts > 1 {
format!("{reason} (after {} attempts)", self.policy.max_attempts)
} else {
reason
};
self.state = State::Done(reason.clone(), Some(delay));
return Step::Done {
reason,
retry_after: Some(delay),
};
}
self.state = State::AwaitingSleep;
Step::Sleep(delay)
}
fn backoff(&mut self) -> Duration {
let jitter = if self.policy.jitter_max_seconds > self.policy.jitter_min_seconds {
self.rng
.gen_range(self.policy.jitter_min_seconds..=self.policy.jitter_max_seconds)
} else {
self.policy.jitter_min_seconds
};
let seconds = self.policy.backoff_base_seconds.powi(self.attempts as i32) + jitter;
Duration::try_from_secs_f64(seconds.min(self.policy.max_wait.as_secs_f64()))
.unwrap_or(self.policy.max_wait)
}
fn finish(&mut self, reason: String) -> Step {
self.state = State::Done(reason.clone(), None);
Step::Done {
reason,
retry_after: None,
}
}
fn invalid_state(&mut self, reason: &str) -> Step {
if let State::Done(previous, retry_after) = &self.state {
return Step::Done {
reason: previous.clone(),
retry_after: *retry_after,
};
}
self.finish(reason.to_owned())
}
}
/// The digest `git hash-object` prints, which main records as the source revision
pub fn git_blob_id(body: &[u8]) -> String {
let mut hasher = Sha1::new();
hasher.update(format!("blob {}\0", body.len()).as_bytes());
hasher.update(body);
hasher
.finalize()
.iter()
.map(|byte| format!("{byte:02x}"))
.collect()
}
#[cfg(test)]
mod tests {
use rstest::{fixture, rstest};
use super::*;
fn source() -> Source {
Source::parse("https://example.com/map.json").unwrap()
}
#[fixture]
fn bundled() -> Installed {
Installed {
origin: Origin::Bundled,
etag: None,
revision: git_blob_id(BUNDLED_REGISTRY),
model_count: *BUNDLED_MODEL_COUNT,
}
}
fn response(status: u16) -> Reply {
Reply::Response {
status,
etag: None,
retry_after: None,
body: None,
}
}
fn done(step: Step) -> String {
match step {
Step::Done { reason, .. } => reason,
other => panic!("expected done, got {other:?}"),
}
}
fn slept(step: Step) -> Duration {
match step {
Step::Sleep(wait) => wait,
other => panic!("expected a sleep, got {other:?}"),
}
}
fn started(installed: &Installed) -> Revalidation {
Revalidation::new(&source(), installed, RetryPolicy::default(), Some(0)).0
}
#[rstest]
#[case::too_many_requests(429)]
#[case::internal_error(500)]
#[case::bad_gateway(502)]
#[case::unavailable(503)]
#[case::gateway_timeout(504)]
fn mains_retryable_statuses_sleep_before_the_next_attempt(
bundled: Installed,
#[case] status: u16,
) {
let mut revalidation = started(&bundled);
assert!(matches!(
revalidation.on_reply(response(status)),
Step::Sleep(_)
));
}
#[rstest]
#[case::not_implemented(501)]
#[case::version_not_supported(505)]
#[case::not_found(404)]
fn other_statuses_are_final(bundled: Installed, #[case] status: u16) {
let mut revalidation = started(&bundled);
assert_eq!(
done(revalidation.on_reply(response(status))),
format!("HTTP {status} from https://example.com/map.json")
);
}
#[rstest]
#[case::honored(Duration::from_secs(7), Duration::from_secs(7))]
#[case::capped(Duration::from_secs(120), Duration::from_secs(30))]
fn retry_after_sets_the_wait_up_to_the_cap(
bundled: Installed,
#[case] header: Duration,
#[case] wait: Duration,
) {
let mut revalidation = started(&bundled);
let reply = Reply::Response {
status: 429,
etag: None,
retry_after: Some(header),
body: None,
};
assert_eq!(slept(revalidation.on_reply(reply)), wait);
}
#[rstest]
fn backoff_grows_from_two_seconds_with_jitter(bundled: Installed) {
let mut revalidation = started(&bundled);
let first = slept(revalidation.on_reply(response(503)));
revalidation.on_slept();
let second = slept(revalidation.on_reply(response(503)));
assert!((2.0..=3.0).contains(&first.as_secs_f64()), "{first:?}");
assert!((4.0..=5.0).contains(&second.as_secs_f64()), "{second:?}");
}
#[rstest]
fn the_last_attempt_names_how_many_were_made(bundled: Installed) {
let mut revalidation = started(&bundled);
revalidation.on_reply(response(503));
revalidation.on_slept();
revalidation.on_reply(response(503));
revalidation.on_slept();
assert_eq!(
done(revalidation.on_reply(Reply::Transport {
reason: "connection refused".into()
})),
"connection refused (after 3 attempts)"
);
}
#[rstest]
fn a_single_attempt_reports_the_wait_a_retry_should_use(bundled: Installed) {
let policy = RetryPolicy {
max_attempts: 1,
..RetryPolicy::default()
};
let (mut revalidation, _) = Revalidation::new(&source(), &bundled, policy, Some(0));
let reply = Reply::Response {
status: 503,
etag: None,
retry_after: Some(Duration::from_secs(4)),
body: None,
};
let Step::Done {
reason,
retry_after,
} = revalidation.on_reply(reply)
else {
panic!("expected done");
};
assert_eq!(reason, "HTTP 503 from https://example.com/map.json");
assert_eq!(retry_after, Some(Duration::from_secs(4)));
}
#[rstest]
fn a_body_that_is_not_a_catalog_is_final(bundled: Installed) {
let mut revalidation = started(&bundled);
let reply = Reply::Response {
status: 200,
etag: None,
retry_after: None,
body: Some(b"not json".to_vec()),
};
let reason = done(revalidation.on_reply(reply));
assert!(reason.contains("invalid JSON"), "{reason}");
}
#[rstest]
fn the_bundled_body_is_adopted_with_its_revision_and_etag(bundled: Installed) {
let mut revalidation = started(&bundled);
let reply = Reply::Response {
status: 200,
etag: Some("W/\"abc\"".into()),
retry_after: None,
body: Some(BUNDLED_REGISTRY.to_vec()),
};
let Step::Adopt { installed, .. } = revalidation.on_reply(reply) else {
panic!("expected adopt");
};
assert_eq!(installed.revision, bundled.revision);
assert_eq!(installed.etag.as_deref(), Some("W/\"abc\""));
assert_eq!(installed.origin, Origin::Remote { source: source() });
}
#[rstest]
#[case::same_source(Origin::Remote { source: source() }, Some("v1"))]
#[case::another_source(
Origin::Remote { source: Source::parse("https://example.com/other.json").unwrap() },
None
)]
#[case::bundled(Origin::Bundled, None)]
fn the_etag_is_sent_only_back_to_the_source_that_gave_it(
bundled: Installed,
#[case] origin: Origin,
#[case] expected: Option<&str>,
) {
let installed = Installed {
origin,
etag: Some("v1".into()),
..bundled
};
let (_, step) = Revalidation::new(&source(), &installed, RetryPolicy::default(), Some(0));
let Step::Fetch(request) = step else {
panic!("expected a fetch");
};
assert_eq!(request.if_none_match.as_deref(), expected);
}
#[rstest]
fn git_blob_id_is_what_git_hash_object_prints() {
assert_eq!(
git_blob_id(b"{\"gpt-5.4-mini\": {\"mode\": \"chat\"}}\n"),
"18b9a8381e13a3b38a2128f184f631f95829e987"
);
}
}

View file

@ -0,0 +1,171 @@
use aws_config::BehaviorVersion;
use aws_sdk_s3::config::SharedCredentialsProvider;
use aws_sdk_s3::config::retry::RetryConfig;
use aws_sdk_s3::error::SdkError;
use aws_smithy_runtime_api::http::Response;
use litellm_s3::{ReqwestHttpClient, S3ClientConfig};
use crate::source::S3Object;
use crate::step::{FetchRequest, Reply};
const BUCKET_REGION_HEADER: &str = "x-amz-bucket-region";
const DEFAULT_REGION: &str = "us-east-1";
/// One S3 object as a catalog source. SDK retries are off, so `RetryPolicy` owns every attempt
#[derive(Clone, Debug)]
pub struct S3Source {
object: S3Object,
http: litellm_http::Client,
credentials: SharedCredentialsProvider,
endpoint: Option<String>,
client: aws_sdk_s3::Client,
}
impl S3Source {
pub fn new(
object: S3Object,
http: litellm_http::Client,
credentials: SharedCredentialsProvider,
config: S3ClientConfig,
) -> Self {
let endpoint = config.endpoint.clone();
let client = without_retries(litellm_s3::client(
config,
http.clone(),
credentials.clone(),
));
Self {
object,
http,
credentials,
endpoint,
client,
}
}
/// The chain `boto3.client("s3")` used on main. Region from `AWS_REGION_NAME` first, then the SDK chain
/// (`AWS_REGION`, `AWS_DEFAULT_REGION`, the profile). `AWS_ENDPOINT_URL_S3` or `AWS_ENDPOINT_URL` for
/// S3-compatible stores
pub async fn from_environment(object: S3Object, http: litellm_http::Client) -> Self {
let shared = aws_config::defaults(BehaviorVersion::latest())
.http_client(ReqwestHttpClient(http.clone()))
.load()
.await;
let region = non_empty_env("AWS_REGION_NAME")
.or_else(|| shared.region().map(ToString::to_string))
.unwrap_or_else(|| DEFAULT_REGION.to_owned());
let endpoint =
non_empty_env("AWS_ENDPOINT_URL_S3").or_else(|| non_empty_env("AWS_ENDPOINT_URL"));
let credentials = shared.credentials_provider().unwrap_or_else(|| {
SharedCredentialsProvider::new(aws_sdk_s3::config::Credentials::new(
"",
"",
None,
None,
"no AWS credentials found",
))
});
Self::new(
object,
http,
credentials,
S3ClientConfig { region, endpoint },
)
}
pub async fn fetch(&self, request: FetchRequest) -> Reply {
match self.attempt(&self.client, &request).await {
Redirect::To(region) => {
let client = without_retries(litellm_s3::client(
S3ClientConfig {
region,
endpoint: self.endpoint.clone(),
},
self.http.clone(),
self.credentials.clone(),
));
match self.attempt(&client, &request).await {
Redirect::To(region) => Reply::Transport {
reason: format!("{} redirected twice, last to {region}", self.display()),
},
Redirect::No(reply) => reply,
}
}
Redirect::No(reply) => reply,
}
}
async fn attempt(&self, client: &aws_sdk_s3::Client, request: &FetchRequest) -> Redirect {
let call = client
.get_object()
.bucket(&self.object.bucket)
.key(&self.object.key)
.set_if_none_match(request.if_none_match.clone())
.send();
let output = match tokio::time::timeout(request.timeout, call).await {
Err(_) => {
return Redirect::No(Reply::Transport {
reason: format!("fetching {} timed out", self.display()),
});
}
Ok(Err(SdkError::ServiceError(error))) => return rejected(error.raw()),
Ok(Err(SdkError::ResponseError(error))) => return rejected(error.raw()),
Ok(Err(error)) => {
return Redirect::No(Reply::Transport {
reason: format!("fetching {} failed: {error}", self.display()),
});
}
Ok(Ok(output)) => output,
};
let etag = output.e_tag().map(str::to_owned);
match output.body.collect().await {
Ok(body) => Redirect::No(Reply::Response {
status: 200,
etag,
retry_after: None,
body: Some(body.into_bytes().to_vec()),
}),
Err(error) => Redirect::No(Reply::Transport {
reason: format!("reading {} failed: {error}", self.display()),
}),
}
}
fn display(&self) -> String {
format!("s3://{}/{}", self.object.bucket, self.object.key)
}
}
enum Redirect {
/// The bucket lives in another region, repeated once as boto3 does
To(String),
No(Reply),
}
fn rejected(raw: &Response) -> Redirect {
let status = raw.status().as_u16();
let header = |name: &str| raw.headers().get(name).map(str::to_owned);
match (status, header(BUCKET_REGION_HEADER)) {
(301, Some(region)) => Redirect::To(region),
_ => Redirect::No(Reply::Response {
status,
etag: header("etag"),
retry_after: None,
body: None,
}),
}
}
fn without_retries(client: aws_sdk_s3::Client) -> aws_sdk_s3::Client {
aws_sdk_s3::Client::from_conf(
client
.config()
.to_builder()
.retry_config(RetryConfig::disabled())
.build(),
)
}
fn non_empty_env(name: &str) -> Option<String> {
std::env::var(name).ok().filter(|value| !value.is_empty())
}

View file

@ -1,7 +1,10 @@
use std::fmt;
use std::path::PathBuf;
use url::Url;
use crate::error::SourceError;
/// Where a fetched catalog comes from
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Source {
@ -16,8 +19,84 @@ pub struct S3Object {
pub key: String,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum SourceError {
UnsupportedScheme { scheme: String },
Invalid { reason: String },
impl Source {
pub fn parse(text: &str) -> Result<Self, SourceError> {
let url = Url::parse(text).map_err(|error| SourceError::Invalid {
reason: format!("invalid URL {text}: {error}"),
})?;
match url.scheme() {
"http" | "https" => Ok(Self::Http(url)),
"file" => url
.to_file_path()
.map(Self::File)
.map_err(|()| SourceError::Invalid {
reason: format!("invalid file URL {text}"),
}),
"s3" => {
let bucket = url.host_str().unwrap_or_default();
let key = url.path().trim_start_matches('/');
if bucket.is_empty() || key.is_empty() {
return Err(SourceError::Invalid {
reason: format!("invalid S3 URL {text}: expected s3://bucket/key"),
});
}
Ok(Self::S3(S3Object {
bucket: bucket.to_owned(),
key: key.to_owned(),
}))
}
scheme => Err(SourceError::UnsupportedScheme {
scheme: scheme.to_owned(),
}),
}
}
}
impl fmt::Display for Source {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Http(url) => write!(formatter, "{url}"),
Self::File(path) => write!(formatter, "file://{}", path.display()),
Self::S3(object) => write!(formatter, "s3://{}/{}", object.bucket, object.key),
}
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
#[rstest]
#[case::https("https://example.com/map.json", Source::Http(Url::parse("https://example.com/map.json").unwrap()))]
#[case::file("file:///tmp/map.json", Source::File(PathBuf::from("/tmp/map.json")))]
#[case::s3(
"s3://bucket/prices/map.json",
Source::S3(S3Object { bucket: "bucket".into(), key: "prices/map.json".into() })
)]
fn parses_each_supported_scheme(#[case] text: &str, #[case] expected: Source) {
assert_eq!(Source::parse(text), Ok(expected));
}
#[rstest]
#[case::no_key("s3://bucket")]
#[case::no_bucket("s3:///key")]
#[case::not_a_url("http://[bad/model_prices.json")]
fn rejects_an_unusable_url(#[case] text: &str) {
assert!(matches!(
Source::parse(text),
Err(SourceError::Invalid { .. })
));
}
#[rstest]
fn rejects_an_unknown_scheme() {
assert_eq!(
Source::parse("ftp://example.com/map.json"),
Err(SourceError::UnsupportedScheme {
scheme: "ftp".into()
})
);
}
}

View file

@ -38,6 +38,8 @@ pub enum Step {
/// 304, a final status, or retries exhausted
Done {
reason: String,
/// The wait before the next attempt when the last failure was retryable, so a caller can retry later
retry_after: Option<Duration>,
},
}

View file

@ -0,0 +1,197 @@
use std::time::Duration;
use aws_credential_types::Credentials;
use aws_sdk_s3::config::SharedCredentialsProvider;
use litellm_model_catalog::BUNDLED_REGISTRY;
use litellm_model_catalog_refresh::{
Installed, Origin, RetryPolicy, S3Object, S3Source, Source, Step, Transports, fetch_with_retry,
git_blob_id,
};
use litellm_s3::S3ClientConfig;
use rstest::rstest;
use wiremock::matchers::{header, method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
fn installed(origin: Origin, etag: Option<&str>) -> Installed {
Installed {
origin,
etag: etag.map(str::to_owned),
revision: git_blob_id(BUNDLED_REGISTRY),
model_count: 0,
}
}
async fn no_sleep(_: Duration) {}
fn s3_source(server: &MockServer, object: S3Object) -> S3Source {
S3Source::new(
object,
litellm_http::Client::plain_for_test(),
SharedCredentialsProvider::new(Credentials::new("key", "secret", None, None, "test")),
S3ClientConfig {
region: "us-east-1".into(),
endpoint: Some(server.uri()),
},
)
}
#[rstest]
#[tokio::test]
async fn http_retries_a_503_then_adopts_the_map_with_its_etag() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/map.json"))
.respond_with(ResponseTemplate::new(503))
.up_to_n_times(1)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/map.json"))
.respond_with(
ResponseTemplate::new(200)
.insert_header("ETag", "W/\"v2\"")
.set_body_bytes(BUNDLED_REGISTRY),
)
.mount(&server)
.await;
let source = Source::parse(&format!("{}/map.json", server.uri())).unwrap();
let http = litellm_http::Client::plain_for_test();
let step = fetch_with_retry(
&source,
&installed(Origin::Bundled, None),
RetryPolicy::default(),
Some(0),
&Transports {
http: &http,
s3: None,
},
no_sleep,
)
.await;
let Step::Adopt { installed, .. } = step else {
panic!("expected adopt, got {step:?}");
};
assert_eq!(installed.etag.as_deref(), Some("W/\"v2\""));
assert_eq!(server.received_requests().await.unwrap().len(), 2);
}
#[rstest]
#[tokio::test]
async fn s3_retries_slow_down_and_sends_if_none_match() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/bucket/prices/map.json"))
.and(header("if-none-match", "\"v1\""))
.respond_with(ResponseTemplate::new(503).set_body_string(
"<Error><Code>SlowDown</Code><Message>Please reduce your request rate.</Message></Error>",
))
.up_to_n_times(1)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/bucket/prices/map.json"))
.and(header("if-none-match", "\"v1\""))
.respond_with(
ResponseTemplate::new(200)
.insert_header("ETag", "\"v2\"")
.set_body_bytes(BUNDLED_REGISTRY),
)
.mount(&server)
.await;
let object = S3Object {
bucket: "bucket".into(),
key: "prices/map.json".into(),
};
let s3 = s3_source(&server, object.clone());
let source = Source::S3(object);
let http = litellm_http::Client::plain_for_test();
let step = fetch_with_retry(
&source,
&installed(
Origin::Remote {
source: source.clone(),
},
Some("\"v1\""),
),
RetryPolicy::default(),
Some(0),
&Transports {
http: &http,
s3: Some(&s3),
},
no_sleep,
)
.await;
let Step::Adopt { installed, .. } = step else {
panic!("expected adopt, got {step:?}");
};
assert_eq!(installed.etag.as_deref(), Some("\"v2\""));
assert_eq!(server.received_requests().await.unwrap().len(), 2);
}
#[rstest]
#[tokio::test]
async fn s3_not_found_is_final() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.respond_with(
ResponseTemplate::new(404).set_body_string("<Error><Code>NoSuchKey</Code></Error>"),
)
.mount(&server)
.await;
let object = S3Object {
bucket: "bucket".into(),
key: "missing.json".into(),
};
let s3 = s3_source(&server, object.clone());
let http = litellm_http::Client::plain_for_test();
let step = fetch_with_retry(
&Source::S3(object),
&installed(Origin::Bundled, None),
RetryPolicy::default(),
Some(0),
&Transports {
http: &http,
s3: Some(&s3),
},
no_sleep,
)
.await;
let Step::Done { reason, .. } = step else {
panic!("expected done, got {step:?}");
};
assert_eq!(reason, "HTTP 404 from s3://bucket/missing.json");
assert_eq!(server.received_requests().await.unwrap().len(), 1);
}
#[rstest]
#[tokio::test]
async fn a_file_source_is_read_without_a_transport() {
let directory = std::env::temp_dir().join(format!("catalog-refresh-{}", std::process::id()));
std::fs::create_dir_all(&directory).unwrap();
let file = directory.join("map.json");
std::fs::write(&file, BUNDLED_REGISTRY).unwrap();
let http = litellm_http::Client::plain_for_test();
let step = fetch_with_retry(
&Source::File(file.clone()),
&installed(Origin::Bundled, None),
RetryPolicy::default(),
Some(0),
&Transports {
http: &http,
s3: None,
},
no_sleep,
)
.await;
std::fs::remove_dir_all(&directory).unwrap();
assert!(matches!(step, Step::Adopt { .. }), "{step:?}");
}

View file

@ -49,8 +49,8 @@ litellm-llms-types.workspace = true
litellm-router-types.workspace = true
litellm-host-python.workspace = true
litellm-model-catalog.workspace = true
litellm-model-catalog-refresh.workspace = true
indexmap = "2.14.0"
sha1.workspace = true
litellm-token-counter = { path = "../token-counter", default-features = false }
pyo3.workspace = true
prost.workspace = true

View file

@ -34,7 +34,9 @@ mod _native {
#[pymodule_export]
use crate::logger::NativeDiagnosticProcessor;
#[pymodule_export]
use crate::model_registry::{CatalogHandle, Fetched, Installed, RowError, Unavailable};
use crate::model_registry::{
Bundled, CatalogHandle, Fetched, File, Installed, Provenance, Remote, RowError, Unavailable,
};
#[pymodule_export]
use crate::routes::audio_transcription::{atranscription, transcription};
#[pymodule_export]
@ -157,6 +159,10 @@ mod tests {
"Fetched",
"Installed",
"Unavailable",
"Remote",
"File",
"Bundled",
"Provenance",
"RowError",
"RowErrorKind",
"NativeClickHouseSpendConfig",

View file

@ -1,19 +1,21 @@
mod fetch;
mod install;
mod row_error;
use std::collections::HashMap;
use std::sync::{Arc, LazyLock, Mutex, PoisonError};
use std::time::SystemTime;
use litellm_host_python::{release_gil, to_py};
use litellm_model_catalog::wire::Fields;
use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog, IntegrityLimits, Layers};
use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog, Layers, Located};
use litellm_model_catalog_refresh::{Source, git_blob_id};
use pyo3::exceptions::PyTypeError;
use pyo3::prelude::*;
use pyo3::pybacked::PyBackedBytes;
use pyo3::types::{PyDict, PyModule};
use serde::Serialize;
use sha1::{Digest, Sha1};
pub(crate) use fetch::{Bundled, File, Provenance, Remote};
use install::{Record, write_into};
pub(crate) use row_error::{RowError, RowErrorKind, row_error};
@ -30,7 +32,7 @@ static BUNDLED: LazyLock<Arc<Parsed>> = LazyLock::new(|| {
static BUNDLED_REVISION: LazyLock<String> = LazyLock::new(|| git_blob_id(BUNDLED_REGISTRY));
struct Parsed {
pub(crate) struct Parsed {
/// What install layers over bundled, rows decision 15 rejects replaced or left out
catalog: Arc<Catalog>,
/// The document as written
@ -46,6 +48,10 @@ pub(crate) struct Fetched {
revision: String,
#[pyo3(get)]
etag: Option<String>,
/// `None` for the bundled map
source: Option<Source>,
/// When the load that produced this map started, main's `loaded_at`
started_at: SystemTime,
}
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
@ -68,11 +74,15 @@ pub(crate) struct Installed {
pub(crate) struct Unavailable {
#[pyo3(get)]
reason: String,
/// Seconds to wait before retrying, set when the last reply was retryable
#[pyo3(get)]
retry_after: Option<f64>,
}
struct State {
layers: Layers,
record: Option<Record>,
provenance: Option<Provenance>,
}
#[pyclass(frozen, name = "Catalog", module = "litellm.rust_bridge._native")]
@ -117,6 +127,18 @@ impl Fetched {
}
}
#[pymethods]
impl Unavailable {
#[new]
#[pyo3(signature = (reason, retry_after = None))]
fn new(reason: String, retry_after: Option<f64>) -> Self {
Self {
reason,
retry_after,
}
}
}
#[pymethods]
impl CatalogHandle {
#[staticmethod]
@ -127,49 +149,58 @@ impl CatalogHandle {
state: Mutex::new(State {
layers: Layers::new(Arc::clone(&BUNDLED.catalog)),
record: None,
provenance: None,
}),
}
}
fn bundled(&self) -> Fetched {
Fetched {
parsed: Arc::clone(&BUNDLED),
revision: BUNDLED_REVISION.clone(),
etag: None,
}
bundled()
}
/// Parses a fetched body and applies main's whole-map checks against the bundled model count
#[staticmethod]
#[pyo3(signature = (body, *, etag, min_model_count, max_shrink_ratio))]
fn parse(
/// Fetches and validates `source` with no install, retrying up to `attempts` times
#[pyo3(signature = (source, *, timeout, attempts))]
fn fetch(
&self,
py: Python<'_>,
body: PyBackedBytes,
etag: Option<String>,
min_model_count: usize,
max_shrink_ratio: f64,
source: &Bound<'_, PyAny>,
timeout: f64,
attempts: u32,
) -> PyResult<Py<PyAny>> {
let parsed = release_gil(py, || {
parse_remote(&body, min_model_count, max_shrink_ratio)
});
match parsed {
Ok((parsed, revision)) => Ok(Fetched {
parsed: Arc::new(parsed),
revision,
etag,
}
fetch::fetch(py, fetch::requested(source)?, timeout, attempts)
}
/// Fetch with main's reload policy, 3 attempts of 5 s, then install
fn reload(&self, py: Python<'_>, source: &Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
let requested = fetch::requested(source)?;
let env_forced = matches!(requested, fetch::Requested::Bundled);
let fetched = fetch::fetch(py, requested, 5.0, 3)?;
let Ok(fetched) = fetched.bind(py).cast::<Fetched>() else {
return Ok(fetched);
};
Ok(self
.install(py, fetched.get(), env_forced, None, None)?
.into_pyobject(py)?
.into_any()
.unbind()),
Err(reason) => Ok(Unavailable { reason }
.into_pyobject(py)?
.into_any()
.unbind()),
}
.unbind())
}
/// Writes the composed rows into the dict bound to the attribute, never over a field the user owns (decision 25)
fn install(&self, py: Python<'_>, fetched: &Fetched) -> PyResult<Installed> {
/// Where the installed map came from, `None` before the first install
fn provenance(&self) -> Option<Provenance> {
self.lock().provenance.clone()
}
/// Writes the composed rows into the dict bound to the attribute, never over a field the user owns (decision 25).
/// `env_forced`, `fallback_reason` and `url` are the provenance facts only the caller knows
#[pyo3(signature = (fetched, *, env_forced = false, fallback_reason = None, url = None))]
fn install(
&self,
py: Python<'_>,
fetched: &Fetched,
env_forced: bool,
fallback_reason: Option<String>,
url: Option<String>,
) -> PyResult<Installed> {
let bound = self.module.bind(py).getattr(self.attr.as_str())?;
let dict = bound
.cast::<PyDict>()
@ -188,7 +219,12 @@ impl CatalogHandle {
};
let composed = release_gil(py, || layers.compose());
let outcome = write_into(py, dict, &composed, &layers, record)?;
self.lock().record = Some(outcome.record);
let provenance = Provenance::of(fetched, env_forced, fallback_reason, url);
{
let mut state = self.lock();
state.record = Some(outcome.record);
state.provenance = Some(provenance);
}
Ok(Installed {
count: composed.len(),
written: outcome.written,
@ -204,47 +240,29 @@ impl CatalogHandle {
}
}
fn parse_remote(
body: &[u8],
min_model_count: usize,
max_shrink_ratio: f64,
) -> Result<(Parsed, String), String> {
let limits = IntegrityLimits::new(min_model_count, max_shrink_ratio)
.ok_or_else(|| format!("invalid shrink ratio {max_shrink_ratio}"))?;
let loaded = Catalog::parse(body).map_err(|error| error.to_string())?;
loaded
.value
.check_size(BUNDLED.raw.model_count(), limits)
.map_err(|error| error.to_string())?;
let raw = Arc::new(loaded.value.clone());
let errors = loaded.errors.iter().map(row_error).collect();
let (catalog, rejections) = loaded
.value
.with_rejected_replaced(&loaded.errors, &BUNDLED.raw);
let rejected = rejections
.iter()
.flat_map(|rejection| rejection.errors.iter().copied().map(row_error))
.collect();
Ok((
Parsed {
catalog: Arc::new(catalog),
raw,
errors,
rejected,
},
git_blob_id(body),
))
pub(crate) fn bundled() -> Fetched {
Fetched {
parsed: Arc::clone(&BUNDLED),
revision: BUNDLED_REVISION.clone(),
etag: None,
source: None,
started_at: SystemTime::now(),
}
}
fn git_blob_id(body: &[u8]) -> String {
let mut hasher = Sha1::new();
hasher.update(format!("blob {}\0", body.len()).as_bytes());
hasher.update(body);
hasher
.finalize()
.iter()
.map(|byte| format!("{byte:02x}"))
.collect()
/// A fetched document with the rows decision 15 rejects replaced by bundled or left out
pub(crate) fn remote(catalog: Catalog, errors: &[Located]) -> Parsed {
let raw = Arc::new(catalog.clone());
let (catalog, rejections) = catalog.with_rejected_replaced(errors, &BUNDLED.raw);
Parsed {
catalog: Arc::new(catalog),
raw,
errors: errors.iter().map(row_error).collect(),
rejected: rejections
.iter()
.flat_map(|rejection| rejection.errors.iter().copied().map(row_error))
.collect(),
}
}
fn served_rows(catalog: &Catalog) -> Vec<(String, String, Fields)> {
@ -281,18 +299,3 @@ impl Serialize for WrittenDocument<'_> {
serializer.collect_map(self.0.written())
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
#[rstest]
fn the_revision_is_the_git_blob_id_of_the_body() {
assert_eq!(
git_blob_id(b"hello\n"),
"ce013625030ba8dba906f756967f9e9ca394464a"
);
}
}

View file

@ -0,0 +1,307 @@
use std::path::PathBuf;
use std::sync::Arc;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use litellm_host_python::release_gil;
use litellm_http::media::PublicDnsResolver;
use litellm_http::{ClientVariant, HttpClientConfig, HttpClientPool};
use litellm_model_catalog::IntegrityLimits;
use litellm_model_catalog_refresh::{
Installed, Origin, RetryPolicy, Source, Step, Transports, fetch_with_retry,
};
use pyo3::exceptions::{PyTypeError, PyValueError};
use pyo3::prelude::*;
use pyo3::types::PyDict;
use super::{BUNDLED, BUNDLED_REVISION, Fetched, Parsed, Unavailable, bundled, remote};
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
pub(crate) struct Remote {
#[pyo3(get)]
url: String,
}
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
pub(crate) struct File {
#[pyo3(get)]
path: String,
}
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
pub(crate) struct Bundled;
/// One snapshot of where the installed map came from, replacing main's field-by-field `ModelCostMapSourceInfo`
#[pyclass(frozen, skip_from_py_object, module = "litellm.rust_bridge._native")]
#[derive(Clone)]
pub(crate) struct Provenance {
/// `"local"` or `"remote"`, main's values
#[pyo3(get)]
source: &'static str,
#[pyo3(get)]
url: Option<String>,
#[pyo3(get)]
is_env_forced: bool,
#[pyo3(get)]
fallback_reason: Option<String>,
loaded_at: SystemTime,
#[pyo3(get)]
source_revision: String,
#[pyo3(get)]
etag: Option<String>,
}
#[pymethods]
impl Remote {
#[new]
fn new(url: String) -> Self {
Self { url }
}
}
#[pymethods]
impl File {
#[new]
fn new(path: PathBuf) -> Self {
Self {
path: path.to_string_lossy().into_owned(),
}
}
}
#[pymethods]
impl Bundled {
#[new]
fn new() -> Self {
Self
}
}
#[pymethods]
impl Provenance {
/// When the load that installed this map started, as an aware UTC datetime
#[getter]
fn loaded_at<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
let seconds = self
.loaded_at
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs_f64();
let datetime = py.import("datetime")?;
let utc = datetime.getattr("timezone")?.getattr("utc")?;
datetime
.getattr("datetime")?
.call_method1("fromtimestamp", (seconds, utc))
}
}
impl Provenance {
pub(crate) fn of(
fetched: &Fetched,
env_forced: bool,
fallback_reason: Option<String>,
url: Option<String>,
) -> Self {
Self {
source: if fetched.source.is_some() {
"remote"
} else {
"local"
},
url: url.or_else(|| fetched.source.as_ref().map(ToString::to_string)),
is_env_forced: env_forced,
fallback_reason,
loaded_at: fetched.started_at,
source_revision: fetched.revision.clone(),
etag: fetched.etag.clone(),
}
}
}
/// What a Python `Source` names
pub(crate) enum Requested {
Bundled,
Source(Source),
Invalid(String),
}
pub(crate) fn requested(source: &Bound<'_, PyAny>) -> PyResult<Requested> {
if source.is_instance_of::<Bundled>() {
return Ok(Requested::Bundled);
}
if let Ok(remote) = source.cast::<Remote>() {
let url = &remote.get().url;
return Ok(Source::parse(url).map_or_else(
|error| Requested::Invalid(error.to_string()),
Requested::Source,
));
}
if let Ok(file) = source.cast::<File>() {
return Ok(Requested::Source(Source::File(PathBuf::from(
&file.get().path,
))));
}
Err(PyTypeError::new_err(
"source must be Remote, File or Bundled",
))
}
enum Outcome {
Fetched {
parsed: Parsed,
revision: String,
etag: Option<String>,
},
Unavailable {
reason: String,
retry_after: Option<Duration>,
},
}
/// Fetches with the GIL released, retrying under main's policy, and validates the body before returning it
pub(crate) fn fetch(
py: Python<'_>,
requested: Requested,
timeout: f64,
attempts: u32,
) -> PyResult<Py<PyAny>> {
let source = match requested {
Requested::Bundled => return Ok(bundled().into_pyobject(py)?.into_any().unbind()),
Requested::Invalid(reason) => return unavailable(py, reason, None),
Requested::Source(source) => source,
};
let timeout = Duration::try_from_secs_f64(timeout)
.map_err(|_| PyValueError::new_err("timeout must be a non-negative number of seconds"))?;
let policy = RetryPolicy {
max_attempts: attempts.max(1),
timeout,
limits: integrity_limits(py)?,
..RetryPolicy::default()
};
let config = crate::http::call_config(py, &PyDict::new(py), false)?;
let started_at = SystemTime::now();
let fetched_from = source.clone();
match release_gil(py, move || drive(&source, policy, config)) {
Outcome::Fetched {
parsed,
revision,
etag,
} => Ok(Fetched {
parsed: Arc::new(parsed),
revision,
etag,
source: Some(fetched_from),
started_at,
}
.into_pyobject(py)?
.into_any()
.unbind()),
Outcome::Unavailable {
reason,
retry_after,
} => unavailable(py, reason, retry_after),
}
}
fn unavailable(
py: Python<'_>,
reason: String,
retry_after: Option<Duration>,
) -> PyResult<Py<PyAny>> {
Ok(Unavailable {
reason,
retry_after: retry_after.map(|wait| wait.as_secs_f64()),
}
.into_pyobject(py)?
.into_any()
.unbind())
}
/// Main's `MODEL_COST_MAP_MIN_MODEL_COUNT` and `MODEL_COST_MAP_MAX_SHRINK_RATIO`, which are env-overridable
fn integrity_limits(py: Python<'_>) -> PyResult<IntegrityLimits> {
let constants = py.import("litellm.constants")?;
let minimum: usize = constants
.getattr("MODEL_COST_MAP_MIN_MODEL_COUNT")?
.extract()?;
let ratio: f64 = constants
.getattr("MODEL_COST_MAP_MAX_SHRINK_RATIO")?
.extract()?;
IntegrityLimits::new(minimum, ratio).ok_or_else(|| {
PyValueError::new_err("MODEL_COST_MAP_MAX_SHRINK_RATIO must be between 0 and 1")
})
}
/// A current-thread runtime of its own, so a fetch at import never starts the shared native runtime, and a client
/// built inside it, so no pooled connection outlives the runtime that spawned it
fn drive(source: &Source, policy: RetryPolicy, config: HttpClientConfig) -> Outcome {
if tokio::runtime::Handle::try_current().is_ok() {
return Outcome::Unavailable {
reason: "the model cost map fetch cannot run from a Tokio context".to_owned(),
retry_after: None,
};
}
let runtime = match tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
{
Ok(runtime) => runtime,
Err(error) => {
return Outcome::Unavailable {
reason: format!("the model cost map fetch runtime could not start: {error}"),
retry_after: None,
};
}
};
let step = runtime.block_on(async {
let pool = HttpClientPool::new(Arc::new(PublicDnsResolver));
let client = match pool.client(&config, ClientVariant::Provider) {
Ok(client) => client,
Err(error) => {
return Step::Done {
reason: format!("the HTTP client could not be built: {error}"),
retry_after: None,
};
}
};
let installed = Installed {
origin: Origin::Bundled,
etag: None,
revision: BUNDLED_REVISION.clone(),
model_count: BUNDLED.raw.model_count(),
};
fetch_with_retry(
source,
&installed,
policy,
None,
&Transports {
http: &client,
s3: None,
},
tokio::time::sleep,
)
.await
});
runtime.shutdown_timeout(Duration::from_secs(1));
match step {
Step::Adopt {
catalog,
installed,
errors,
} => Outcome::Fetched {
parsed: remote(*catalog, &errors),
revision: installed.revision,
etag: installed.etag,
},
Step::Done {
reason,
retry_after,
} => Outcome::Unavailable {
reason,
retry_after,
},
Step::Fetch(_) | Step::Sleep(_) => Outcome::Unavailable {
reason: "the model cost map fetch stopped before it finished".to_owned(),
retry_after: None,
},
}
}

View file

@ -0,0 +1,15 @@
[package]
name = "litellm-s3"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
aws-credential-types = "1.3.0"
aws-sdk-s3 = { version = "1.146.1", default-features = false, features = ["rustls", "rt-tokio"] }
aws-smithy-runtime-api = { version = "1.16.2", features = ["client", "http-1x"] }
aws-smithy-types = { version = "1.6.0", features = ["http-body-1-x"] }
http.workspace = true
litellm-http.workspace = true
reqwest.workspace = true

View file

@ -0,0 +1,35 @@
mod transport;
use aws_credential_types::provider::ProvideCredentials;
use aws_sdk_s3::config::{
BehaviorVersion, Region, RequestChecksumCalculation, ResponseChecksumValidation,
};
pub use transport::ReqwestHttpClient;
/// Where an S3 client sends its requests
pub struct S3ClientConfig {
pub region: String,
/// An S3-compatible endpoint (MinIO, R2, Ceph), addressed path-style
pub endpoint: Option<String>,
}
/// One way to build an S3 client over a reqwest client, shared by every S3 caller
pub fn client(
config: S3ClientConfig,
http: litellm_http::Client,
credentials: impl ProvideCredentials + 'static,
) -> aws_sdk_s3::Client {
let base = aws_sdk_s3::Config::builder()
.behavior_version(BehaviorVersion::latest())
.region(Region::new(config.region))
.http_client(ReqwestHttpClient(http))
.credentials_provider(credentials)
.request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
.response_checksum_validation(ResponseChecksumValidation::WhenRequired);
let builder = match config.endpoint {
Some(url) => base.endpoint_url(url).force_path_style(true),
None => base,
};
aws_sdk_s3::Client::from_conf(builder.build())
}

View file

@ -9,7 +9,7 @@ use aws_smithy_runtime_api::client::{
use aws_smithy_types::body::SdkBody;
#[derive(Clone, Debug)]
pub(crate) struct ReqwestHttpClient(pub(crate) litellm_http::Client);
pub struct ReqwestHttpClient(pub litellm_http::Client);
impl HttpClient for ReqwestHttpClient {
fn http_connector(

View file

@ -1,10 +1,47 @@
"""Internal catalog API backed by the Rust ``model-catalog`` crate. Not public, its shape may change in any release"""
import os
import sys
from typing import Final
from litellm.rust_bridge._native import Catalog, Fetched, Installed, RowError, Unavailable
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.rust_bridge._native import (
Bundled,
Catalog,
Fetched,
File,
Installed,
Provenance,
Remote,
RowError,
Unavailable,
)
Source = Remote | File | Bundled
CATALOG: Final = Catalog.bound_to(sys.modules["litellm"], "model_cost")
__all__ = ["CATALOG", "Catalog", "Fetched", "Installed", "RowError", "Unavailable"]
def default_source() -> Source:
"""Main's reload rule, the bundled map under offline mode or ``LITELLM_LOCAL_MODEL_COST_MAP``"""
import litellm
if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
return Bundled()
return Remote(litellm.model_cost_map_url)
__all__ = [
"CATALOG",
"Bundled",
"Catalog",
"Fetched",
"File",
"Installed",
"Provenance",
"Remote",
"RowError",
"Source",
"Unavailable",
"default_source",
]

View file

@ -9,31 +9,23 @@ export LITELLM_LOCAL_MODEL_COST_MAP=True
```
"""
import asyncio
import os
import random
import sys
import threading
import time
import urllib.parse
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass
from datetime import datetime, timezone
import warnings
from collections.abc import Callable, Mapping
from datetime import datetime
from itertools import chain
from pathlib import Path
from types import MappingProxyType
from typing import Final, Protocol, cast # noqa: TID251 # adapts boto3 and botocore's dynamic client responses
from typing import Final
import httpx
from pydantic import JsonValue, TypeAdapter
from typing_extensions import NotRequired, ReadOnly, TypedDict
from typing_extensions import ReadOnly, TypedDict
from litellm import verbose_logger
from litellm._catalog import CATALOG, Catalog, Fetched, Installed, Unavailable
from litellm.constants import (
MODEL_COST_MAP_MAX_SHRINK_RATIO,
MODEL_COST_MAP_MIN_MODEL_COUNT,
)
from litellm._catalog import CATALOG, Catalog, Fetched, Installed, Remote, Source, Unavailable
from litellm.litellm_core_utils.fallback_generalizations import (
set_fallback_generalizations,
)
@ -42,12 +34,25 @@ from litellm.litellm_core_utils.offline_mode import is_offline_mode
FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations"
_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
_CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"})
MODEL_COST_MAP_FETCH_MAX_ATTEMPTS: Final = 3
_IMPORT_FETCH_TIMEOUT_SECONDS: Final = 5
_litellm_import_complete = threading.Event()
def _is_cli_process() -> bool:
return Path(sys.argv[0]).stem in _CLI_ENTRYPOINT_NAMES
def _local_map_forced() -> bool:
# Note: can't use get_secret_bool here — this runs during litellm.__init__
# before litellm._key_management_settings is set.
return is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true"
def mark_litellm_import_complete() -> None:
_litellm_import_complete.set()
class GetModelCostMap:
"""The catalog as last loaded, for readers that want it without later registrations"""
@ -71,304 +76,6 @@ class GetModelCostMap:
cls._loaded_catalog_fields = frozenset(chain.from_iterable(entry.keys() for entry in raw.values()))
RETRYABLE_FETCH_STATUS_CODES: Final = frozenset({429, 500, 502, 503, 504})
MODEL_COST_MAP_FETCH_MAX_ATTEMPTS: Final = 3
MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS: Final = 30.0
_litellm_import_complete = threading.Event()
def mark_litellm_import_complete() -> None:
_litellm_import_complete.set()
@dataclass(frozen=True, slots=True)
class ModelCostMapReloaded:
fetched: Fetched
@property
def revision(self) -> str:
return self.fetched.revision
@property
def etag(self) -> str | None:
return self.fetched.etag
@dataclass(frozen=True, slots=True)
class ModelCostMapReloadUnavailable:
reason: str
ModelCostMapReloadResult = ModelCostMapReloaded | ModelCostMapReloadUnavailable
@dataclass(frozen=True, slots=True)
class _FetchAttemptRetryable:
reason: str
retry_after_seconds: float | None
def _parse_retry_after_seconds(response: httpx.Response) -> float | None:
try:
header: Final = response.headers["Retry-After"]
except KeyError:
return None
try:
seconds: Final = float(header)
except ValueError:
return None
return seconds if seconds >= 0 else None
def _retry_wait_seconds(outcome: _FetchAttemptRetryable, attempt: int, rng: random.Random) -> float:
if outcome.retry_after_seconds is not None:
return min(outcome.retry_after_seconds, MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS)
return min(2.0**attempt + rng.uniform(0.0, 1.0), MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS)
class _AsyncGetClient(Protocol):
def get(self, url: str, *, timeout: float | None = None) -> Awaitable[httpx.Response]: ...
class _SyncGetClient(Protocol):
def get(self, url: str, *, timeout: float | None = None) -> httpx.Response: ...
class _S3Body(Protocol):
def read(self) -> bytes: ...
class _S3GetObjectResponse(TypedDict):
Body: ReadOnly[_S3Body]
ETag: NotRequired[ReadOnly[str]]
class _S3Client(Protocol):
def get_object(self, *, Bucket: str, Key: str) -> _S3GetObjectResponse: ...
_FetchAttemptOutcome = ModelCostMapReloaded | ModelCostMapReloadUnavailable | _FetchAttemptRetryable
def _default_reload_client() -> _AsyncGetClient:
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.llms.custom_http import httpxSpecialProvider
return get_async_httpx_client(llm_provider=httpxSpecialProvider.ModelCostMap)
def _default_s3_client(timeout: int) -> _S3Client:
import boto3
from botocore.config import Config
return cast( # cast-ok: boto3's generic client factory does not expose its generated S3 response type
_S3Client,
boto3.client( # pyright: ignore[reportUnknownMemberType] # generic factory returns Unknown service clients
"s3",
region_name=os.getenv("AWS_REGION_NAME") or None,
config=Config(connect_timeout=timeout, read_timeout=timeout, retries={"total_max_attempts": 1}),
),
)
def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) -> _FetchAttemptOutcome:
reason: Final = f"{type(error).__name__} fetching {url}: {error}"
if isinstance(error, (httpx.InvalidURL, httpx.UnsupportedProtocol)):
return ModelCostMapReloadUnavailable(reason=reason)
return _FetchAttemptRetryable(reason=reason, retry_after_seconds=None)
def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _FetchAttemptOutcome:
parsed: Final = CATALOG.parse(
content,
etag=etag,
min_model_count=MODEL_COST_MAP_MIN_MODEL_COUNT,
max_shrink_ratio=MODEL_COST_MAP_MAX_SHRINK_RATIO,
)
if isinstance(parsed, Unavailable):
return ModelCostMapReloadUnavailable(reason=f"model cost map from {url} is unusable: {parsed.reason}")
return ModelCostMapReloaded(fetched=parsed)
def _split_url(url: str) -> urllib.parse.SplitResult | ModelCostMapReloadUnavailable:
try:
return urllib.parse.urlsplit(url)
except ValueError as error:
return ModelCostMapReloadUnavailable(reason=f"invalid URL {url}: {error}")
def _attempt_fetch_s3(parts: urllib.parse.SplitResult, url: str, timeout: int) -> _FetchAttemptOutcome:
from botocore.exceptions import BotoCoreError, ClientError
bucket: Final = parts.netloc
key: Final = parts.path.lstrip("/")
if not bucket or not key:
return ModelCostMapReloadUnavailable(reason=f"invalid S3 URL {url}: expected s3://bucket/key")
try:
client: Final = _default_s3_client(timeout)
response: Final = client.get_object(Bucket=bucket, Key=key)
content: Final = response["Body"].read()
except (BotoCoreError, ClientError) as error:
return ModelCostMapReloadUnavailable(reason=f"S3 {type(error).__name__} fetching {url}: {error}")
return _classify_fetch_content(content, url, response.get("ETag"))
async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
parts: Final = _split_url(url)
if isinstance(parts, ModelCostMapReloadUnavailable):
return parts
if parts.scheme == "s3":
return await asyncio.to_thread(_attempt_fetch_s3, parts, url, timeout)
try:
response: Final = await client.get(url, timeout=timeout)
except (httpx.HTTPError, httpx.InvalidURL) as e:
return _classify_fetch_error(e, url)
return _classify_fetch_response(response, url)
def _attempt_fetch_sync(client: _SyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
parts: Final = _split_url(url)
if isinstance(parts, ModelCostMapReloadUnavailable):
return parts
if parts.scheme == "s3":
return _attempt_fetch_s3(parts, url, timeout)
try:
response: Final = client.get(url, timeout=timeout)
except (httpx.HTTPError, httpx.InvalidURL) as e:
return _classify_fetch_error(e, url)
return _classify_fetch_response(response, url)
def _classify_fetch_response(response: httpx.Response, url: str) -> _FetchAttemptOutcome:
if response.status_code in RETRYABLE_FETCH_STATUS_CODES:
return _FetchAttemptRetryable(
reason=f"HTTP {response.status_code} from {url}",
retry_after_seconds=_parse_retry_after_seconds(response),
)
if response.is_error:
return ModelCostMapReloadUnavailable(reason=f"HTTP {response.status_code} from {url}")
etag_values: Final = response.headers.get_list("etag")
return _classify_fetch_content(response.content, url, ", ".join(etag_values) if etag_values else None)
def _next_retry_wait(
outcome: _FetchAttemptRetryable, attempt: int, max_attempts: int, rng: random.Random
) -> float | ModelCostMapReloadUnavailable:
if attempt == max_attempts:
return ModelCostMapReloadUnavailable(reason=f"{outcome.reason} (after {max_attempts} attempts)")
wait_seconds: Final = _retry_wait_seconds(outcome=outcome, attempt=attempt, rng=rng)
verbose_logger.warning(
"LiteLLM: model cost map fetch attempt %d/%d failed (%s); retrying in %.1fs",
attempt,
max_attempts,
outcome.reason,
wait_seconds,
)
return wait_seconds
async def _fetch_remote_model_cost_map_with_retry(
url: str,
timeout: int,
max_attempts: int,
sleep: Callable[[float], Awaitable[None]],
rng: random.Random,
client: _AsyncGetClient,
) -> ModelCostMapReloadResult:
for attempt in range(1, max_attempts + 1):
outcome = await _attempt_fetch(client=client, url=url, timeout=timeout)
if not isinstance(outcome, _FetchAttemptRetryable):
return outcome
wait_seconds = _next_retry_wait(outcome=outcome, attempt=attempt, max_attempts=max_attempts, rng=rng)
if isinstance(wait_seconds, ModelCostMapReloadUnavailable):
return wait_seconds
await sleep(wait_seconds)
return ModelCostMapReloadUnavailable(reason="model cost map fetch failed")
def _fetch_remote_model_cost_map_with_retry_sync(
url: str,
timeout: int,
attempts: range,
sleep: Callable[[float], None],
rng: random.Random,
client: _SyncGetClient,
) -> ModelCostMapReloadResult:
max_attempts: Final = attempts.stop - 1
for attempt in attempts:
outcome = _attempt_fetch_sync(client=client, url=url, timeout=timeout)
if not isinstance(outcome, _FetchAttemptRetryable):
return outcome
wait_seconds = _next_retry_wait(outcome=outcome, attempt=attempt, max_attempts=max_attempts, rng=rng)
if isinstance(wait_seconds, ModelCostMapReloadUnavailable):
return wait_seconds
sleep(wait_seconds)
return ModelCostMapReloadUnavailable(reason="model cost map fetch failed")
async def refetch_model_cost_map(
url: str,
timeout: int = 5,
max_attempts: int = MODEL_COST_MAP_FETCH_MAX_ATTEMPTS,
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
rng: random.Random | None = None,
client: "_AsyncGetClient | None" = None,
) -> ModelCostMapReloadResult:
"""
Re-fetch the model cost map for a runtime reload, retrying transient HTTP
errors (429/5xx/transport) with Retry-After-aware backoff.
Unlike ``get_model_cost_map`` this never falls back to the packaged backup:
on failure it returns ``ModelCostMapReloadUnavailable`` so callers keep the
map they already have.
"""
if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
_cost_map_source_info.source = "local"
_cost_map_source_info.url = None
_cost_map_source_info.is_env_forced = True
_cost_map_source_info.fallback_reason = None
return _note_loaded(ModelCostMapReloaded(fetched=CATALOG.bundled()))
result: Final = await _fetch_remote_model_cost_map_with_retry(
url=url,
timeout=timeout,
max_attempts=max_attempts,
sleep=sleep,
rng=rng if rng is not None else random.Random(),
client=client if client is not None else _default_reload_client(),
)
if isinstance(result, ModelCostMapReloadUnavailable):
verbose_logger.warning(
"LiteLLM: model cost map reload failed: %s. Keeping the currently loaded map.",
result.reason,
)
return result
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
_cost_map_source_info.source = "remote"
_cost_map_source_info.url = url
_cost_map_source_info.is_env_forced = False
_cost_map_source_info.fallback_reason = None
return _note_loaded(result)
class ModelCostMapSourceInfo:
"""Tracks the source of the currently loaded model cost map."""
source: str = "local" # "local" or "remote"
url: str | None = None
is_env_forced: bool = False
fallback_reason: str | None = None
loaded_at: "datetime | None" = None
source_revision: str | None = None
etag: str | None = None
# Module-level singleton tracking the source of the current cost map
_cost_map_source_info: Final = ModelCostMapSourceInfo()
class CostMapProvenance(TypedDict):
source_revision: ReadOnly[str | None]
etag: ReadOnly[str | None]
@ -383,9 +90,10 @@ class CostMapSourceInfo(CostMapProvenance):
def get_model_cost_map_provenance() -> CostMapProvenance:
provenance: Final = CATALOG.provenance()
return {
"source_revision": _cost_map_source_info.source_revision,
"etag": _cost_map_source_info.etag,
"source_revision": provenance.source_revision if provenance is not None else None,
"etag": provenance.etag if provenance is not None else None,
}
@ -402,21 +110,32 @@ def get_model_cost_map_source_info() -> CostMapSourceInfo:
- source_revision: git blob id of the loaded file's bytes
- etag: the ETag of the remote fetch (None for the bundled backup)
"""
loaded_at: Final = _cost_map_source_info.loaded_at
provenance: Final = CATALOG.provenance()
if provenance is None:
return {
"source": "local",
"url": None,
"is_env_forced": False,
"fallback_reason": None,
"loaded_at": None,
"source_revision": None,
"etag": None,
}
return {
"source": _cost_map_source_info.source,
"url": _cost_map_source_info.url,
"is_env_forced": _cost_map_source_info.is_env_forced,
"fallback_reason": _cost_map_source_info.fallback_reason,
"loaded_at": loaded_at.isoformat() if loaded_at is not None else None,
"source_revision": _cost_map_source_info.source_revision,
"etag": _cost_map_source_info.etag,
"source": provenance.source,
"url": provenance.url,
"is_env_forced": provenance.is_env_forced,
"fallback_reason": provenance.fallback_reason,
"loaded_at": provenance.loaded_at.isoformat(),
"source_revision": provenance.source_revision,
"etag": provenance.etag,
}
def get_model_cost_map_loaded_at() -> "datetime | None":
"""When this process last loaded its cost map, stamped at the start of every load"""
return _cost_map_source_info.loaded_at
provenance: Final = CATALOG.provenance()
return provenance.loaded_at if provenance is not None else None
class AliasedCostEntry(dict[str, object]):
@ -430,14 +149,12 @@ class AliasedCostEntry(dict[str, object]):
self.registered_fields: frozenset[str] = registered_fields
def _note_loaded(loaded: ModelCostMapReloaded) -> ModelCostMapReloaded:
_cost_map_source_info.source_revision = loaded.fetched.revision
_cost_map_source_info.etag = loaded.fetched.etag
set_fallback_generalizations(loaded.fetched.fallback_rules())
def _note_loaded(fetched: Fetched) -> Fetched:
set_fallback_generalizations(fetched.fallback_rules())
GetModelCostMap._snapshot_loaded_catalog( # pyright: ignore[reportPrivateUsage] # same module
loaded.fetched.served()
fetched.served()
)
return loaded
return fetched
class _InstallGeneration:
@ -450,8 +167,15 @@ def model_cost_install_generation() -> int:
return _InstallGeneration.value
def _install(fetched: Fetched, catalog: Catalog = CATALOG) -> Installed:
installed: Final = catalog.install(fetched)
def _install(
fetched: Fetched,
catalog: Catalog = CATALOG,
*,
env_forced: bool = False,
fallback_reason: str | None = None,
url: str | None = None,
) -> Installed:
installed: Final = catalog.install(fetched, env_forced=env_forced, fallback_reason=fallback_reason, url=url)
_InstallGeneration.value += 1
if installed.rejected:
verbose_logger.warning(
@ -461,40 +185,38 @@ def _install(fetched: Fetched, catalog: Catalog = CATALOG) -> Installed:
return installed
def adopt_model_cost_map(fetched: Fetched) -> int:
def _refresh_derived_state() -> None:
import litellm
from litellm import utils
installed: Final = _install(fetched)
utils._invalidate_model_cost_lowercase_map() # pyright: ignore[reportPrivateUsage] # required cache invalidation
litellm.add_known_models(model_cost_map=litellm.model_cost)
utils.reapply_runtime_model_cost_registrations()
return installed.count
def reload_model_cost_map(source: Source, catalog: Catalog = CATALOG) -> Installed | Unavailable:
"""Fetch with main's reload policy, then install and refresh what Python still derives from the map. A failed
fetch keeps the map in use"""
fetched: Final = catalog.fetch(
source, timeout=_IMPORT_FETCH_TIMEOUT_SECONDS, attempts=MODEL_COST_MAP_FETCH_MAX_ATTEMPTS
)
if isinstance(fetched, Unavailable):
verbose_logger.warning(
"LiteLLM: model cost map reload failed: %s. Keeping the currently loaded map.", fetched.reason
)
return fetched
installed: Final = _install(_note_loaded(fetched), catalog, env_forced=_local_map_forced())
_refresh_derived_state()
return installed
def _retry_remote_fetch_in_background(
url: str,
timeout: int,
max_attempts: int,
sleep: Callable[[float], None],
rng: random.Random,
client: _SyncGetClient,
first_outcome: _FetchAttemptRetryable,
url: str, max_attempts: int, first_wait: float, sleep: Callable[[float], None], catalog: Catalog
) -> None:
try:
first_wait: Final = _next_retry_wait(outcome=first_outcome, attempt=1, max_attempts=max_attempts, rng=rng)
if isinstance(first_wait, ModelCostMapReloadUnavailable):
return
sleep(first_wait)
result: Final = _fetch_remote_model_cost_map_with_retry_sync(
url=url,
timeout=timeout,
attempts=range(2, max_attempts + 1),
sleep=sleep,
rng=rng,
client=client,
)
if isinstance(result, ModelCostMapReloadUnavailable):
fetched: Final = catalog.fetch(Remote(url), timeout=_IMPORT_FETCH_TIMEOUT_SECONDS, attempts=max_attempts - 1)
if isinstance(fetched, Unavailable):
verbose_logger.warning(
"LiteLLM: Failed to fetch remote model cost map from %s after %d attempts; keeping local backup",
url,
@ -502,100 +224,83 @@ def _retry_remote_fetch_in_background(
)
return
_litellm_import_complete.wait()
_note_loaded(result)
_cost_map_source_info.source = "remote"
_cost_map_source_info.fallback_reason = None
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
adopt_model_cost_map(result.fetched)
_install(_note_loaded(fetched), catalog, url=url)
_refresh_derived_state()
except Exception as e: # noqa: BLE001 # a failed background retry must not kill the task; the backup stays
verbose_logger.warning("LiteLLM: Background model cost map retry failed: %s", e)
def _pick_model_cost_map(
url: str,
timeout: int,
max_attempts: int,
sleep: Callable[[float], None],
rng: random.Random | None,
client: "_SyncGetClient | None",
) -> Fetched:
url: str, timeout: int, max_attempts: int, sleep: Callable[[float], None], catalog: Catalog
) -> tuple[Fetched, bool, str | None]:
"""
The map main loads at import. The bundled map when ``LITELLM_LOCAL_MODEL_COST_MAP`` is set, in offline mode or in a
``lite`` / ``litellm-proxy`` CLI process. Otherwise one remote attempt, with a background retry on a transient
failure, and the bundled map on any failure
The map main loads at import, with the provenance facts install records. The bundled map when
``LITELLM_LOCAL_MODEL_COST_MAP`` is set, in offline mode or in a ``lite`` / ``litellm-proxy`` CLI process.
Otherwise one remote attempt, with a background retry on a transient failure, and the bundled map on any failure
"""
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
# Note: can't use get_secret_bool here — this runs during litellm.__init__
# before litellm._key_management_settings is set.
if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process():
_cost_map_source_info.source = "local"
_cost_map_source_info.url = None
_cost_map_source_info.is_env_forced = True
_cost_map_source_info.fallback_reason = None
return _note_loaded(ModelCostMapReloaded(fetched=CATALOG.bundled())).fetched
_cost_map_source_info.url = url
_cost_map_source_info.is_env_forced = False
fetch_client: Final = client if client is not None else httpx
fetch_rng: Final = rng if rng is not None else random.Random()
outcome: Final = _attempt_fetch_sync(client=fetch_client, url=url, timeout=timeout)
if isinstance(outcome, _FetchAttemptRetryable) and max_attempts > 1:
if _local_map_forced() or _is_cli_process():
return _note_loaded(catalog.bundled()), True, None
fetched: Final = catalog.fetch(Remote(url or ""), timeout=timeout, attempts=1)
if isinstance(fetched, Fetched):
return _note_loaded(fetched), False, None
if fetched.retry_after is not None and max_attempts > 1:
threading.Thread(
target=_retry_remote_fetch_in_background,
kwargs={
"url": url,
"timeout": timeout,
"max_attempts": max_attempts,
"first_wait": fetched.retry_after,
"sleep": sleep,
"rng": fetch_rng,
"client": fetch_client,
"first_outcome": outcome,
"catalog": catalog,
},
name="litellm-model-cost-map-retry",
daemon=True,
).start()
if not isinstance(outcome, ModelCostMapReloaded):
verbose_logger.warning(
"LiteLLM: Failed to fetch remote model cost map from %s: %s. Falling back to local backup.",
url,
outcome.reason,
)
_cost_map_source_info.source = "local"
_cost_map_source_info.fallback_reason = f"Remote fetch failed: {outcome.reason}"
return _note_loaded(ModelCostMapReloaded(fetched=CATALOG.bundled())).fetched
_cost_map_source_info.source = "remote"
_cost_map_source_info.fallback_reason = None
return _note_loaded(outcome).fetched
verbose_logger.warning(
"LiteLLM: Failed to fetch remote model cost map from %s: %s. Falling back to local backup.",
url,
fetched.reason,
)
return _note_loaded(catalog.bundled()), False, f"Remote fetch failed: {fetched.reason}"
def install_initial_model_cost_map(url: str) -> None:
def install_initial_model_cost_map(
url: str,
catalog: Catalog = CATALOG,
sleep: Callable[[float], None] = time.sleep,
max_attempts: int = MODEL_COST_MAP_FETCH_MAX_ATTEMPTS,
) -> Installed:
"""Fills ``litellm.model_cost`` at import with the map main's rules pick"""
_install(
_pick_model_cost_map(
url=url,
timeout=5,
max_attempts=MODEL_COST_MAP_FETCH_MAX_ATTEMPTS,
sleep=time.sleep,
rng=None,
client=None,
)
fetched, env_forced, fallback_reason = _pick_model_cost_map(
url=url,
timeout=_IMPORT_FETCH_TIMEOUT_SECONDS,
max_attempts=max_attempts,
sleep=sleep,
catalog=catalog,
)
return _install(
fetched, catalog, env_forced=env_forced, fallback_reason=fallback_reason, url=None if env_forced else url
)
def get_model_cost_map(
url: str,
timeout: int = 5,
timeout: int = _IMPORT_FETCH_TIMEOUT_SECONDS,
max_attempts: int = MODEL_COST_MAP_FETCH_MAX_ATTEMPTS,
sleep: Callable[[float], None] = time.sleep,
rng: random.Random | None = None,
client: "_SyncGetClient | None" = None,
rng: object = None,
client: object = None,
) -> dict[str, dict[str, JsonValue]]:
"""
Public entry point, returns a fresh model cost map dict picked by main's rules. A remote body is parsed and
validated by the Rust catalog, and the bundled map is returned on any failure
Public entry point, returns a fresh model cost map dict picked by main's rules. The fetch, its retries and the
validation of the body run in Rust, and the bundled map is returned on any failure. ``rng`` and ``client`` are
accepted for compatibility and ignored, since Rust owns the HTTP client and the backoff jitter
"""
return _pick_model_cost_map(
url=url, timeout=timeout, max_attempts=max_attempts, sleep=sleep, rng=rng, client=client
).served()
if rng is not None or client is not None:
warnings.warn(
"get_model_cost_map ignores rng and client, the fetch runs in Rust", DeprecationWarning, stacklevel=2
)
fetched, _, _ = _pick_model_cost_map(
url=url, timeout=timeout, max_attempts=max_attempts, sleep=sleep, catalog=CATALOG
)
return fetched.served()

View file

@ -211,7 +211,6 @@ if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from prisma import models as prisma_models
from litellm._catalog import Fetched
from litellm.integrations.opentelemetry import OpenTelemetry
from litellm.proxy.health_check_utils.shared_health_check_manager import SharedHealthCheckManager
@ -5386,12 +5385,6 @@ def resolve_classifier_plugin(
return resolved
def _swap_in_model_cost_map(fetched: "Fetched") -> int:
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
return adopt_model_cost_map(fetched)
def _websearch_handler_params(stored: Mapping[str, object]) -> dict[str, object]:
"""
Translate stored web search interception settings into handler kwargs.
@ -9091,13 +9084,11 @@ class ProxyConfig:
if not is_due:
return
from litellm.litellm_core_utils.get_model_cost_map import (
ModelCostMapReloadUnavailable,
refetch_model_cost_map,
)
from litellm._catalog import Unavailable, default_source
from litellm.litellm_core_utils.get_model_cost_map import reload_model_cost_map
reload_result = await refetch_model_cost_map(url=litellm.model_cost_map_url)
if isinstance(reload_result, ModelCostMapReloadUnavailable):
reload_result = await asyncio.to_thread(reload_model_cost_map, default_source())
if isinstance(reload_result, Unavailable):
verbose_proxy_logger.warning(
"Model cost map reload failed (%s); keeping current pricing data. The revision stays "
"unapplied so this pod retries on its next poll",
@ -9105,7 +9096,7 @@ class ProxyConfig:
)
return
models_count = _swap_in_model_cost_map(reload_result.fetched)
models_count = reload_result.count
self.model_cost_map_loaded_at = current_time
await record_reload_run(prisma_client, MODEL_COST_MAP_RELOAD_PARAM_NAME, current_time)
# Adopted last, so neither a failed fetch nor a failed status write is recorded
@ -20999,20 +20990,20 @@ async def reload_model_cost_map(
raise HTTPException(status_code=500, detail="Database connection not available")
# Immediately reload the model cost map in the current pod
from litellm._catalog import Unavailable, default_source
from litellm.litellm_core_utils.get_model_cost_map import (
ModelCostMapReloadUnavailable,
get_model_cost_map_provenance,
refetch_model_cost_map,
reload_model_cost_map,
)
reload_result = await refetch_model_cost_map(url=litellm.model_cost_map_url)
if isinstance(reload_result, ModelCostMapReloadUnavailable):
reload_result = await asyncio.to_thread(reload_model_cost_map, default_source())
if isinstance(reload_result, Unavailable):
raise HTTPException(
status_code=502,
detail=f"Failed to reload model cost map: {reload_result.reason}. Current pricing data was kept.",
)
models_count = _swap_in_model_cost_map(reload_result.fetched)
models_count = reload_result.count
current_time = utc_now()
proxy_config.model_cost_map_loaded_at = current_time
provenance: Final = get_model_cost_map_provenance()

View file

@ -1,5 +1,7 @@
import os
from asyncio import Future
from collections.abc import Mapping, Sequence
from datetime import datetime
from types import ModuleType
from typing import Literal, Never, final
@ -70,17 +72,50 @@ class Installed:
@final
class Unavailable:
reason: str
retry_after: float | None
def __new__(cls, reason: str, retry_after: float | None = None) -> Unavailable: ...
@final
class Remote:
url: str
def __new__(cls, url: str) -> Remote: ...
@final
class File:
path: str
def __new__(cls, path: str | os.PathLike[str]) -> File: ...
@final
class Bundled:
def __new__(cls) -> Bundled: ...
@final
class Provenance:
source: str
url: str | None
is_env_forced: bool
fallback_reason: str | None
source_revision: str
etag: str | None
@property
def loaded_at(self) -> datetime: ...
@final
class Catalog:
@staticmethod
def bound_to(module: ModuleType, attr: str) -> Catalog: ...
def bundled(self) -> Fetched: ...
@staticmethod
def parse(
body: bytes, *, etag: str | None, min_model_count: int, max_shrink_ratio: float
) -> Fetched | Unavailable: ...
def install(self, fetched: Fetched) -> Installed: ...
def fetch(self, source: Remote | File | Bundled, *, timeout: float, attempts: int) -> Fetched | Unavailable: ...
def reload(self, source: Remote | File | Bundled) -> Installed | Unavailable: ...
def provenance(self) -> Provenance | None: ...
def install(
self,
fetched: Fetched,
*,
env_forced: bool = False,
fallback_reason: str | None = None,
url: str | None = None,
) -> Installed: ...
@final
class NativeClickHouseSpendConfig:

View file

View file

@ -0,0 +1,88 @@
import contextlib
import datetime
import ipaddress
import ssl
import threading
import types
from collections.abc import Generator
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import pytest
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.x509.oid import NameOID
import litellm
from litellm._catalog import CATALOG, Catalog, Installed, Remote, Unavailable
def _self_signed(directory: Path) -> tuple[Path, Path]:
key = ec.generate_private_key(ec.SECP256R1())
name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "127.0.0.1")])
now = datetime.datetime.now(datetime.timezone.utc)
certificate = (
x509.CertificateBuilder()
.subject_name(name)
.issuer_name(name)
.public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(now - datetime.timedelta(days=1))
.not_valid_after(now + datetime.timedelta(days=1))
.add_extension(x509.SubjectAlternativeName([x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), critical=False)
.sign(key, hashes.SHA256())
)
cert_path = directory / "cert.pem"
key_path = directory / "key.pem"
cert_path.write_bytes(certificate.public_bytes(serialization.Encoding.PEM))
key_path.write_bytes(
key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption())
)
return cert_path, key_path
@contextlib.contextmanager
def _https_serving(directory: Path, body: bytes) -> Generator[str]:
class Handler(BaseHTTPRequestHandler):
def do_GET(self) -> None:
self.send_response(200)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, *args: object) -> None:
pass
cert_path, key_path = _self_signed(directory)
httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
context.load_cert_chain(cert_path, key_path)
httpd.socket = context.wrap_socket(httpd.socket, server_side=True)
thread = threading.Thread(target=httpd.serve_forever, daemon=True)
thread.start()
try:
yield f"https://127.0.0.1:{httpd.server_address[1]}/model_prices.json"
finally:
httpd.shutdown()
httpd.server_close()
@pytest.mark.parametrize(("ssl_verify", "installed"), [(False, True), (True, False)])
def test_the_fetch_uses_the_http_settings_provider_calls_use(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ssl_verify: bool, installed: bool
):
monkeypatch.setattr(litellm, "ssl_verify", ssl_verify)
module = types.ModuleType("fake_litellm")
module.model_cost = {}
catalog = Catalog.bound_to(module, "model_cost")
body = Path(litellm.__file__).resolve().parents[1].joinpath("model_prices_and_context_window.json").read_bytes()
with _https_serving(tmp_path, body) as url:
result = catalog.reload(Remote(url))
assert isinstance(result, Installed if installed else Unavailable), getattr(result, "reason", None)
def test_a_reload_of_an_unparseable_url_is_unavailable():
assert isinstance(CATALOG.fetch(Remote("not a url"), timeout=1, attempts=1), Unavailable)

File diff suppressed because it is too large Load diff

View file

@ -64,13 +64,22 @@ _EXPECTED_RESYNC_APPLIED_CONFIG_PARAM_NAMES = frozenset(
_STARTUP_ONLY_CONFIG_PARAM_NAMES = ("environment_variables",)
def _reloaded(cost_map: dict) -> "ModelCostMapReloaded":
from litellm._catalog import Catalog, Fetched
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
from litellm.litellm_core_utils.get_model_cost_map import reload_model_cost_map as _REAL_RELOAD
parsed = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0)
assert isinstance(parsed, Fetched)
return ModelCostMapReloaded(fetched=parsed)
def _reloading(cost_map: dict):
"""A stand-in reload that installs ``cost_map`` over the bundled map, read from a file like a fetched map"""
import tempfile
from pathlib import Path
from litellm._catalog import CATALOG, File
def reload(source, catalog=CATALOG):
path = Path(tempfile.mkdtemp()) / "model_prices.json"
path.write_text(json.dumps({**CATALOG.bundled().document(), **cost_map}))
return _REAL_RELOAD(File(path), catalog)
return reload
class _RecordingRedisClient(Redis):
@ -864,12 +873,11 @@ async def test_model_cost_map_reload_does_not_publish_config_change() -> None:
original_model_cost = litellm.model_cost.copy()
_set_redis_usage_cache(_FakeRedisCache(client))
try:
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
with patch(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
new=AsyncMock(
return_value=_reloaded({"gpt-5.2": {"input_cost_per_token": 0.001}})
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map",
new=MagicMock(
side_effect=_reloading({"gpt-5.2": {"input_cost_per_token": 0.001}})
),
):
await ProxyConfig()._check_and_reload_model_cost_map(prisma_client=prisma_client)

View file

@ -32,13 +32,22 @@ _ROOT_COST_MAP = Path(__file__).resolve().parents[4] / "model_prices_and_context
# ---------------------------------------------------------------------------
def _reloaded(cost_map: dict) -> "ModelCostMapReloaded":
from litellm._catalog import Catalog, Fetched
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
from litellm.litellm_core_utils.get_model_cost_map import reload_model_cost_map as _REAL_RELOAD
parsed = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0)
assert isinstance(parsed, Fetched)
return ModelCostMapReloaded(fetched=parsed)
def _reloading(cost_map: dict):
"""A stand-in reload that installs ``cost_map`` over the bundled map, read from a file like a fetched map"""
import tempfile
from pathlib import Path
from litellm._catalog import CATALOG, File
def reload(source, catalog=CATALOG):
path = Path(tempfile.mkdtemp()) / "model_prices.json"
path.write_text(json.dumps({**CATALOG.bundled().document(), **cost_map}))
return _REAL_RELOAD(File(path), catalog)
return reload
def _attach_litellm_config(mock_prisma):
@ -73,8 +82,8 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma):
fake_cost_map = {"gpt-4": {"input_cost": 0.03}, "gpt-3.5": {"input_cost": 0.002}}
monkeypatch.setattr(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
AsyncMock(return_value=_reloaded(fake_cost_map)),
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map",
MagicMock(side_effect=_reloading(fake_cost_map)),
)
monkeypatch.setattr("litellm.add_known_models", lambda model_cost_map=None: None)
monkeypatch.setattr("litellm.model_cost", {}, raising=False)
@ -110,7 +119,6 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma):
def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every_status_surface(
client, auth_as, monkeypatch, mock_prisma
):
import httpx
import litellm
import hashlib
@ -125,11 +133,23 @@ def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every
"source_revision": hashlib.sha1(b"blob %d\0" % len(body) + body, usedforsecurity=False).hexdigest(),
"etag": _SERVED_ETAG,
}
served = httpx.Response(200, headers={"ETag": _SERVED_ETAG}, content=body)
monkeypatch.setattr(
"litellm.litellm_core_utils.get_model_cost_map._default_reload_client",
lambda: httpx.AsyncClient(transport=httpx.MockTransport(lambda request: served)),
)
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
class Handler(BaseHTTPRequestHandler):
def do_GET(self) -> None:
self.send_response(200)
self.send_header("ETag", _SERVED_ETAG)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, *args: object) -> None:
pass
httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
threading.Thread(target=httpd.serve_forever, daemon=True).start()
monkeypatch.setattr(litellm, "model_cost_map_url", f"http://127.0.0.1:{httpd.server_address[1]}/map.json")
monkeypatch.setattr("litellm.add_known_models", lambda model_cost_map=None: None)
monkeypatch.setattr("litellm.model_cost", {}, raising=False)
@ -155,6 +175,8 @@ def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every
assert {key: status_response.json()[key] for key in expected} == expected
assert public_response.status_code == 200
assert "gpt-4o" in public_response.json()
httpd.shutdown()
httpd.server_close()
assert reload_body["models_count"] == len(litellm.model_cost)
@ -166,9 +188,7 @@ def test_reload_model_cost_map_fetch_failure_502_keeps_map(
Regression: the endpoint used to report success after silently swapping in
the stale packaged backup.
"""
from litellm.litellm_core_utils.get_model_cost_map import (
ModelCostMapReloadUnavailable,
)
from litellm._catalog import Unavailable
from litellm.proxy import proxy_server as ps
from litellm.proxy._types import LitellmUserRoles
@ -178,9 +198,9 @@ def test_reload_model_cost_map_fetch_failure_502_keeps_map(
sentinel_map = {"existing-model": {"input_cost": 0.01}}
monkeypatch.setattr("litellm.model_cost", sentinel_map, raising=False)
monkeypatch.setattr(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
AsyncMock(
return_value=ModelCostMapReloadUnavailable(
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map",
MagicMock(
return_value=Unavailable(
reason="HTTP 429 from upstream (after 3 attempts)"
)
),
@ -415,7 +435,6 @@ def test_get_model_cost_map_reload_status_no_config_not_scheduled(
client, auth_as, monkeypatch, mock_prisma
):
"""A row left behind by a manual reload (interval_hours=None) → not scheduled."""
from datetime import datetime, timezone
from litellm.proxy import proxy_server as ps
from litellm.proxy._types import LitellmUserRoles

View file

@ -33,7 +33,6 @@ from litellm._logging import turn_on_json, verbose_proxy_logger
from litellm.caching.caching import RedisCache
from litellm.caching.dual_cache import DualCache
from litellm.caching.redis_cluster_cache import RedisClusterCache
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
from litellm.proxy._types import (
AwsErrorTypeProxyException,
LitellmUserRoles,
@ -80,13 +79,22 @@ example_embedding_result = {
}
def _reloaded(cost_map: dict) -> "ModelCostMapReloaded":
from litellm._catalog import Catalog, Fetched
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
from litellm.litellm_core_utils.get_model_cost_map import reload_model_cost_map as _REAL_RELOAD
parsed = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0)
assert isinstance(parsed, Fetched)
return ModelCostMapReloaded(fetched=parsed)
def _reloading(cost_map: dict):
"""A stand-in reload that installs ``cost_map`` over the bundled map, read from a file like a fetched map"""
import tempfile
from pathlib import Path
from litellm._catalog import CATALOG, File
def reload(source, catalog=CATALOG):
path = Path(tempfile.mkdtemp()) / "model_prices.json"
path.write_text(json.dumps({**CATALOG.bundled().document(), **cost_map}))
return _REAL_RELOAD(File(path), catalog)
return reload
def mock_patch_aembedding():
@ -4553,14 +4561,13 @@ class TestPriceDataReloadAPI:
# Save the original model_cost so the endpoint's direct assignment
# (litellm.model_cost = new_model_cost_map) does not contaminate
# subsequent tests running in the same worker process.
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
original_model_cost = litellm.model_cost.copy()
try:
with patch(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
new=AsyncMock(
return_value=_reloaded({"gpt-3.5-turbo": {"input_cost_per_token": 0.001}})
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map",
new=MagicMock(
side_effect=_reloading({"gpt-3.5-turbo": {"input_cost_per_token": 0.001}})
),
):
# Mock the database connection
@ -4614,8 +4621,8 @@ class TestPriceDataReloadAPI:
def test_reload_model_cost_map_error_handling(self, client_with_auth):
"""Test error handling in the reload endpoint"""
with patch(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
new=AsyncMock(side_effect=Exception("Network error")),
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map",
new=MagicMock(side_effect=Exception("Network error")),
):
# Mock the database connection
with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma:
@ -4828,13 +4835,11 @@ class TestPriceDataReloadIntegration:
"gpt-4": {"input_cost_per_token": 0.03, "output_cost_per_token": 0.06},
}
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
original_model_cost = litellm.model_cost.copy()
try:
with patch(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
new=AsyncMock(return_value=_reloaded(mock_cost_map)),
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map",
new=MagicMock(side_effect=_reloading(mock_cost_map)),
):
# Mock the database connection
with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma:
@ -4897,17 +4902,15 @@ class TestPriceDataReloadIntegration:
proxy_config.model_cost_map_loaded_at = frozen_now - timedelta(minutes=1)
proxy_config.model_cost_map_applied_revision = 3
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
original_model_cost = litellm.model_cost.copy()
try:
with (
patch(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new_callable=AsyncMock
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", new_callable=MagicMock
) as mock_get_map,
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
):
mock_get_map.return_value = _reloaded({"gpt-3.5-turbo": {"input_cost_per_token": 0.001}}
mock_get_map.side_effect = _reloading({"gpt-3.5-turbo": {"input_cost_per_token": 0.001}}
)
asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma))
@ -4950,7 +4953,7 @@ class TestPriceDataReloadIntegration:
try:
with (
patch(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new_callable=AsyncMock
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", new_callable=MagicMock
) as mock_get_map,
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
):
@ -4987,11 +4990,11 @@ class TestPriceDataReloadIntegration:
try:
with (
patch(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new_callable=AsyncMock
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", new_callable=MagicMock
) as mock_get_map,
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
):
mock_get_map.return_value = _reloaded({"gpt-4-test": {"input_cost_per_token": 0.5}}
mock_get_map.side_effect = _reloading({"gpt-4-test": {"input_cost_per_token": 0.5}}
)
asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma))
@ -5031,11 +5034,11 @@ class TestPriceDataReloadIntegration:
try:
with (
patch(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new_callable=AsyncMock
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", new_callable=MagicMock
) as mock_get_map,
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
):
mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}}
mock_get_map.side_effect = _reloading({"gpt-4": {"input_cost_per_token": 0.001}}
)
for _ in range(3):
@ -5074,11 +5077,11 @@ class TestPriceDataReloadIntegration:
try:
with (
patch(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new_callable=AsyncMock
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", new_callable=MagicMock
) as mock_get_map,
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
):
mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}}
mock_get_map.side_effect = _reloading({"gpt-4": {"input_cost_per_token": 0.001}}
)
asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma))
@ -5109,11 +5112,11 @@ class TestPriceDataReloadIntegration:
try:
with (
patch(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new_callable=AsyncMock
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", new_callable=MagicMock
) as mock_get_map,
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
):
mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}}
mock_get_map.side_effect = _reloading({"gpt-4": {"input_cost_per_token": 0.001}}
)
asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma))
@ -5149,12 +5152,12 @@ class TestPriceDataReloadIntegration:
try:
with (
patch(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
new_callable=AsyncMock,
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map",
new_callable=MagicMock,
) as mock_get_map,
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
):
mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.1}}
mock_get_map.side_effect = _reloading({"gpt-4": {"input_cost_per_token": 0.1}}
)
asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma))
@ -5174,9 +5177,7 @@ class TestPriceDataReloadIntegration:
so a pod that records it as applied without the data stays mispriced until someone
clicks again
"""
from litellm.litellm_core_utils.get_model_cost_map import (
ModelCostMapReloadUnavailable,
)
from litellm._catalog import Unavailable
from litellm.proxy.proxy_server import ProxyConfig
proxy_config = ProxyConfig()
@ -5193,8 +5194,8 @@ class TestPriceDataReloadIntegration:
original_model_cost = litellm.model_cost
with (
patch(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
new=AsyncMock(return_value=ModelCostMapReloadUnavailable(reason="HTTP 429 from upstream")),
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map",
new=MagicMock(return_value=Unavailable(reason="HTTP 429 from upstream")),
),
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
):
@ -5219,7 +5220,6 @@ class TestPriceDataReloadIntegration:
after it is swallowed by the surrounding except and would otherwise
leave the metadata correct while the path is quietly broken"""
from litellm import utils as litellm_utils
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
from litellm.proxy.proxy_server import ProxyConfig
proxy_config = ProxyConfig()
@ -5240,9 +5240,9 @@ class TestPriceDataReloadIntegration:
with (
patch(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
new=AsyncMock(
return_value=_reloaded({"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}}
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map",
new=MagicMock(
side_effect=_reloading({"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}}
)
),
),
@ -5261,13 +5261,12 @@ class TestPriceDataReloadIntegration:
litellm_utils._runtime_registered_model_cost.update(original_registry)
_invalidate_model_cost_lowercase_map()
def test_swap_in_model_cost_map_counts_the_fetched_catalog_only(self):
def test_a_reload_counts_the_fetched_catalog_only(self):
"""The count the reload endpoints report describes the price data, so it
is taken before the runtime registrations are written back into the same
dict. Counting after would inflate it by however many deployments and
overrides this pod happens to be carrying"""
from litellm import utils as litellm_utils
from litellm.proxy.proxy_server import _swap_in_model_cost_map
original_model_cost = litellm.model_cost
original_registry = dict(litellm_utils._runtime_registered_model_cost)
@ -5276,9 +5275,9 @@ class TestPriceDataReloadIntegration:
model_cost={"custom/deployment-model": {"litellm_provider": "custom", "max_input_tokens": 4321}}
)
models_count = _swap_in_model_cost_map(_reloaded({"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}}).fetched)
installed = _reloading({"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}})(None)
assert models_count == len(litellm.model_cost) - 1
assert installed.count == len(litellm.model_cost) - 1
assert litellm.model_cost["custom/deployment-model"]["max_input_tokens"] == 4321
finally:
litellm.model_cost = original_model_cost
@ -5305,18 +5304,16 @@ class TestPriceDataReloadIntegration:
client = TestClient(app)
frozen_now = datetime(2024, 1, 1, 7, 0, tzinfo=timezone.utc)
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
original_model_cost = litellm.model_cost.copy()
try:
with (
patch(
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new_callable=AsyncMock
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", new_callable=MagicMock
) as mock_get_map,
patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma,
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
):
mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}}
mock_get_map.side_effect = _reloading({"gpt-4": {"input_cost_per_token": 0.001}}
)
mock_prisma.db.litellm_config.upsert = AsyncMock(
return_value=_reload_schedule_row({}, reload_revision=9)

View file

@ -1,4 +1,5 @@
import asyncio
import json
import sys
from collections.abc import Iterator
from dataclasses import dataclass
@ -12,8 +13,8 @@ from litellm import Router
from litellm import router as litellm_router_module
from litellm import utils as litellm_utils_module
from litellm.caching.caching import DualCache
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from tests.unit import conftest as unit_harness
CANARY_MODEL = "conftest-isolation-canary-model"
@ -179,11 +180,14 @@ def registered_collection_models() -> Iterator[None]:
litellm_router_module._live_routers.update(original_routers)
def test_collection_model_metadata_is_not_replayed_into_a_test_catalog() -> None:
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
def test_collection_model_metadata_is_not_replayed_into_a_test_catalog(tmp_path) -> None:
from litellm._catalog import CATALOG, File
from litellm.litellm_core_utils.get_model_cost_map import reload_model_cost_map
catalog: Final = {"unit-test-catalog-model": {"litellm_provider": "openai", "input_cost_per_token": 0.001}}
adopt_model_cost_map(catalog)
path: Final = tmp_path / "model_prices.json"
path.write_text(json.dumps({**CATALOG.bundled().document(), **catalog}))
reload_model_cost_map(File(path))
assert "collection-isolation-ledger" not in litellm.model_cost
assert "collection-isolation-router" not in litellm.model_cost
assert "unit-test-catalog-model" in litellm.model_cost

View file

@ -4,19 +4,31 @@ Tests for the ``aliases`` feature in the model cost map.
The Rust catalog serves every name in an ``aliases`` list as its own key, sharing one dict with the canonical entry
"""
import functools
import json
import tempfile
from pathlib import Path
from litellm._catalog import Catalog, Fetched, Unavailable
from litellm._catalog import CATALOG, Fetched, File, Unavailable
def _parsed(model_cost: dict) -> Fetched | Unavailable:
return Catalog.parse(json.dumps(model_cost).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0)
@functools.lru_cache(maxsize=1)
def _bundled_document() -> dict:
return CATALOG.bundled().document()
def _parsed(model_cost: dict, over_bundled: bool = True) -> Fetched | Unavailable:
path = Path(tempfile.mkdtemp()) / "model_prices.json"
path.write_text(json.dumps({**_bundled_document(), **model_cost} if over_bundled else model_cost))
return CATALOG.fetch(File(path), timeout=5, attempts=1)
def _expand(model_cost: dict) -> dict:
"""The rows the fetched map serves beyond the bundled ones, aliases included"""
parsed = _parsed(model_cost)
assert isinstance(parsed, Fetched)
return parsed.served()
bundled = _bundled_document()
return {key: row for key, row in parsed.served().items() if key not in bundled or key in model_cost}
def _error_kinds(model_cost: dict) -> list[str]:
@ -247,4 +259,4 @@ class TestAliasIntegration:
assert "model-with-alias" in result
def test_an_empty_map_is_unavailable(self):
assert isinstance(_parsed({}), Unavailable)
assert isinstance(_parsed({}, over_bundled=False), Unavailable)

View file

@ -14,22 +14,22 @@ import itertools
import json
import os
from collections.abc import Callable, Mapping
from pathlib import Path
from typing import Final, NoReturn
import pytest
import litellm
from litellm._catalog import Catalog, Fetched
from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry, adopt_model_cost_map
from litellm._catalog import File
from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry, reload_model_cost_map
from litellm.main import _build_custom_pricing_entry
from litellm.utils import _invalidate_model_cost_lowercase_map
def _fetched(cost_map: Mapping[str, object]) -> Fetched:
parsed: Final = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0)
assert isinstance(parsed, Fetched)
return parsed
def _reload_from(cost_map: Mapping[str, object], directory: Path) -> object:
path: Final = directory / "model_prices.json"
path.write_text(json.dumps(cost_map))
return reload_model_cost_map(File(path))
def _snapshot_model_cost_entries(keys):
@ -1027,7 +1027,9 @@ def test_register_model_prices_a_provider_less_azure_ai_key_without_repricing_th
_invalidate_model_cost_lowercase_map()
def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers(monkeypatch: pytest.MonkeyPatch) -> None:
def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
catalog_key: Final = "au.anthropic.lit-5853-reloaded-model"
deployment_key: Final = f"bedrock/{catalog_key}"
catalog_row: Final = {
@ -1046,12 +1048,12 @@ def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers(monk
catalog_key: {**catalog_row, "input_cost_per_token": reloaded_input_cost},
}
real_lookup: Final = litellm.utils.get_model_info
reloads: Final[list[int]] = [] # mutable-ok: counts reloads the racing lookup triggered
reloads: Final[list[object]] = [] # mutable-ok: counts reloads the racing lookup triggered
def lookup_racing_a_price_data_reload(model: str, custom_llm_provider: str | None = None) -> object:
info: Final = real_lookup(model=model, custom_llm_provider=custom_llm_provider)
if not reloads:
reloads.append(adopt_model_cost_map(_fetched(reloaded_cost_map)))
reloads.append(_reload_from(reloaded_cost_map, tmp_path))
return info
with pytest.MonkeyPatch.context() as lookup_patch:
@ -1363,10 +1365,10 @@ def test_update_dictionary_merges_nested_dicts_without_aliasing():
object stays untouched, and the caller's incoming nested dict is never
inserted by reference into the merged result.
"""
from litellm.utils import update_dictionary
from typing import cast
from litellm.utils import update_dictionary
existing_nested = cast(dict[str, object], {"hours_utc": "01:00-02:00", 1: "existing"})
existing = {"off_peak_pricing": existing_nested}
incoming_nested = cast(

View file

@ -54,8 +54,8 @@ _COPILOT_TOKEN_EXPIRES_AT: Final = 4_102_444_800
def _simulate_price_data_reload(fetched_catalog):
"""Drive what a price data reload does to this process's litellm state.
Mirrors `litellm.proxy.proxy_server._swap_in_model_cost_map`, which is the
one place both reload paths adopt a freshly fetched catalog; that wiring is
Mirrors `litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map`, which is
the one place both reload paths install a freshly fetched catalog; that wiring is
covered in the proxy's own tests, so these exercise the replay itself
without dragging the proxy in. The provider model sets that helper also
repopulates are left alone, since nothing here reads them and rebuilding
@ -3858,7 +3858,7 @@ def test_every_declaring_deployment_is_named(caplog):
def _simulate_price_data_reload_with_provider_sets(monkeypatch, fetched_catalog):
"""Like `_simulate_price_data_reload`, plus the provider model-set refresh the proxy's
`_swap_in_model_cost_map` does before replaying, so bare names in the new catalog resolve."""
`reload_model_cost_map` does before replaying, so bare names in the new catalog resolve."""
monkeypatch.setattr(litellm, "model_cost", fetched_catalog)
_invalidate_model_cost_lowercase_map()
litellm.add_known_models(model_cost_map=fetched_catalog)

View file

@ -2454,16 +2454,34 @@ def test_register_model_with_scientific_notation():
@pytest.mark.usefixtures("isolated_openai_model_sets")
@respx.mock
def test_register_model_url_fetch_uses_single_attempt(monkeypatch):
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False)
monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost))
before = dict(litellm.model_cost)
threads_before = {thread.name for thread in threading.enumerate()}
route = respx.get("https://example.invalid/custom_pricing.json").mock(return_value=httpx.Response(503))
requests: list[str] = []
litellm.register_model(model_cost="https://example.invalid/custom_pricing.json")
class Unavailable(BaseHTTPRequestHandler):
def do_GET(self) -> None:
requests.append(self.path)
self.send_response(503)
self.send_header("Content-Length", "0")
self.end_headers()
def log_message(self, *args: object) -> None:
pass
httpd = ThreadingHTTPServer(("127.0.0.1", 0), Unavailable)
threading.Thread(target=httpd.serve_forever, daemon=True).start()
try:
litellm.register_model(model_cost=f"http://127.0.0.1:{httpd.server_address[1]}/custom_pricing.json")
finally:
httpd.shutdown()
httpd.server_close()
threads_after = {thread.name for thread in threading.enumerate()}
assert route.call_count == 1
assert requests == ["/custom_pricing.json"]
assert not (threads_after - threads_before) & {"litellm-model-cost-map-retry"}
assert not any(
thread.name == "litellm-model-cost-map-retry" and thread.is_alive() for thread in threading.enumerate()