From 18d2c77042cff3f75f71ccd4ede39b775c10dce8 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 28 Oct 2026 00:00:42 -0700 Subject: [PATCH] refactor(model-cost): fetch and reload the cost map through Rust model-catalog-refresh fetches over HTTP, S3 and file with main's retry rules, python-bridge drives it with the GIL released, and provenance is one Rust snapshot recorded by install. The S3 client setup moves into a shared s3 crate used by cache-s3. Co-Authored-By: Claude Opus 5.5 --- litellm-rust/Cargo.lock | 30 +- litellm-rust/Cargo.toml | 1 + litellm-rust/crates/cache-s3/Cargo.toml | 1 + litellm-rust/crates/cache-s3/src/cache.rs | 31 +- litellm-rust/crates/cache-s3/src/lib.rs | 1 - .../crates/model-catalog-refresh/Cargo.toml | 18 + .../crates/model-catalog-refresh/src/error.rs | 9 + .../crates/model-catalog-refresh/src/file.rs | 25 + .../crates/model-catalog-refresh/src/http.rs | 82 ++ .../crates/model-catalog-refresh/src/lib.rs | 60 + .../model-catalog-refresh/src/policy.rs | 14 + .../model-catalog-refresh/src/revalidate.rs | 410 +++++++ .../crates/model-catalog-refresh/src/s3.rs | 171 +++ .../model-catalog-refresh/src/source.rs | 87 +- .../crates/model-catalog-refresh/src/step.rs | 2 + .../model-catalog-refresh/tests/sources.rs | 197 ++++ litellm-rust/crates/python-bridge/Cargo.toml | 2 +- litellm-rust/crates/python-bridge/src/lib.rs | 8 +- .../python-bridge/src/model_registry.rs | 181 +-- .../python-bridge/src/model_registry/fetch.rs | 307 +++++ litellm-rust/crates/s3/Cargo.toml | 15 + litellm-rust/crates/s3/src/lib.rs | 35 + .../crates/{cache-s3 => s3}/src/transport.rs | 2 +- litellm/_catalog/__init__.py | 41 +- .../litellm_core_utils/get_model_cost_map.py | 545 ++------- litellm/proxy/proxy_server.py | 29 +- litellm/rust_bridge/_native.pyi | 45 +- tests/unit/_catalog/__init__.py | 0 tests/unit/_catalog/test_fetch.py | 88 ++ .../test_get_model_cost_map.py | 1045 +++++++---------- .../common_utils/test_config_sync_pubsub.py | 28 +- .../test_routes_model_cost_map.py | 61 +- ...test_proxy_server_endpoints_and_startup.py | 97 +- tests/unit/test_conftest_isolation.py | 12 +- tests/unit/test_model_cost_aliases.py | 22 +- .../test_register_model_custom_pricing.py | 26 +- .../unit/test_router_model_cost_isolation.py | 6 +- tests/unit/test_utils.py | 24 +- 38 files changed, 2451 insertions(+), 1307 deletions(-) create mode 100644 litellm-rust/crates/model-catalog-refresh/src/error.rs create mode 100644 litellm-rust/crates/model-catalog-refresh/src/file.rs create mode 100644 litellm-rust/crates/model-catalog-refresh/src/http.rs create mode 100644 litellm-rust/crates/model-catalog-refresh/src/revalidate.rs create mode 100644 litellm-rust/crates/model-catalog-refresh/src/s3.rs create mode 100644 litellm-rust/crates/model-catalog-refresh/tests/sources.rs create mode 100644 litellm-rust/crates/python-bridge/src/model_registry/fetch.rs create mode 100644 litellm-rust/crates/s3/Cargo.toml create mode 100644 litellm-rust/crates/s3/src/lib.rs rename litellm-rust/crates/{cache-s3 => s3}/src/transport.rs (95%) create mode 100644 tests/unit/_catalog/__init__.py create mode 100644 tests/unit/_catalog/test_fetch.py diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 076785c2dd4..6f8e3b52432 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3618,6 +3618,7 @@ dependencies = [ "litellm-cache", "litellm-cache-testing", "litellm-http", + "litellm-s3", "reqwest 0.12.28", "rstest", "serde_json", @@ -4255,8 +4256,22 @@ dependencies = [ name = "litellm-model-catalog-refresh" version = "0.1.0" dependencies = [ + "aws-config", + "aws-credential-types", + "aws-sdk-s3", + "aws-smithy-runtime-api", + "litellm-http", "litellm-model-catalog", + "litellm-s3", + "rand 0.8.7", + "reqwest 0.12.28", + "rstest", + "serde_json", + "sha1 0.10.7", + "thiserror 2.0.19", + "tokio", "url", + "wiremock", ] [[package]] @@ -4289,6 +4304,7 @@ dependencies = [ "litellm-llms", "litellm-llms-types", "litellm-model-catalog", + "litellm-model-catalog-refresh", "litellm-router-types", "litellm-secrets", "litellm-secrets-aws", @@ -4306,7 +4322,6 @@ dependencies = [ "serde", "serde_json", "serde_with", - "sha1 0.10.7", "sqlx", "strum", "thiserror 2.0.19", @@ -4376,6 +4391,19 @@ dependencies = [ "serde_with", ] +[[package]] +name = "litellm-s3" +version = "0.1.0" +dependencies = [ + "aws-credential-types", + "aws-sdk-s3", + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.4.2", + "litellm-http", + "reqwest 0.12.28", +] + [[package]] name = "litellm-secrets" version = "0.1.0" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index f1acd430a51..035f55c9f53 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -75,6 +75,7 @@ litellm-redis-testing = { path = "crates/redis-testing" } litellm-cost = { path = "crates/cost" } litellm-model-catalog = { path = "crates/model-catalog" } litellm-model-catalog-refresh = { path = "crates/model-catalog-refresh" } +litellm-s3 = { path = "crates/s3" } askama = { version = "0.16.1", default-features = false, features = ["derive", "std"] } tracing = "0.1" diff --git a/litellm-rust/crates/cache-s3/Cargo.toml b/litellm-rust/crates/cache-s3/Cargo.toml index eb3a2fff1ac..22eff594fa8 100644 --- a/litellm-rust/crates/cache-s3/Cargo.toml +++ b/litellm-rust/crates/cache-s3/Cargo.toml @@ -9,6 +9,7 @@ repository.workspace = true litellm-http.workspace = true litellm-cache.workspace = true litellm-auth-aws.workspace = true +litellm-s3.workspace = true aws-sdk-s3 = { version = "1.146.1", default-features = false, features = ["rustls", "rt-tokio"] } aws-credential-types = "1.3.0" aws-smithy-runtime-api = { version = "1.16.2", features = ["client", "http-1x"] } diff --git a/litellm-rust/crates/cache-s3/src/cache.rs b/litellm-rust/crates/cache-s3/src/cache.rs index ced948e80c6..1d5a266275a 100644 --- a/litellm-rust/crates/cache-s3/src/cache.rs +++ b/litellm-rust/crates/cache-s3/src/cache.rs @@ -4,11 +4,7 @@ use std::{ time::{Duration, SystemTime}, }; -use aws_sdk_s3::{ - config::{BehaviorVersion, Region, RequestChecksumCalculation, ResponseChecksumValidation}, - error::SdkError, - primitives::ByteStream, -}; +use aws_sdk_s3::{error::SdkError, primitives::ByteStream}; use aws_smithy_types::{DateTime, date_time::Format}; use futures_util::future::try_join_all; use litellm_auth_aws::AwsAuthConfig; @@ -17,7 +13,9 @@ use litellm_cache::{ }; use tokio::runtime::Handle; -use crate::{auth::S3Credentials, transport::ReqwestHttpClient}; +use litellm_s3::S3ClientConfig; + +use crate::auth::S3Credentials; pub struct S3Endpoint { pub url: String, @@ -49,19 +47,16 @@ impl S3Cache { runtime: Handle, ) -> Self { let endpoint_url: Option = config.endpoint.map(|endpoint| endpoint.url); - let base = aws_sdk_s3::Config::builder() - .behavior_version(BehaviorVersion::latest()) - .region(Region::new(config.region.clone())) - .http_client(ReqwestHttpClient(http)) - .credentials_provider(S3Credentials::new(config.auth)) - .request_checksum_calculation(RequestChecksumCalculation::WhenRequired) - .response_checksum_validation(ResponseChecksumValidation::WhenRequired); - let builder = match &endpoint_url { - Some(url) => base.endpoint_url(url).force_path_style(true), - None => base, - }; + let client = litellm_s3::client( + S3ClientConfig { + region: config.region.clone(), + endpoint: endpoint_url.clone(), + }, + http, + S3Credentials::new(config.auth), + ); Self { - client: aws_sdk_s3::Client::from_conf(builder.build()), + client, codec, runtime, bucket: config.bucket.into(), diff --git a/litellm-rust/crates/cache-s3/src/lib.rs b/litellm-rust/crates/cache-s3/src/lib.rs index 9be0210aad3..9e722913148 100644 --- a/litellm-rust/crates/cache-s3/src/lib.rs +++ b/litellm-rust/crates/cache-s3/src/lib.rs @@ -1,6 +1,5 @@ mod auth; mod cache; -mod transport; pub use auth::S3Credentials; pub use cache::{S3Cache, S3CacheConfig, S3Endpoint}; diff --git a/litellm-rust/crates/model-catalog-refresh/Cargo.toml b/litellm-rust/crates/model-catalog-refresh/Cargo.toml index 16dbe02c95d..a98c5aa9989 100644 --- a/litellm-rust/crates/model-catalog-refresh/Cargo.toml +++ b/litellm-rust/crates/model-catalog-refresh/Cargo.toml @@ -6,5 +6,23 @@ license.workspace = true repository.workspace = true [dependencies] +aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"] } +aws-sdk-s3 = { version = "1.146.1", default-features = false, features = ["rustls", "rt-tokio"] } +aws-smithy-runtime-api = { version = "1.16.2", features = ["client", "http-1x"] } litellm-model-catalog.workspace = true +litellm-s3.workspace = true +rand.workspace = true +litellm-http.workspace = true +reqwest.workspace = true +sha1.workspace = true +thiserror.workspace = true +tokio = { workspace = true, features = ["fs", "time"] } url.workspace = true + +[dev-dependencies] +litellm-http = { workspace = true, features = ["test-support"] } +aws-credential-types = "1.3.0" +rstest.workspace = true +serde_json.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +wiremock.workspace = true diff --git a/litellm-rust/crates/model-catalog-refresh/src/error.rs b/litellm-rust/crates/model-catalog-refresh/src/error.rs new file mode 100644 index 00000000000..5dd63e2552c --- /dev/null +++ b/litellm-rust/crates/model-catalog-refresh/src/error.rs @@ -0,0 +1,9 @@ +use thiserror::Error; + +#[derive(Clone, Debug, PartialEq, Eq, Error)] +pub enum SourceError { + #[error("unsupported model cost map URL scheme {scheme}")] + UnsupportedScheme { scheme: String }, + #[error("invalid model cost map URL: {reason}")] + Invalid { reason: String }, +} diff --git a/litellm-rust/crates/model-catalog-refresh/src/file.rs b/litellm-rust/crates/model-catalog-refresh/src/file.rs new file mode 100644 index 00000000000..91f392f2bc8 --- /dev/null +++ b/litellm-rust/crates/model-catalog-refresh/src/file.rs @@ -0,0 +1,25 @@ +use std::io::ErrorKind; +use std::path::Path; + +use crate::step::Reply; + +/// A file has no validator, so every read is a fresh 200 +pub(crate) async fn fetch(path: &Path) -> Reply { + match tokio::fs::read(path).await { + Ok(body) => Reply::Response { + status: 200, + etag: None, + retry_after: None, + body: Some(body), + }, + Err(error) if error.kind() == ErrorKind::NotFound => Reply::Response { + status: 404, + etag: None, + retry_after: None, + body: None, + }, + Err(error) => Reply::Transport { + reason: format!("reading {} failed: {error}", path.display()), + }, + } +} diff --git a/litellm-rust/crates/model-catalog-refresh/src/http.rs b/litellm-rust/crates/model-catalog-refresh/src/http.rs new file mode 100644 index 00000000000..d197a54a94e --- /dev/null +++ b/litellm-rust/crates/model-catalog-refresh/src/http.rs @@ -0,0 +1,82 @@ +use std::time::Duration; + +use reqwest::header::{ETAG, IF_NONE_MATCH, RETRY_AFTER}; +use url::Url; + +use crate::step::{FetchRequest, Reply}; + +pub(crate) async fn fetch( + client: &litellm_http::Client, + url: &Url, + request: FetchRequest, +) -> Reply { + let builder = client.get(url.clone()).timeout(request.timeout); + let builder = match &request.if_none_match { + Some(etag) => builder.header(IF_NONE_MATCH, etag), + None => builder, + }; + let response = match builder.send().await { + Ok(response) => response, + Err(error) => { + return Reply::Transport { + reason: format!("fetching {url} failed: {error}"), + }; + } + }; + let status = response.status().as_u16(); + let etags: Vec<&str> = response + .headers() + .get_all(ETAG) + .iter() + .filter_map(|value| value.to_str().ok()) + .collect(); + let etag = (!etags.is_empty()).then(|| etags.join(", ")); + let retry_after = response + .headers() + .get(RETRY_AFTER) + .and_then(|value| value.to_str().ok()) + .and_then(retry_after_seconds); + if status != 200 { + return Reply::Response { + status, + etag, + retry_after, + body: None, + }; + } + match response.bytes().await { + Ok(body) => Reply::Response { + status, + etag, + retry_after, + body: Some(body.to_vec()), + }, + Err(error) => Reply::Transport { + reason: format!("reading {url} failed: {error}"), + }, + } +} + +/// Main reads `Retry-After` as a number of seconds and ignores an HTTP date or a negative number +fn retry_after_seconds(header: &str) -> Option { + let seconds = header.trim().parse::().ok()?; + (seconds.is_finite() && seconds >= 0.0) + .then(|| Duration::try_from_secs_f64(seconds).ok()) + .flatten() +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::seconds("7", Some(Duration::from_secs(7)))] + #[case::fraction("0.5", Some(Duration::from_millis(500)))] + #[case::negative("-1", None)] + #[case::http_date("Wed, 21 Oct 2015 07:28:00 GMT", None)] + fn retry_after_is_seconds(#[case] header: &str, #[case] expected: Option) { + assert_eq!(retry_after_seconds(header), expected); + } +} diff --git a/litellm-rust/crates/model-catalog-refresh/src/lib.rs b/litellm-rust/crates/model-catalog-refresh/src/lib.rs index 470a04f2d9d..6cfc2340149 100644 --- a/litellm-rust/crates/model-catalog-refresh/src/lib.rs +++ b/litellm-rust/crates/model-catalog-refresh/src/lib.rs @@ -1,7 +1,67 @@ +mod error; +mod file; +mod http; mod policy; +mod revalidate; +mod s3; mod source; mod step; +use std::future::Future; +use std::time::Duration; + +pub use error::*; pub use policy::*; +pub use revalidate::*; +pub use s3::*; pub use source::*; pub use step::*; + +/// The transports a fetch can use. The HTTP client comes from the caller's pool, the S3 client is built per source +pub struct Transports<'a> { + pub http: &'a litellm_http::Client, + pub s3: Option<&'a S3Source>, +} + +/// One attempt at `source`, never retried here +pub async fn fetch(source: &Source, request: FetchRequest, transports: &Transports<'_>) -> Reply { + match source { + Source::Http(url) => http::fetch(transports.http, url, request).await, + Source::File(path) => file::fetch(path).await, + Source::S3(object) => match transports.s3 { + Some(s3) => s3.fetch(request).await, + None => { + S3Source::from_environment(object.clone(), transports.http.clone()) + .await + .fetch(request) + .await + } + }, + } +} + +/// Drives `Revalidation` to `Adopt` or `Done`, sleeping between attempts with the injected `sleep` +pub async fn fetch_with_retry( + source: &Source, + installed: &Installed, + policy: RetryPolicy, + seed: Option, + transports: &Transports<'_>, + sleep: S, +) -> Step +where + S: Fn(Duration) -> F, + F: Future, +{ + let (mut revalidation, mut step) = Revalidation::new(source, installed, policy, seed); + loop { + step = match step { + Step::Fetch(request) => revalidation.on_reply(fetch(source, request, transports).await), + Step::Sleep(delay) => { + sleep(delay).await; + revalidation.on_slept() + } + done => return done, + }; + } +} diff --git a/litellm-rust/crates/model-catalog-refresh/src/policy.rs b/litellm-rust/crates/model-catalog-refresh/src/policy.rs index bc32e40408e..9145f4c27e4 100644 --- a/litellm-rust/crates/model-catalog-refresh/src/policy.rs +++ b/litellm-rust/crates/model-catalog-refresh/src/policy.rs @@ -18,3 +18,17 @@ pub struct RetryPolicy { /// Main's default `LITELLM_MODEL_COST_MAP_URL` pub const DEFAULT_REGISTRY_URL: &str = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"; + +impl Default for RetryPolicy { + fn default() -> Self { + Self { + max_attempts: 3, + max_wait: Duration::from_secs(30), + timeout: Duration::from_secs(5), + backoff_base_seconds: 2.0, + jitter_min_seconds: 0.0, + jitter_max_seconds: 1.0, + limits: IntegrityLimits::new(50, 0.5).expect("0.5 is a ratio"), + } + } +} diff --git a/litellm-rust/crates/model-catalog-refresh/src/revalidate.rs b/litellm-rust/crates/model-catalog-refresh/src/revalidate.rs new file mode 100644 index 00000000000..ab719fda00c --- /dev/null +++ b/litellm-rust/crates/model-catalog-refresh/src/revalidate.rs @@ -0,0 +1,410 @@ +use std::sync::LazyLock; +use std::time::Duration; + +use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog}; +use rand::rngs::StdRng; +use rand::{Rng, SeedableRng}; +use sha1::{Digest, Sha1}; + +use crate::policy::RetryPolicy; +use crate::source::Source; +use crate::step::{FetchRequest, Installed, Origin, Reply, Step}; + +/// Main's `RETRYABLE_FETCH_STATUS_CODES` +const RETRYABLE_STATUSES: [u16; 5] = [429, 500, 502, 503, 504]; + +static BUNDLED_MODEL_COUNT: LazyLock = LazyLock::new(|| { + Catalog::parse(BUNDLED_REGISTRY) + .expect("the bundled model registry parses") + .value + .model_count() +}); + +/// The fetch state machine. It asks its driver for an attempt or a sleep and never does IO itself +#[derive(Debug)] +pub struct Revalidation { + source: Source, + if_none_match: Option, + policy: RetryPolicy, + attempts: u32, + rng: StdRng, + state: State, +} + +#[derive(Debug)] +enum State { + AwaitingReply, + AwaitingSleep, + Done(String, Option), +} + +impl Revalidation { + pub fn new( + source: &Source, + installed: &Installed, + policy: RetryPolicy, + seed: Option, + ) -> (Self, Step) { + let if_none_match = match &installed.origin { + Origin::Remote { source: from } if from == source => installed.etag.clone(), + Origin::Remote { .. } | Origin::Bundled => None, + }; + let revalidation = Self { + source: source.clone(), + if_none_match, + policy, + attempts: 1, + rng: seed.map_or_else(StdRng::from_entropy, StdRng::seed_from_u64), + state: State::AwaitingReply, + }; + let step = revalidation.fetch(); + (revalidation, step) + } + + pub fn on_reply(&mut self, reply: Reply) -> Step { + if !matches!(self.state, State::AwaitingReply) { + return self.invalid_state("revalidation is not awaiting a reply"); + } + match reply { + Reply::Transport { reason } => self.retry_or_done(reason, None), + Reply::Response { + status: 200, + etag, + body, + .. + } => self.adopt(body.unwrap_or_default(), etag), + Reply::Response { status: 304, .. } => { + self.finish(format!("{} is not modified", self.source)) + } + Reply::Response { + status, + retry_after, + .. + } if RETRYABLE_STATUSES.contains(&status) => { + self.retry_or_done(format!("HTTP {status} from {}", self.source), retry_after) + } + Reply::Response { status, .. } => { + self.finish(format!("HTTP {status} from {}", self.source)) + } + } + } + + pub fn on_slept(&mut self) -> Step { + if !matches!(self.state, State::AwaitingSleep) { + return self.invalid_state("revalidation is not awaiting a sleep"); + } + self.attempts += 1; + self.state = State::AwaitingReply; + self.fetch() + } + + fn fetch(&self) -> Step { + Step::Fetch(FetchRequest { + if_none_match: self.if_none_match.clone(), + timeout: self.policy.timeout, + }) + } + + fn adopt(&mut self, body: Vec, etag: Option) -> Step { + let loaded = match Catalog::parse(&body) { + Ok(loaded) => loaded, + Err(error) => { + return self.finish(format!( + "model cost map from {} is unusable: {error}", + self.source + )); + } + }; + if let Err(error) = loaded + .value + .check_size(*BUNDLED_MODEL_COUNT, self.policy.limits) + { + return self.finish(format!( + "model cost map from {} is unusable: {error}", + self.source + )); + } + let installed = Installed { + origin: Origin::Remote { + source: self.source.clone(), + }, + etag, + revision: git_blob_id(&body), + model_count: loaded.value.model_count(), + }; + self.state = State::Done("adopted".to_owned(), None); + Step::Adopt { + catalog: Box::new(loaded.value), + installed, + errors: loaded.errors, + } + } + + fn retry_or_done(&mut self, reason: String, retry_after: Option) -> Step { + let delay = retry_after + .unwrap_or_else(|| self.backoff()) + .min(self.policy.max_wait); + if self.attempts >= self.policy.max_attempts { + let reason = if self.policy.max_attempts > 1 { + format!("{reason} (after {} attempts)", self.policy.max_attempts) + } else { + reason + }; + self.state = State::Done(reason.clone(), Some(delay)); + return Step::Done { + reason, + retry_after: Some(delay), + }; + } + self.state = State::AwaitingSleep; + Step::Sleep(delay) + } + + fn backoff(&mut self) -> Duration { + let jitter = if self.policy.jitter_max_seconds > self.policy.jitter_min_seconds { + self.rng + .gen_range(self.policy.jitter_min_seconds..=self.policy.jitter_max_seconds) + } else { + self.policy.jitter_min_seconds + }; + let seconds = self.policy.backoff_base_seconds.powi(self.attempts as i32) + jitter; + Duration::try_from_secs_f64(seconds.min(self.policy.max_wait.as_secs_f64())) + .unwrap_or(self.policy.max_wait) + } + + fn finish(&mut self, reason: String) -> Step { + self.state = State::Done(reason.clone(), None); + Step::Done { + reason, + retry_after: None, + } + } + + fn invalid_state(&mut self, reason: &str) -> Step { + if let State::Done(previous, retry_after) = &self.state { + return Step::Done { + reason: previous.clone(), + retry_after: *retry_after, + }; + } + self.finish(reason.to_owned()) + } +} + +/// The digest `git hash-object` prints, which main records as the source revision +pub fn git_blob_id(body: &[u8]) -> String { + let mut hasher = Sha1::new(); + hasher.update(format!("blob {}\0", body.len()).as_bytes()); + hasher.update(body); + hasher + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +#[cfg(test)] +mod tests { + use rstest::{fixture, rstest}; + + use super::*; + + fn source() -> Source { + Source::parse("https://example.com/map.json").unwrap() + } + + #[fixture] + fn bundled() -> Installed { + Installed { + origin: Origin::Bundled, + etag: None, + revision: git_blob_id(BUNDLED_REGISTRY), + model_count: *BUNDLED_MODEL_COUNT, + } + } + + fn response(status: u16) -> Reply { + Reply::Response { + status, + etag: None, + retry_after: None, + body: None, + } + } + + fn done(step: Step) -> String { + match step { + Step::Done { reason, .. } => reason, + other => panic!("expected done, got {other:?}"), + } + } + + fn slept(step: Step) -> Duration { + match step { + Step::Sleep(wait) => wait, + other => panic!("expected a sleep, got {other:?}"), + } + } + + fn started(installed: &Installed) -> Revalidation { + Revalidation::new(&source(), installed, RetryPolicy::default(), Some(0)).0 + } + + #[rstest] + #[case::too_many_requests(429)] + #[case::internal_error(500)] + #[case::bad_gateway(502)] + #[case::unavailable(503)] + #[case::gateway_timeout(504)] + fn mains_retryable_statuses_sleep_before_the_next_attempt( + bundled: Installed, + #[case] status: u16, + ) { + let mut revalidation = started(&bundled); + assert!(matches!( + revalidation.on_reply(response(status)), + Step::Sleep(_) + )); + } + + #[rstest] + #[case::not_implemented(501)] + #[case::version_not_supported(505)] + #[case::not_found(404)] + fn other_statuses_are_final(bundled: Installed, #[case] status: u16) { + let mut revalidation = started(&bundled); + assert_eq!( + done(revalidation.on_reply(response(status))), + format!("HTTP {status} from https://example.com/map.json") + ); + } + + #[rstest] + #[case::honored(Duration::from_secs(7), Duration::from_secs(7))] + #[case::capped(Duration::from_secs(120), Duration::from_secs(30))] + fn retry_after_sets_the_wait_up_to_the_cap( + bundled: Installed, + #[case] header: Duration, + #[case] wait: Duration, + ) { + let mut revalidation = started(&bundled); + let reply = Reply::Response { + status: 429, + etag: None, + retry_after: Some(header), + body: None, + }; + assert_eq!(slept(revalidation.on_reply(reply)), wait); + } + + #[rstest] + fn backoff_grows_from_two_seconds_with_jitter(bundled: Installed) { + let mut revalidation = started(&bundled); + let first = slept(revalidation.on_reply(response(503))); + revalidation.on_slept(); + let second = slept(revalidation.on_reply(response(503))); + assert!((2.0..=3.0).contains(&first.as_secs_f64()), "{first:?}"); + assert!((4.0..=5.0).contains(&second.as_secs_f64()), "{second:?}"); + } + + #[rstest] + fn the_last_attempt_names_how_many_were_made(bundled: Installed) { + let mut revalidation = started(&bundled); + revalidation.on_reply(response(503)); + revalidation.on_slept(); + revalidation.on_reply(response(503)); + revalidation.on_slept(); + assert_eq!( + done(revalidation.on_reply(Reply::Transport { + reason: "connection refused".into() + })), + "connection refused (after 3 attempts)" + ); + } + + #[rstest] + fn a_single_attempt_reports_the_wait_a_retry_should_use(bundled: Installed) { + let policy = RetryPolicy { + max_attempts: 1, + ..RetryPolicy::default() + }; + let (mut revalidation, _) = Revalidation::new(&source(), &bundled, policy, Some(0)); + let reply = Reply::Response { + status: 503, + etag: None, + retry_after: Some(Duration::from_secs(4)), + body: None, + }; + let Step::Done { + reason, + retry_after, + } = revalidation.on_reply(reply) + else { + panic!("expected done"); + }; + assert_eq!(reason, "HTTP 503 from https://example.com/map.json"); + assert_eq!(retry_after, Some(Duration::from_secs(4))); + } + + #[rstest] + fn a_body_that_is_not_a_catalog_is_final(bundled: Installed) { + let mut revalidation = started(&bundled); + let reply = Reply::Response { + status: 200, + etag: None, + retry_after: None, + body: Some(b"not json".to_vec()), + }; + let reason = done(revalidation.on_reply(reply)); + assert!(reason.contains("invalid JSON"), "{reason}"); + } + + #[rstest] + fn the_bundled_body_is_adopted_with_its_revision_and_etag(bundled: Installed) { + let mut revalidation = started(&bundled); + let reply = Reply::Response { + status: 200, + etag: Some("W/\"abc\"".into()), + retry_after: None, + body: Some(BUNDLED_REGISTRY.to_vec()), + }; + let Step::Adopt { installed, .. } = revalidation.on_reply(reply) else { + panic!("expected adopt"); + }; + assert_eq!(installed.revision, bundled.revision); + assert_eq!(installed.etag.as_deref(), Some("W/\"abc\"")); + assert_eq!(installed.origin, Origin::Remote { source: source() }); + } + + #[rstest] + #[case::same_source(Origin::Remote { source: source() }, Some("v1"))] + #[case::another_source( + Origin::Remote { source: Source::parse("https://example.com/other.json").unwrap() }, + None + )] + #[case::bundled(Origin::Bundled, None)] + fn the_etag_is_sent_only_back_to_the_source_that_gave_it( + bundled: Installed, + #[case] origin: Origin, + #[case] expected: Option<&str>, + ) { + let installed = Installed { + origin, + etag: Some("v1".into()), + ..bundled + }; + let (_, step) = Revalidation::new(&source(), &installed, RetryPolicy::default(), Some(0)); + let Step::Fetch(request) = step else { + panic!("expected a fetch"); + }; + assert_eq!(request.if_none_match.as_deref(), expected); + } + + #[rstest] + fn git_blob_id_is_what_git_hash_object_prints() { + assert_eq!( + git_blob_id(b"{\"gpt-5.4-mini\": {\"mode\": \"chat\"}}\n"), + "18b9a8381e13a3b38a2128f184f631f95829e987" + ); + } +} diff --git a/litellm-rust/crates/model-catalog-refresh/src/s3.rs b/litellm-rust/crates/model-catalog-refresh/src/s3.rs new file mode 100644 index 00000000000..aa092a825bb --- /dev/null +++ b/litellm-rust/crates/model-catalog-refresh/src/s3.rs @@ -0,0 +1,171 @@ +use aws_config::BehaviorVersion; +use aws_sdk_s3::config::SharedCredentialsProvider; +use aws_sdk_s3::config::retry::RetryConfig; +use aws_sdk_s3::error::SdkError; +use aws_smithy_runtime_api::http::Response; +use litellm_s3::{ReqwestHttpClient, S3ClientConfig}; + +use crate::source::S3Object; +use crate::step::{FetchRequest, Reply}; + +const BUCKET_REGION_HEADER: &str = "x-amz-bucket-region"; +const DEFAULT_REGION: &str = "us-east-1"; + +/// One S3 object as a catalog source. SDK retries are off, so `RetryPolicy` owns every attempt +#[derive(Clone, Debug)] +pub struct S3Source { + object: S3Object, + http: litellm_http::Client, + credentials: SharedCredentialsProvider, + endpoint: Option, + client: aws_sdk_s3::Client, +} + +impl S3Source { + pub fn new( + object: S3Object, + http: litellm_http::Client, + credentials: SharedCredentialsProvider, + config: S3ClientConfig, + ) -> Self { + let endpoint = config.endpoint.clone(); + let client = without_retries(litellm_s3::client( + config, + http.clone(), + credentials.clone(), + )); + Self { + object, + http, + credentials, + endpoint, + client, + } + } + + /// The chain `boto3.client("s3")` used on main. Region from `AWS_REGION_NAME` first, then the SDK chain + /// (`AWS_REGION`, `AWS_DEFAULT_REGION`, the profile). `AWS_ENDPOINT_URL_S3` or `AWS_ENDPOINT_URL` for + /// S3-compatible stores + pub async fn from_environment(object: S3Object, http: litellm_http::Client) -> Self { + let shared = aws_config::defaults(BehaviorVersion::latest()) + .http_client(ReqwestHttpClient(http.clone())) + .load() + .await; + let region = non_empty_env("AWS_REGION_NAME") + .or_else(|| shared.region().map(ToString::to_string)) + .unwrap_or_else(|| DEFAULT_REGION.to_owned()); + let endpoint = + non_empty_env("AWS_ENDPOINT_URL_S3").or_else(|| non_empty_env("AWS_ENDPOINT_URL")); + let credentials = shared.credentials_provider().unwrap_or_else(|| { + SharedCredentialsProvider::new(aws_sdk_s3::config::Credentials::new( + "", + "", + None, + None, + "no AWS credentials found", + )) + }); + Self::new( + object, + http, + credentials, + S3ClientConfig { region, endpoint }, + ) + } + + pub async fn fetch(&self, request: FetchRequest) -> Reply { + match self.attempt(&self.client, &request).await { + Redirect::To(region) => { + let client = without_retries(litellm_s3::client( + S3ClientConfig { + region, + endpoint: self.endpoint.clone(), + }, + self.http.clone(), + self.credentials.clone(), + )); + match self.attempt(&client, &request).await { + Redirect::To(region) => Reply::Transport { + reason: format!("{} redirected twice, last to {region}", self.display()), + }, + Redirect::No(reply) => reply, + } + } + Redirect::No(reply) => reply, + } + } + + async fn attempt(&self, client: &aws_sdk_s3::Client, request: &FetchRequest) -> Redirect { + let call = client + .get_object() + .bucket(&self.object.bucket) + .key(&self.object.key) + .set_if_none_match(request.if_none_match.clone()) + .send(); + let output = match tokio::time::timeout(request.timeout, call).await { + Err(_) => { + return Redirect::No(Reply::Transport { + reason: format!("fetching {} timed out", self.display()), + }); + } + Ok(Err(SdkError::ServiceError(error))) => return rejected(error.raw()), + Ok(Err(SdkError::ResponseError(error))) => return rejected(error.raw()), + Ok(Err(error)) => { + return Redirect::No(Reply::Transport { + reason: format!("fetching {} failed: {error}", self.display()), + }); + } + Ok(Ok(output)) => output, + }; + let etag = output.e_tag().map(str::to_owned); + match output.body.collect().await { + Ok(body) => Redirect::No(Reply::Response { + status: 200, + etag, + retry_after: None, + body: Some(body.into_bytes().to_vec()), + }), + Err(error) => Redirect::No(Reply::Transport { + reason: format!("reading {} failed: {error}", self.display()), + }), + } + } + + fn display(&self) -> String { + format!("s3://{}/{}", self.object.bucket, self.object.key) + } +} + +enum Redirect { + /// The bucket lives in another region, repeated once as boto3 does + To(String), + No(Reply), +} + +fn rejected(raw: &Response) -> Redirect { + let status = raw.status().as_u16(); + let header = |name: &str| raw.headers().get(name).map(str::to_owned); + match (status, header(BUCKET_REGION_HEADER)) { + (301, Some(region)) => Redirect::To(region), + _ => Redirect::No(Reply::Response { + status, + etag: header("etag"), + retry_after: None, + body: None, + }), + } +} + +fn without_retries(client: aws_sdk_s3::Client) -> aws_sdk_s3::Client { + aws_sdk_s3::Client::from_conf( + client + .config() + .to_builder() + .retry_config(RetryConfig::disabled()) + .build(), + ) +} + +fn non_empty_env(name: &str) -> Option { + std::env::var(name).ok().filter(|value| !value.is_empty()) +} diff --git a/litellm-rust/crates/model-catalog-refresh/src/source.rs b/litellm-rust/crates/model-catalog-refresh/src/source.rs index 5cd82130d33..6220ad2dafa 100644 --- a/litellm-rust/crates/model-catalog-refresh/src/source.rs +++ b/litellm-rust/crates/model-catalog-refresh/src/source.rs @@ -1,7 +1,10 @@ +use std::fmt; use std::path::PathBuf; use url::Url; +use crate::error::SourceError; + /// Where a fetched catalog comes from #[derive(Clone, Debug, PartialEq, Eq)] pub enum Source { @@ -16,8 +19,84 @@ pub struct S3Object { pub key: String, } -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum SourceError { - UnsupportedScheme { scheme: String }, - Invalid { reason: String }, +impl Source { + pub fn parse(text: &str) -> Result { + let url = Url::parse(text).map_err(|error| SourceError::Invalid { + reason: format!("invalid URL {text}: {error}"), + })?; + match url.scheme() { + "http" | "https" => Ok(Self::Http(url)), + "file" => url + .to_file_path() + .map(Self::File) + .map_err(|()| SourceError::Invalid { + reason: format!("invalid file URL {text}"), + }), + "s3" => { + let bucket = url.host_str().unwrap_or_default(); + let key = url.path().trim_start_matches('/'); + if bucket.is_empty() || key.is_empty() { + return Err(SourceError::Invalid { + reason: format!("invalid S3 URL {text}: expected s3://bucket/key"), + }); + } + Ok(Self::S3(S3Object { + bucket: bucket.to_owned(), + key: key.to_owned(), + })) + } + scheme => Err(SourceError::UnsupportedScheme { + scheme: scheme.to_owned(), + }), + } + } +} + +impl fmt::Display for Source { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Http(url) => write!(formatter, "{url}"), + Self::File(path) => write!(formatter, "file://{}", path.display()), + Self::S3(object) => write!(formatter, "s3://{}/{}", object.bucket, object.key), + } + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::https("https://example.com/map.json", Source::Http(Url::parse("https://example.com/map.json").unwrap()))] + #[case::file("file:///tmp/map.json", Source::File(PathBuf::from("/tmp/map.json")))] + #[case::s3( + "s3://bucket/prices/map.json", + Source::S3(S3Object { bucket: "bucket".into(), key: "prices/map.json".into() }) + )] + fn parses_each_supported_scheme(#[case] text: &str, #[case] expected: Source) { + assert_eq!(Source::parse(text), Ok(expected)); + } + + #[rstest] + #[case::no_key("s3://bucket")] + #[case::no_bucket("s3:///key")] + #[case::not_a_url("http://[bad/model_prices.json")] + fn rejects_an_unusable_url(#[case] text: &str) { + assert!(matches!( + Source::parse(text), + Err(SourceError::Invalid { .. }) + )); + } + + #[rstest] + fn rejects_an_unknown_scheme() { + assert_eq!( + Source::parse("ftp://example.com/map.json"), + Err(SourceError::UnsupportedScheme { + scheme: "ftp".into() + }) + ); + } } diff --git a/litellm-rust/crates/model-catalog-refresh/src/step.rs b/litellm-rust/crates/model-catalog-refresh/src/step.rs index 2b174c35e2c..6d9dc21c6bb 100644 --- a/litellm-rust/crates/model-catalog-refresh/src/step.rs +++ b/litellm-rust/crates/model-catalog-refresh/src/step.rs @@ -38,6 +38,8 @@ pub enum Step { /// 304, a final status, or retries exhausted Done { reason: String, + /// The wait before the next attempt when the last failure was retryable, so a caller can retry later + retry_after: Option, }, } diff --git a/litellm-rust/crates/model-catalog-refresh/tests/sources.rs b/litellm-rust/crates/model-catalog-refresh/tests/sources.rs new file mode 100644 index 00000000000..3ffca3c79a0 --- /dev/null +++ b/litellm-rust/crates/model-catalog-refresh/tests/sources.rs @@ -0,0 +1,197 @@ +use std::time::Duration; + +use aws_credential_types::Credentials; +use aws_sdk_s3::config::SharedCredentialsProvider; +use litellm_model_catalog::BUNDLED_REGISTRY; +use litellm_model_catalog_refresh::{ + Installed, Origin, RetryPolicy, S3Object, S3Source, Source, Step, Transports, fetch_with_retry, + git_blob_id, +}; +use litellm_s3::S3ClientConfig; +use rstest::rstest; +use wiremock::matchers::{header, method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +fn installed(origin: Origin, etag: Option<&str>) -> Installed { + Installed { + origin, + etag: etag.map(str::to_owned), + revision: git_blob_id(BUNDLED_REGISTRY), + model_count: 0, + } +} + +async fn no_sleep(_: Duration) {} + +fn s3_source(server: &MockServer, object: S3Object) -> S3Source { + S3Source::new( + object, + litellm_http::Client::plain_for_test(), + SharedCredentialsProvider::new(Credentials::new("key", "secret", None, None, "test")), + S3ClientConfig { + region: "us-east-1".into(), + endpoint: Some(server.uri()), + }, + ) +} + +#[rstest] +#[tokio::test] +async fn http_retries_a_503_then_adopts_the_map_with_its_etag() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/map.json")) + .respond_with(ResponseTemplate::new(503)) + .up_to_n_times(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/map.json")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("ETag", "W/\"v2\"") + .set_body_bytes(BUNDLED_REGISTRY), + ) + .mount(&server) + .await; + let source = Source::parse(&format!("{}/map.json", server.uri())).unwrap(); + let http = litellm_http::Client::plain_for_test(); + + let step = fetch_with_retry( + &source, + &installed(Origin::Bundled, None), + RetryPolicy::default(), + Some(0), + &Transports { + http: &http, + s3: None, + }, + no_sleep, + ) + .await; + + let Step::Adopt { installed, .. } = step else { + panic!("expected adopt, got {step:?}"); + }; + assert_eq!(installed.etag.as_deref(), Some("W/\"v2\"")); + assert_eq!(server.received_requests().await.unwrap().len(), 2); +} + +#[rstest] +#[tokio::test] +async fn s3_retries_slow_down_and_sends_if_none_match() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/bucket/prices/map.json")) + .and(header("if-none-match", "\"v1\"")) + .respond_with(ResponseTemplate::new(503).set_body_string( + "SlowDownPlease reduce your request rate.", + )) + .up_to_n_times(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/bucket/prices/map.json")) + .and(header("if-none-match", "\"v1\"")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("ETag", "\"v2\"") + .set_body_bytes(BUNDLED_REGISTRY), + ) + .mount(&server) + .await; + let object = S3Object { + bucket: "bucket".into(), + key: "prices/map.json".into(), + }; + let s3 = s3_source(&server, object.clone()); + let source = Source::S3(object); + let http = litellm_http::Client::plain_for_test(); + + let step = fetch_with_retry( + &source, + &installed( + Origin::Remote { + source: source.clone(), + }, + Some("\"v1\""), + ), + RetryPolicy::default(), + Some(0), + &Transports { + http: &http, + s3: Some(&s3), + }, + no_sleep, + ) + .await; + + let Step::Adopt { installed, .. } = step else { + panic!("expected adopt, got {step:?}"); + }; + assert_eq!(installed.etag.as_deref(), Some("\"v2\"")); + assert_eq!(server.received_requests().await.unwrap().len(), 2); +} + +#[rstest] +#[tokio::test] +async fn s3_not_found_is_final() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with( + ResponseTemplate::new(404).set_body_string("NoSuchKey"), + ) + .mount(&server) + .await; + let object = S3Object { + bucket: "bucket".into(), + key: "missing.json".into(), + }; + let s3 = s3_source(&server, object.clone()); + let http = litellm_http::Client::plain_for_test(); + + let step = fetch_with_retry( + &Source::S3(object), + &installed(Origin::Bundled, None), + RetryPolicy::default(), + Some(0), + &Transports { + http: &http, + s3: Some(&s3), + }, + no_sleep, + ) + .await; + + let Step::Done { reason, .. } = step else { + panic!("expected done, got {step:?}"); + }; + assert_eq!(reason, "HTTP 404 from s3://bucket/missing.json"); + assert_eq!(server.received_requests().await.unwrap().len(), 1); +} + +#[rstest] +#[tokio::test] +async fn a_file_source_is_read_without_a_transport() { + let directory = std::env::temp_dir().join(format!("catalog-refresh-{}", std::process::id())); + std::fs::create_dir_all(&directory).unwrap(); + let file = directory.join("map.json"); + std::fs::write(&file, BUNDLED_REGISTRY).unwrap(); + let http = litellm_http::Client::plain_for_test(); + + let step = fetch_with_retry( + &Source::File(file.clone()), + &installed(Origin::Bundled, None), + RetryPolicy::default(), + Some(0), + &Transports { + http: &http, + s3: None, + }, + no_sleep, + ) + .await; + + std::fs::remove_dir_all(&directory).unwrap(); + assert!(matches!(step, Step::Adopt { .. }), "{step:?}"); +} diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 40301a71431..832dfeeac58 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -49,8 +49,8 @@ litellm-llms-types.workspace = true litellm-router-types.workspace = true litellm-host-python.workspace = true litellm-model-catalog.workspace = true +litellm-model-catalog-refresh.workspace = true indexmap = "2.14.0" -sha1.workspace = true litellm-token-counter = { path = "../token-counter", default-features = false } pyo3.workspace = true prost.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 462ded582c6..702e4271a2f 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -34,7 +34,9 @@ mod _native { #[pymodule_export] use crate::logger::NativeDiagnosticProcessor; #[pymodule_export] - use crate::model_registry::{CatalogHandle, Fetched, Installed, RowError, Unavailable}; + use crate::model_registry::{ + Bundled, CatalogHandle, Fetched, File, Installed, Provenance, Remote, RowError, Unavailable, + }; #[pymodule_export] use crate::routes::audio_transcription::{atranscription, transcription}; #[pymodule_export] @@ -157,6 +159,10 @@ mod tests { "Fetched", "Installed", "Unavailable", + "Remote", + "File", + "Bundled", + "Provenance", "RowError", "RowErrorKind", "NativeClickHouseSpendConfig", diff --git a/litellm-rust/crates/python-bridge/src/model_registry.rs b/litellm-rust/crates/python-bridge/src/model_registry.rs index 6dead98763e..74c045fc16f 100644 --- a/litellm-rust/crates/python-bridge/src/model_registry.rs +++ b/litellm-rust/crates/python-bridge/src/model_registry.rs @@ -1,19 +1,21 @@ +mod fetch; mod install; mod row_error; use std::collections::HashMap; use std::sync::{Arc, LazyLock, Mutex, PoisonError}; +use std::time::SystemTime; use litellm_host_python::{release_gil, to_py}; use litellm_model_catalog::wire::Fields; -use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog, IntegrityLimits, Layers}; +use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog, Layers, Located}; +use litellm_model_catalog_refresh::{Source, git_blob_id}; use pyo3::exceptions::PyTypeError; use pyo3::prelude::*; -use pyo3::pybacked::PyBackedBytes; use pyo3::types::{PyDict, PyModule}; use serde::Serialize; -use sha1::{Digest, Sha1}; +pub(crate) use fetch::{Bundled, File, Provenance, Remote}; use install::{Record, write_into}; pub(crate) use row_error::{RowError, RowErrorKind, row_error}; @@ -30,7 +32,7 @@ static BUNDLED: LazyLock> = LazyLock::new(|| { static BUNDLED_REVISION: LazyLock = LazyLock::new(|| git_blob_id(BUNDLED_REGISTRY)); -struct Parsed { +pub(crate) struct Parsed { /// What install layers over bundled, rows decision 15 rejects replaced or left out catalog: Arc, /// The document as written @@ -46,6 +48,10 @@ pub(crate) struct Fetched { revision: String, #[pyo3(get)] etag: Option, + /// `None` for the bundled map + source: Option, + /// When the load that produced this map started, main's `loaded_at` + started_at: SystemTime, } #[pyclass(frozen, module = "litellm.rust_bridge._native")] @@ -68,11 +74,15 @@ pub(crate) struct Installed { pub(crate) struct Unavailable { #[pyo3(get)] reason: String, + /// Seconds to wait before retrying, set when the last reply was retryable + #[pyo3(get)] + retry_after: Option, } struct State { layers: Layers, record: Option, + provenance: Option, } #[pyclass(frozen, name = "Catalog", module = "litellm.rust_bridge._native")] @@ -117,6 +127,18 @@ impl Fetched { } } +#[pymethods] +impl Unavailable { + #[new] + #[pyo3(signature = (reason, retry_after = None))] + fn new(reason: String, retry_after: Option) -> Self { + Self { + reason, + retry_after, + } + } +} + #[pymethods] impl CatalogHandle { #[staticmethod] @@ -127,49 +149,58 @@ impl CatalogHandle { state: Mutex::new(State { layers: Layers::new(Arc::clone(&BUNDLED.catalog)), record: None, + provenance: None, }), } } fn bundled(&self) -> Fetched { - Fetched { - parsed: Arc::clone(&BUNDLED), - revision: BUNDLED_REVISION.clone(), - etag: None, - } + bundled() } - /// Parses a fetched body and applies main's whole-map checks against the bundled model count - #[staticmethod] - #[pyo3(signature = (body, *, etag, min_model_count, max_shrink_ratio))] - fn parse( + /// Fetches and validates `source` with no install, retrying up to `attempts` times + #[pyo3(signature = (source, *, timeout, attempts))] + fn fetch( + &self, py: Python<'_>, - body: PyBackedBytes, - etag: Option, - min_model_count: usize, - max_shrink_ratio: f64, + source: &Bound<'_, PyAny>, + timeout: f64, + attempts: u32, ) -> PyResult> { - let parsed = release_gil(py, || { - parse_remote(&body, min_model_count, max_shrink_ratio) - }); - match parsed { - Ok((parsed, revision)) => Ok(Fetched { - parsed: Arc::new(parsed), - revision, - etag, - } + fetch::fetch(py, fetch::requested(source)?, timeout, attempts) + } + + /// Fetch with main's reload policy, 3 attempts of 5 s, then install + fn reload(&self, py: Python<'_>, source: &Bound<'_, PyAny>) -> PyResult> { + let requested = fetch::requested(source)?; + let env_forced = matches!(requested, fetch::Requested::Bundled); + let fetched = fetch::fetch(py, requested, 5.0, 3)?; + let Ok(fetched) = fetched.bind(py).cast::() else { + return Ok(fetched); + }; + Ok(self + .install(py, fetched.get(), env_forced, None, None)? .into_pyobject(py)? .into_any() - .unbind()), - Err(reason) => Ok(Unavailable { reason } - .into_pyobject(py)? - .into_any() - .unbind()), - } + .unbind()) } - /// Writes the composed rows into the dict bound to the attribute, never over a field the user owns (decision 25) - fn install(&self, py: Python<'_>, fetched: &Fetched) -> PyResult { + /// Where the installed map came from, `None` before the first install + fn provenance(&self) -> Option { + self.lock().provenance.clone() + } + + /// Writes the composed rows into the dict bound to the attribute, never over a field the user owns (decision 25). + /// `env_forced`, `fallback_reason` and `url` are the provenance facts only the caller knows + #[pyo3(signature = (fetched, *, env_forced = false, fallback_reason = None, url = None))] + fn install( + &self, + py: Python<'_>, + fetched: &Fetched, + env_forced: bool, + fallback_reason: Option, + url: Option, + ) -> PyResult { let bound = self.module.bind(py).getattr(self.attr.as_str())?; let dict = bound .cast::() @@ -188,7 +219,12 @@ impl CatalogHandle { }; let composed = release_gil(py, || layers.compose()); let outcome = write_into(py, dict, &composed, &layers, record)?; - self.lock().record = Some(outcome.record); + let provenance = Provenance::of(fetched, env_forced, fallback_reason, url); + { + let mut state = self.lock(); + state.record = Some(outcome.record); + state.provenance = Some(provenance); + } Ok(Installed { count: composed.len(), written: outcome.written, @@ -204,47 +240,29 @@ impl CatalogHandle { } } -fn parse_remote( - body: &[u8], - min_model_count: usize, - max_shrink_ratio: f64, -) -> Result<(Parsed, String), String> { - let limits = IntegrityLimits::new(min_model_count, max_shrink_ratio) - .ok_or_else(|| format!("invalid shrink ratio {max_shrink_ratio}"))?; - let loaded = Catalog::parse(body).map_err(|error| error.to_string())?; - loaded - .value - .check_size(BUNDLED.raw.model_count(), limits) - .map_err(|error| error.to_string())?; - let raw = Arc::new(loaded.value.clone()); - let errors = loaded.errors.iter().map(row_error).collect(); - let (catalog, rejections) = loaded - .value - .with_rejected_replaced(&loaded.errors, &BUNDLED.raw); - let rejected = rejections - .iter() - .flat_map(|rejection| rejection.errors.iter().copied().map(row_error)) - .collect(); - Ok(( - Parsed { - catalog: Arc::new(catalog), - raw, - errors, - rejected, - }, - git_blob_id(body), - )) +pub(crate) fn bundled() -> Fetched { + Fetched { + parsed: Arc::clone(&BUNDLED), + revision: BUNDLED_REVISION.clone(), + etag: None, + source: None, + started_at: SystemTime::now(), + } } -fn git_blob_id(body: &[u8]) -> String { - let mut hasher = Sha1::new(); - hasher.update(format!("blob {}\0", body.len()).as_bytes()); - hasher.update(body); - hasher - .finalize() - .iter() - .map(|byte| format!("{byte:02x}")) - .collect() +/// A fetched document with the rows decision 15 rejects replaced by bundled or left out +pub(crate) fn remote(catalog: Catalog, errors: &[Located]) -> Parsed { + let raw = Arc::new(catalog.clone()); + let (catalog, rejections) = catalog.with_rejected_replaced(errors, &BUNDLED.raw); + Parsed { + catalog: Arc::new(catalog), + raw, + errors: errors.iter().map(row_error).collect(), + rejected: rejections + .iter() + .flat_map(|rejection| rejection.errors.iter().copied().map(row_error)) + .collect(), + } } fn served_rows(catalog: &Catalog) -> Vec<(String, String, Fields)> { @@ -281,18 +299,3 @@ impl Serialize for WrittenDocument<'_> { serializer.collect_map(self.0.written()) } } - -#[cfg(test)] -mod tests { - use rstest::rstest; - - use super::*; - - #[rstest] - fn the_revision_is_the_git_blob_id_of_the_body() { - assert_eq!( - git_blob_id(b"hello\n"), - "ce013625030ba8dba906f756967f9e9ca394464a" - ); - } -} diff --git a/litellm-rust/crates/python-bridge/src/model_registry/fetch.rs b/litellm-rust/crates/python-bridge/src/model_registry/fetch.rs new file mode 100644 index 00000000000..484dc0f29a1 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/model_registry/fetch.rs @@ -0,0 +1,307 @@ +use std::path::PathBuf; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use litellm_host_python::release_gil; +use litellm_http::media::PublicDnsResolver; +use litellm_http::{ClientVariant, HttpClientConfig, HttpClientPool}; +use litellm_model_catalog::IntegrityLimits; +use litellm_model_catalog_refresh::{ + Installed, Origin, RetryPolicy, Source, Step, Transports, fetch_with_retry, +}; +use pyo3::exceptions::{PyTypeError, PyValueError}; +use pyo3::prelude::*; +use pyo3::types::PyDict; + +use super::{BUNDLED, BUNDLED_REVISION, Fetched, Parsed, Unavailable, bundled, remote}; + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct Remote { + #[pyo3(get)] + url: String, +} + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct File { + #[pyo3(get)] + path: String, +} + +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct Bundled; + +/// One snapshot of where the installed map came from, replacing main's field-by-field `ModelCostMapSourceInfo` +#[pyclass(frozen, skip_from_py_object, module = "litellm.rust_bridge._native")] +#[derive(Clone)] +pub(crate) struct Provenance { + /// `"local"` or `"remote"`, main's values + #[pyo3(get)] + source: &'static str, + #[pyo3(get)] + url: Option, + #[pyo3(get)] + is_env_forced: bool, + #[pyo3(get)] + fallback_reason: Option, + loaded_at: SystemTime, + #[pyo3(get)] + source_revision: String, + #[pyo3(get)] + etag: Option, +} + +#[pymethods] +impl Remote { + #[new] + fn new(url: String) -> Self { + Self { url } + } +} + +#[pymethods] +impl File { + #[new] + fn new(path: PathBuf) -> Self { + Self { + path: path.to_string_lossy().into_owned(), + } + } +} + +#[pymethods] +impl Bundled { + #[new] + fn new() -> Self { + Self + } +} + +#[pymethods] +impl Provenance { + /// When the load that installed this map started, as an aware UTC datetime + #[getter] + fn loaded_at<'py>(&self, py: Python<'py>) -> PyResult> { + let seconds = self + .loaded_at + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64(); + let datetime = py.import("datetime")?; + let utc = datetime.getattr("timezone")?.getattr("utc")?; + datetime + .getattr("datetime")? + .call_method1("fromtimestamp", (seconds, utc)) + } +} + +impl Provenance { + pub(crate) fn of( + fetched: &Fetched, + env_forced: bool, + fallback_reason: Option, + url: Option, + ) -> Self { + Self { + source: if fetched.source.is_some() { + "remote" + } else { + "local" + }, + url: url.or_else(|| fetched.source.as_ref().map(ToString::to_string)), + is_env_forced: env_forced, + fallback_reason, + loaded_at: fetched.started_at, + source_revision: fetched.revision.clone(), + etag: fetched.etag.clone(), + } + } +} + +/// What a Python `Source` names +pub(crate) enum Requested { + Bundled, + Source(Source), + Invalid(String), +} + +pub(crate) fn requested(source: &Bound<'_, PyAny>) -> PyResult { + if source.is_instance_of::() { + return Ok(Requested::Bundled); + } + if let Ok(remote) = source.cast::() { + let url = &remote.get().url; + return Ok(Source::parse(url).map_or_else( + |error| Requested::Invalid(error.to_string()), + Requested::Source, + )); + } + if let Ok(file) = source.cast::() { + return Ok(Requested::Source(Source::File(PathBuf::from( + &file.get().path, + )))); + } + Err(PyTypeError::new_err( + "source must be Remote, File or Bundled", + )) +} + +enum Outcome { + Fetched { + parsed: Parsed, + revision: String, + etag: Option, + }, + Unavailable { + reason: String, + retry_after: Option, + }, +} + +/// Fetches with the GIL released, retrying under main's policy, and validates the body before returning it +pub(crate) fn fetch( + py: Python<'_>, + requested: Requested, + timeout: f64, + attempts: u32, +) -> PyResult> { + let source = match requested { + Requested::Bundled => return Ok(bundled().into_pyobject(py)?.into_any().unbind()), + Requested::Invalid(reason) => return unavailable(py, reason, None), + Requested::Source(source) => source, + }; + let timeout = Duration::try_from_secs_f64(timeout) + .map_err(|_| PyValueError::new_err("timeout must be a non-negative number of seconds"))?; + let policy = RetryPolicy { + max_attempts: attempts.max(1), + timeout, + limits: integrity_limits(py)?, + ..RetryPolicy::default() + }; + let config = crate::http::call_config(py, &PyDict::new(py), false)?; + let started_at = SystemTime::now(); + let fetched_from = source.clone(); + match release_gil(py, move || drive(&source, policy, config)) { + Outcome::Fetched { + parsed, + revision, + etag, + } => Ok(Fetched { + parsed: Arc::new(parsed), + revision, + etag, + source: Some(fetched_from), + started_at, + } + .into_pyobject(py)? + .into_any() + .unbind()), + Outcome::Unavailable { + reason, + retry_after, + } => unavailable(py, reason, retry_after), + } +} + +fn unavailable( + py: Python<'_>, + reason: String, + retry_after: Option, +) -> PyResult> { + Ok(Unavailable { + reason, + retry_after: retry_after.map(|wait| wait.as_secs_f64()), + } + .into_pyobject(py)? + .into_any() + .unbind()) +} + +/// Main's `MODEL_COST_MAP_MIN_MODEL_COUNT` and `MODEL_COST_MAP_MAX_SHRINK_RATIO`, which are env-overridable +fn integrity_limits(py: Python<'_>) -> PyResult { + let constants = py.import("litellm.constants")?; + let minimum: usize = constants + .getattr("MODEL_COST_MAP_MIN_MODEL_COUNT")? + .extract()?; + let ratio: f64 = constants + .getattr("MODEL_COST_MAP_MAX_SHRINK_RATIO")? + .extract()?; + IntegrityLimits::new(minimum, ratio).ok_or_else(|| { + PyValueError::new_err("MODEL_COST_MAP_MAX_SHRINK_RATIO must be between 0 and 1") + }) +} + +/// A current-thread runtime of its own, so a fetch at import never starts the shared native runtime, and a client +/// built inside it, so no pooled connection outlives the runtime that spawned it +fn drive(source: &Source, policy: RetryPolicy, config: HttpClientConfig) -> Outcome { + if tokio::runtime::Handle::try_current().is_ok() { + return Outcome::Unavailable { + reason: "the model cost map fetch cannot run from a Tokio context".to_owned(), + retry_after: None, + }; + } + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + return Outcome::Unavailable { + reason: format!("the model cost map fetch runtime could not start: {error}"), + retry_after: None, + }; + } + }; + let step = runtime.block_on(async { + let pool = HttpClientPool::new(Arc::new(PublicDnsResolver)); + let client = match pool.client(&config, ClientVariant::Provider) { + Ok(client) => client, + Err(error) => { + return Step::Done { + reason: format!("the HTTP client could not be built: {error}"), + retry_after: None, + }; + } + }; + let installed = Installed { + origin: Origin::Bundled, + etag: None, + revision: BUNDLED_REVISION.clone(), + model_count: BUNDLED.raw.model_count(), + }; + fetch_with_retry( + source, + &installed, + policy, + None, + &Transports { + http: &client, + s3: None, + }, + tokio::time::sleep, + ) + .await + }); + runtime.shutdown_timeout(Duration::from_secs(1)); + match step { + Step::Adopt { + catalog, + installed, + errors, + } => Outcome::Fetched { + parsed: remote(*catalog, &errors), + revision: installed.revision, + etag: installed.etag, + }, + Step::Done { + reason, + retry_after, + } => Outcome::Unavailable { + reason, + retry_after, + }, + Step::Fetch(_) | Step::Sleep(_) => Outcome::Unavailable { + reason: "the model cost map fetch stopped before it finished".to_owned(), + retry_after: None, + }, + } +} diff --git a/litellm-rust/crates/s3/Cargo.toml b/litellm-rust/crates/s3/Cargo.toml new file mode 100644 index 00000000000..acaaef65ea7 --- /dev/null +++ b/litellm-rust/crates/s3/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "litellm-s3" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +aws-credential-types = "1.3.0" +aws-sdk-s3 = { version = "1.146.1", default-features = false, features = ["rustls", "rt-tokio"] } +aws-smithy-runtime-api = { version = "1.16.2", features = ["client", "http-1x"] } +aws-smithy-types = { version = "1.6.0", features = ["http-body-1-x"] } +http.workspace = true +litellm-http.workspace = true +reqwest.workspace = true diff --git a/litellm-rust/crates/s3/src/lib.rs b/litellm-rust/crates/s3/src/lib.rs new file mode 100644 index 00000000000..ce480be8742 --- /dev/null +++ b/litellm-rust/crates/s3/src/lib.rs @@ -0,0 +1,35 @@ +mod transport; + +use aws_credential_types::provider::ProvideCredentials; +use aws_sdk_s3::config::{ + BehaviorVersion, Region, RequestChecksumCalculation, ResponseChecksumValidation, +}; + +pub use transport::ReqwestHttpClient; + +/// Where an S3 client sends its requests +pub struct S3ClientConfig { + pub region: String, + /// An S3-compatible endpoint (MinIO, R2, Ceph), addressed path-style + pub endpoint: Option, +} + +/// One way to build an S3 client over a reqwest client, shared by every S3 caller +pub fn client( + config: S3ClientConfig, + http: litellm_http::Client, + credentials: impl ProvideCredentials + 'static, +) -> aws_sdk_s3::Client { + let base = aws_sdk_s3::Config::builder() + .behavior_version(BehaviorVersion::latest()) + .region(Region::new(config.region)) + .http_client(ReqwestHttpClient(http)) + .credentials_provider(credentials) + .request_checksum_calculation(RequestChecksumCalculation::WhenRequired) + .response_checksum_validation(ResponseChecksumValidation::WhenRequired); + let builder = match config.endpoint { + Some(url) => base.endpoint_url(url).force_path_style(true), + None => base, + }; + aws_sdk_s3::Client::from_conf(builder.build()) +} diff --git a/litellm-rust/crates/cache-s3/src/transport.rs b/litellm-rust/crates/s3/src/transport.rs similarity index 95% rename from litellm-rust/crates/cache-s3/src/transport.rs rename to litellm-rust/crates/s3/src/transport.rs index eabc54ac9e2..56c03ff5065 100644 --- a/litellm-rust/crates/cache-s3/src/transport.rs +++ b/litellm-rust/crates/s3/src/transport.rs @@ -9,7 +9,7 @@ use aws_smithy_runtime_api::client::{ use aws_smithy_types::body::SdkBody; #[derive(Clone, Debug)] -pub(crate) struct ReqwestHttpClient(pub(crate) litellm_http::Client); +pub struct ReqwestHttpClient(pub litellm_http::Client); impl HttpClient for ReqwestHttpClient { fn http_connector( diff --git a/litellm/_catalog/__init__.py b/litellm/_catalog/__init__.py index 0b9c4f653cb..e928cd9f97f 100644 --- a/litellm/_catalog/__init__.py +++ b/litellm/_catalog/__init__.py @@ -1,10 +1,47 @@ """Internal catalog API backed by the Rust ``model-catalog`` crate. Not public, its shape may change in any release""" +import os import sys from typing import Final -from litellm.rust_bridge._native import Catalog, Fetched, Installed, RowError, Unavailable +from litellm.litellm_core_utils.offline_mode import is_offline_mode +from litellm.rust_bridge._native import ( + Bundled, + Catalog, + Fetched, + File, + Installed, + Provenance, + Remote, + RowError, + Unavailable, +) + +Source = Remote | File | Bundled CATALOG: Final = Catalog.bound_to(sys.modules["litellm"], "model_cost") -__all__ = ["CATALOG", "Catalog", "Fetched", "Installed", "RowError", "Unavailable"] + +def default_source() -> Source: + """Main's reload rule, the bundled map under offline mode or ``LITELLM_LOCAL_MODEL_COST_MAP``""" + import litellm + + if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true": + return Bundled() + return Remote(litellm.model_cost_map_url) + + +__all__ = [ + "CATALOG", + "Bundled", + "Catalog", + "Fetched", + "File", + "Installed", + "Provenance", + "Remote", + "RowError", + "Source", + "Unavailable", + "default_source", +] diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index e727bb8eff1..1aa570215f4 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -9,31 +9,23 @@ export LITELLM_LOCAL_MODEL_COST_MAP=True ``` """ -import asyncio import os -import random import sys import threading import time -import urllib.parse -from collections.abc import Awaitable, Callable, Mapping -from dataclasses import dataclass -from datetime import datetime, timezone +import warnings +from collections.abc import Callable, Mapping +from datetime import datetime from itertools import chain from pathlib import Path from types import MappingProxyType -from typing import Final, Protocol, cast # noqa: TID251 # adapts boto3 and botocore's dynamic client responses +from typing import Final -import httpx from pydantic import JsonValue, TypeAdapter -from typing_extensions import NotRequired, ReadOnly, TypedDict +from typing_extensions import ReadOnly, TypedDict from litellm import verbose_logger -from litellm._catalog import CATALOG, Catalog, Fetched, Installed, Unavailable -from litellm.constants import ( - MODEL_COST_MAP_MAX_SHRINK_RATIO, - MODEL_COST_MAP_MIN_MODEL_COUNT, -) +from litellm._catalog import CATALOG, Catalog, Fetched, Installed, Remote, Source, Unavailable from litellm.litellm_core_utils.fallback_generalizations import ( set_fallback_generalizations, ) @@ -42,12 +34,25 @@ from litellm.litellm_core_utils.offline_mode import is_offline_mode FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations" _CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]]) _CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"}) +MODEL_COST_MAP_FETCH_MAX_ATTEMPTS: Final = 3 +_IMPORT_FETCH_TIMEOUT_SECONDS: Final = 5 +_litellm_import_complete = threading.Event() def _is_cli_process() -> bool: return Path(sys.argv[0]).stem in _CLI_ENTRYPOINT_NAMES +def _local_map_forced() -> bool: + # Note: can't use get_secret_bool here — this runs during litellm.__init__ + # before litellm._key_management_settings is set. + return is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" + + +def mark_litellm_import_complete() -> None: + _litellm_import_complete.set() + + class GetModelCostMap: """The catalog as last loaded, for readers that want it without later registrations""" @@ -71,304 +76,6 @@ class GetModelCostMap: cls._loaded_catalog_fields = frozenset(chain.from_iterable(entry.keys() for entry in raw.values())) -RETRYABLE_FETCH_STATUS_CODES: Final = frozenset({429, 500, 502, 503, 504}) -MODEL_COST_MAP_FETCH_MAX_ATTEMPTS: Final = 3 -MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS: Final = 30.0 -_litellm_import_complete = threading.Event() - - -def mark_litellm_import_complete() -> None: - _litellm_import_complete.set() - - -@dataclass(frozen=True, slots=True) -class ModelCostMapReloaded: - fetched: Fetched - - @property - def revision(self) -> str: - return self.fetched.revision - - @property - def etag(self) -> str | None: - return self.fetched.etag - - -@dataclass(frozen=True, slots=True) -class ModelCostMapReloadUnavailable: - reason: str - - -ModelCostMapReloadResult = ModelCostMapReloaded | ModelCostMapReloadUnavailable - - -@dataclass(frozen=True, slots=True) -class _FetchAttemptRetryable: - reason: str - retry_after_seconds: float | None - - -def _parse_retry_after_seconds(response: httpx.Response) -> float | None: - try: - header: Final = response.headers["Retry-After"] - except KeyError: - return None - try: - seconds: Final = float(header) - except ValueError: - return None - return seconds if seconds >= 0 else None - - -def _retry_wait_seconds(outcome: _FetchAttemptRetryable, attempt: int, rng: random.Random) -> float: - if outcome.retry_after_seconds is not None: - return min(outcome.retry_after_seconds, MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS) - return min(2.0**attempt + rng.uniform(0.0, 1.0), MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS) - - -class _AsyncGetClient(Protocol): - def get(self, url: str, *, timeout: float | None = None) -> Awaitable[httpx.Response]: ... - - -class _SyncGetClient(Protocol): - def get(self, url: str, *, timeout: float | None = None) -> httpx.Response: ... - - -class _S3Body(Protocol): - def read(self) -> bytes: ... - - -class _S3GetObjectResponse(TypedDict): - Body: ReadOnly[_S3Body] - ETag: NotRequired[ReadOnly[str]] - - -class _S3Client(Protocol): - def get_object(self, *, Bucket: str, Key: str) -> _S3GetObjectResponse: ... - - -_FetchAttemptOutcome = ModelCostMapReloaded | ModelCostMapReloadUnavailable | _FetchAttemptRetryable - - -def _default_reload_client() -> _AsyncGetClient: - from litellm.llms.custom_httpx.http_handler import get_async_httpx_client - from litellm.types.llms.custom_http import httpxSpecialProvider - - return get_async_httpx_client(llm_provider=httpxSpecialProvider.ModelCostMap) - - -def _default_s3_client(timeout: int) -> _S3Client: - import boto3 - from botocore.config import Config - - return cast( # cast-ok: boto3's generic client factory does not expose its generated S3 response type - _S3Client, - boto3.client( # pyright: ignore[reportUnknownMemberType] # generic factory returns Unknown service clients - "s3", - region_name=os.getenv("AWS_REGION_NAME") or None, - config=Config(connect_timeout=timeout, read_timeout=timeout, retries={"total_max_attempts": 1}), - ), - ) - - -def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) -> _FetchAttemptOutcome: - reason: Final = f"{type(error).__name__} fetching {url}: {error}" - if isinstance(error, (httpx.InvalidURL, httpx.UnsupportedProtocol)): - return ModelCostMapReloadUnavailable(reason=reason) - return _FetchAttemptRetryable(reason=reason, retry_after_seconds=None) - - -def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _FetchAttemptOutcome: - parsed: Final = CATALOG.parse( - content, - etag=etag, - min_model_count=MODEL_COST_MAP_MIN_MODEL_COUNT, - max_shrink_ratio=MODEL_COST_MAP_MAX_SHRINK_RATIO, - ) - if isinstance(parsed, Unavailable): - return ModelCostMapReloadUnavailable(reason=f"model cost map from {url} is unusable: {parsed.reason}") - return ModelCostMapReloaded(fetched=parsed) - - -def _split_url(url: str) -> urllib.parse.SplitResult | ModelCostMapReloadUnavailable: - try: - return urllib.parse.urlsplit(url) - except ValueError as error: - return ModelCostMapReloadUnavailable(reason=f"invalid URL {url}: {error}") - - -def _attempt_fetch_s3(parts: urllib.parse.SplitResult, url: str, timeout: int) -> _FetchAttemptOutcome: - from botocore.exceptions import BotoCoreError, ClientError - - bucket: Final = parts.netloc - key: Final = parts.path.lstrip("/") - if not bucket or not key: - return ModelCostMapReloadUnavailable(reason=f"invalid S3 URL {url}: expected s3://bucket/key") - - try: - client: Final = _default_s3_client(timeout) - response: Final = client.get_object(Bucket=bucket, Key=key) - content: Final = response["Body"].read() - except (BotoCoreError, ClientError) as error: - return ModelCostMapReloadUnavailable(reason=f"S3 {type(error).__name__} fetching {url}: {error}") - return _classify_fetch_content(content, url, response.get("ETag")) - - -async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome: - parts: Final = _split_url(url) - if isinstance(parts, ModelCostMapReloadUnavailable): - return parts - if parts.scheme == "s3": - return await asyncio.to_thread(_attempt_fetch_s3, parts, url, timeout) - try: - response: Final = await client.get(url, timeout=timeout) - except (httpx.HTTPError, httpx.InvalidURL) as e: - return _classify_fetch_error(e, url) - return _classify_fetch_response(response, url) - - -def _attempt_fetch_sync(client: _SyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome: - parts: Final = _split_url(url) - if isinstance(parts, ModelCostMapReloadUnavailable): - return parts - if parts.scheme == "s3": - return _attempt_fetch_s3(parts, url, timeout) - try: - response: Final = client.get(url, timeout=timeout) - except (httpx.HTTPError, httpx.InvalidURL) as e: - return _classify_fetch_error(e, url) - return _classify_fetch_response(response, url) - - -def _classify_fetch_response(response: httpx.Response, url: str) -> _FetchAttemptOutcome: - if response.status_code in RETRYABLE_FETCH_STATUS_CODES: - return _FetchAttemptRetryable( - reason=f"HTTP {response.status_code} from {url}", - retry_after_seconds=_parse_retry_after_seconds(response), - ) - if response.is_error: - return ModelCostMapReloadUnavailable(reason=f"HTTP {response.status_code} from {url}") - etag_values: Final = response.headers.get_list("etag") - return _classify_fetch_content(response.content, url, ", ".join(etag_values) if etag_values else None) - - -def _next_retry_wait( - outcome: _FetchAttemptRetryable, attempt: int, max_attempts: int, rng: random.Random -) -> float | ModelCostMapReloadUnavailable: - if attempt == max_attempts: - return ModelCostMapReloadUnavailable(reason=f"{outcome.reason} (after {max_attempts} attempts)") - wait_seconds: Final = _retry_wait_seconds(outcome=outcome, attempt=attempt, rng=rng) - verbose_logger.warning( - "LiteLLM: model cost map fetch attempt %d/%d failed (%s); retrying in %.1fs", - attempt, - max_attempts, - outcome.reason, - wait_seconds, - ) - return wait_seconds - - -async def _fetch_remote_model_cost_map_with_retry( - url: str, - timeout: int, - max_attempts: int, - sleep: Callable[[float], Awaitable[None]], - rng: random.Random, - client: _AsyncGetClient, -) -> ModelCostMapReloadResult: - for attempt in range(1, max_attempts + 1): - outcome = await _attempt_fetch(client=client, url=url, timeout=timeout) - if not isinstance(outcome, _FetchAttemptRetryable): - return outcome - wait_seconds = _next_retry_wait(outcome=outcome, attempt=attempt, max_attempts=max_attempts, rng=rng) - if isinstance(wait_seconds, ModelCostMapReloadUnavailable): - return wait_seconds - await sleep(wait_seconds) - return ModelCostMapReloadUnavailable(reason="model cost map fetch failed") - - -def _fetch_remote_model_cost_map_with_retry_sync( - url: str, - timeout: int, - attempts: range, - sleep: Callable[[float], None], - rng: random.Random, - client: _SyncGetClient, -) -> ModelCostMapReloadResult: - max_attempts: Final = attempts.stop - 1 - for attempt in attempts: - outcome = _attempt_fetch_sync(client=client, url=url, timeout=timeout) - if not isinstance(outcome, _FetchAttemptRetryable): - return outcome - wait_seconds = _next_retry_wait(outcome=outcome, attempt=attempt, max_attempts=max_attempts, rng=rng) - if isinstance(wait_seconds, ModelCostMapReloadUnavailable): - return wait_seconds - sleep(wait_seconds) - return ModelCostMapReloadUnavailable(reason="model cost map fetch failed") - - -async def refetch_model_cost_map( - url: str, - timeout: int = 5, - max_attempts: int = MODEL_COST_MAP_FETCH_MAX_ATTEMPTS, - sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, - rng: random.Random | None = None, - client: "_AsyncGetClient | None" = None, -) -> ModelCostMapReloadResult: - """ - Re-fetch the model cost map for a runtime reload, retrying transient HTTP - errors (429/5xx/transport) with Retry-After-aware backoff. - - Unlike ``get_model_cost_map`` this never falls back to the packaged backup: - on failure it returns ``ModelCostMapReloadUnavailable`` so callers keep the - map they already have. - """ - if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true": - _cost_map_source_info.loaded_at = datetime.now(timezone.utc) - _cost_map_source_info.source = "local" - _cost_map_source_info.url = None - _cost_map_source_info.is_env_forced = True - _cost_map_source_info.fallback_reason = None - return _note_loaded(ModelCostMapReloaded(fetched=CATALOG.bundled())) - - result: Final = await _fetch_remote_model_cost_map_with_retry( - url=url, - timeout=timeout, - max_attempts=max_attempts, - sleep=sleep, - rng=rng if rng is not None else random.Random(), - client=client if client is not None else _default_reload_client(), - ) - if isinstance(result, ModelCostMapReloadUnavailable): - verbose_logger.warning( - "LiteLLM: model cost map reload failed: %s. Keeping the currently loaded map.", - result.reason, - ) - return result - _cost_map_source_info.loaded_at = datetime.now(timezone.utc) - _cost_map_source_info.source = "remote" - _cost_map_source_info.url = url - _cost_map_source_info.is_env_forced = False - _cost_map_source_info.fallback_reason = None - return _note_loaded(result) - - -class ModelCostMapSourceInfo: - """Tracks the source of the currently loaded model cost map.""" - - source: str = "local" # "local" or "remote" - url: str | None = None - is_env_forced: bool = False - fallback_reason: str | None = None - loaded_at: "datetime | None" = None - source_revision: str | None = None - etag: str | None = None - - -# Module-level singleton tracking the source of the current cost map -_cost_map_source_info: Final = ModelCostMapSourceInfo() - - class CostMapProvenance(TypedDict): source_revision: ReadOnly[str | None] etag: ReadOnly[str | None] @@ -383,9 +90,10 @@ class CostMapSourceInfo(CostMapProvenance): def get_model_cost_map_provenance() -> CostMapProvenance: + provenance: Final = CATALOG.provenance() return { - "source_revision": _cost_map_source_info.source_revision, - "etag": _cost_map_source_info.etag, + "source_revision": provenance.source_revision if provenance is not None else None, + "etag": provenance.etag if provenance is not None else None, } @@ -402,21 +110,32 @@ def get_model_cost_map_source_info() -> CostMapSourceInfo: - source_revision: git blob id of the loaded file's bytes - etag: the ETag of the remote fetch (None for the bundled backup) """ - loaded_at: Final = _cost_map_source_info.loaded_at + provenance: Final = CATALOG.provenance() + if provenance is None: + return { + "source": "local", + "url": None, + "is_env_forced": False, + "fallback_reason": None, + "loaded_at": None, + "source_revision": None, + "etag": None, + } return { - "source": _cost_map_source_info.source, - "url": _cost_map_source_info.url, - "is_env_forced": _cost_map_source_info.is_env_forced, - "fallback_reason": _cost_map_source_info.fallback_reason, - "loaded_at": loaded_at.isoformat() if loaded_at is not None else None, - "source_revision": _cost_map_source_info.source_revision, - "etag": _cost_map_source_info.etag, + "source": provenance.source, + "url": provenance.url, + "is_env_forced": provenance.is_env_forced, + "fallback_reason": provenance.fallback_reason, + "loaded_at": provenance.loaded_at.isoformat(), + "source_revision": provenance.source_revision, + "etag": provenance.etag, } def get_model_cost_map_loaded_at() -> "datetime | None": """When this process last loaded its cost map, stamped at the start of every load""" - return _cost_map_source_info.loaded_at + provenance: Final = CATALOG.provenance() + return provenance.loaded_at if provenance is not None else None class AliasedCostEntry(dict[str, object]): @@ -430,14 +149,12 @@ class AliasedCostEntry(dict[str, object]): self.registered_fields: frozenset[str] = registered_fields -def _note_loaded(loaded: ModelCostMapReloaded) -> ModelCostMapReloaded: - _cost_map_source_info.source_revision = loaded.fetched.revision - _cost_map_source_info.etag = loaded.fetched.etag - set_fallback_generalizations(loaded.fetched.fallback_rules()) +def _note_loaded(fetched: Fetched) -> Fetched: + set_fallback_generalizations(fetched.fallback_rules()) GetModelCostMap._snapshot_loaded_catalog( # pyright: ignore[reportPrivateUsage] # same module - loaded.fetched.served() + fetched.served() ) - return loaded + return fetched class _InstallGeneration: @@ -450,8 +167,15 @@ def model_cost_install_generation() -> int: return _InstallGeneration.value -def _install(fetched: Fetched, catalog: Catalog = CATALOG) -> Installed: - installed: Final = catalog.install(fetched) +def _install( + fetched: Fetched, + catalog: Catalog = CATALOG, + *, + env_forced: bool = False, + fallback_reason: str | None = None, + url: str | None = None, +) -> Installed: + installed: Final = catalog.install(fetched, env_forced=env_forced, fallback_reason=fallback_reason, url=url) _InstallGeneration.value += 1 if installed.rejected: verbose_logger.warning( @@ -461,40 +185,38 @@ def _install(fetched: Fetched, catalog: Catalog = CATALOG) -> Installed: return installed -def adopt_model_cost_map(fetched: Fetched) -> int: +def _refresh_derived_state() -> None: import litellm from litellm import utils - installed: Final = _install(fetched) utils._invalidate_model_cost_lowercase_map() # pyright: ignore[reportPrivateUsage] # required cache invalidation litellm.add_known_models(model_cost_map=litellm.model_cost) utils.reapply_runtime_model_cost_registrations() - return installed.count + + +def reload_model_cost_map(source: Source, catalog: Catalog = CATALOG) -> Installed | Unavailable: + """Fetch with main's reload policy, then install and refresh what Python still derives from the map. A failed + fetch keeps the map in use""" + fetched: Final = catalog.fetch( + source, timeout=_IMPORT_FETCH_TIMEOUT_SECONDS, attempts=MODEL_COST_MAP_FETCH_MAX_ATTEMPTS + ) + if isinstance(fetched, Unavailable): + verbose_logger.warning( + "LiteLLM: model cost map reload failed: %s. Keeping the currently loaded map.", fetched.reason + ) + return fetched + installed: Final = _install(_note_loaded(fetched), catalog, env_forced=_local_map_forced()) + _refresh_derived_state() + return installed def _retry_remote_fetch_in_background( - url: str, - timeout: int, - max_attempts: int, - sleep: Callable[[float], None], - rng: random.Random, - client: _SyncGetClient, - first_outcome: _FetchAttemptRetryable, + url: str, max_attempts: int, first_wait: float, sleep: Callable[[float], None], catalog: Catalog ) -> None: try: - first_wait: Final = _next_retry_wait(outcome=first_outcome, attempt=1, max_attempts=max_attempts, rng=rng) - if isinstance(first_wait, ModelCostMapReloadUnavailable): - return sleep(first_wait) - result: Final = _fetch_remote_model_cost_map_with_retry_sync( - url=url, - timeout=timeout, - attempts=range(2, max_attempts + 1), - sleep=sleep, - rng=rng, - client=client, - ) - if isinstance(result, ModelCostMapReloadUnavailable): + fetched: Final = catalog.fetch(Remote(url), timeout=_IMPORT_FETCH_TIMEOUT_SECONDS, attempts=max_attempts - 1) + if isinstance(fetched, Unavailable): verbose_logger.warning( "LiteLLM: Failed to fetch remote model cost map from %s after %d attempts; keeping local backup", url, @@ -502,100 +224,83 @@ def _retry_remote_fetch_in_background( ) return _litellm_import_complete.wait() - _note_loaded(result) - _cost_map_source_info.source = "remote" - _cost_map_source_info.fallback_reason = None - _cost_map_source_info.loaded_at = datetime.now(timezone.utc) - adopt_model_cost_map(result.fetched) + _install(_note_loaded(fetched), catalog, url=url) + _refresh_derived_state() except Exception as e: # noqa: BLE001 # a failed background retry must not kill the task; the backup stays verbose_logger.warning("LiteLLM: Background model cost map retry failed: %s", e) def _pick_model_cost_map( - url: str, - timeout: int, - max_attempts: int, - sleep: Callable[[float], None], - rng: random.Random | None, - client: "_SyncGetClient | None", -) -> Fetched: + url: str, timeout: int, max_attempts: int, sleep: Callable[[float], None], catalog: Catalog +) -> tuple[Fetched, bool, str | None]: """ - The map main loads at import. The bundled map when ``LITELLM_LOCAL_MODEL_COST_MAP`` is set, in offline mode or in a - ``lite`` / ``litellm-proxy`` CLI process. Otherwise one remote attempt, with a background retry on a transient - failure, and the bundled map on any failure + The map main loads at import, with the provenance facts install records. The bundled map when + ``LITELLM_LOCAL_MODEL_COST_MAP`` is set, in offline mode or in a ``lite`` / ``litellm-proxy`` CLI process. + Otherwise one remote attempt, with a background retry on a transient failure, and the bundled map on any failure """ - _cost_map_source_info.loaded_at = datetime.now(timezone.utc) - # Note: can't use get_secret_bool here — this runs during litellm.__init__ - # before litellm._key_management_settings is set. - if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process(): - _cost_map_source_info.source = "local" - _cost_map_source_info.url = None - _cost_map_source_info.is_env_forced = True - _cost_map_source_info.fallback_reason = None - return _note_loaded(ModelCostMapReloaded(fetched=CATALOG.bundled())).fetched - - _cost_map_source_info.url = url - _cost_map_source_info.is_env_forced = False - - fetch_client: Final = client if client is not None else httpx - fetch_rng: Final = rng if rng is not None else random.Random() - outcome: Final = _attempt_fetch_sync(client=fetch_client, url=url, timeout=timeout) - if isinstance(outcome, _FetchAttemptRetryable) and max_attempts > 1: + if _local_map_forced() or _is_cli_process(): + return _note_loaded(catalog.bundled()), True, None + fetched: Final = catalog.fetch(Remote(url or ""), timeout=timeout, attempts=1) + if isinstance(fetched, Fetched): + return _note_loaded(fetched), False, None + if fetched.retry_after is not None and max_attempts > 1: threading.Thread( target=_retry_remote_fetch_in_background, kwargs={ "url": url, - "timeout": timeout, "max_attempts": max_attempts, + "first_wait": fetched.retry_after, "sleep": sleep, - "rng": fetch_rng, - "client": fetch_client, - "first_outcome": outcome, + "catalog": catalog, }, name="litellm-model-cost-map-retry", daemon=True, ).start() - if not isinstance(outcome, ModelCostMapReloaded): - verbose_logger.warning( - "LiteLLM: Failed to fetch remote model cost map from %s: %s. Falling back to local backup.", - url, - outcome.reason, - ) - _cost_map_source_info.source = "local" - _cost_map_source_info.fallback_reason = f"Remote fetch failed: {outcome.reason}" - return _note_loaded(ModelCostMapReloaded(fetched=CATALOG.bundled())).fetched - - _cost_map_source_info.source = "remote" - _cost_map_source_info.fallback_reason = None - return _note_loaded(outcome).fetched + verbose_logger.warning( + "LiteLLM: Failed to fetch remote model cost map from %s: %s. Falling back to local backup.", + url, + fetched.reason, + ) + return _note_loaded(catalog.bundled()), False, f"Remote fetch failed: {fetched.reason}" -def install_initial_model_cost_map(url: str) -> None: +def install_initial_model_cost_map( + url: str, + catalog: Catalog = CATALOG, + sleep: Callable[[float], None] = time.sleep, + max_attempts: int = MODEL_COST_MAP_FETCH_MAX_ATTEMPTS, +) -> Installed: """Fills ``litellm.model_cost`` at import with the map main's rules pick""" - _install( - _pick_model_cost_map( - url=url, - timeout=5, - max_attempts=MODEL_COST_MAP_FETCH_MAX_ATTEMPTS, - sleep=time.sleep, - rng=None, - client=None, - ) + fetched, env_forced, fallback_reason = _pick_model_cost_map( + url=url, + timeout=_IMPORT_FETCH_TIMEOUT_SECONDS, + max_attempts=max_attempts, + sleep=sleep, + catalog=catalog, + ) + return _install( + fetched, catalog, env_forced=env_forced, fallback_reason=fallback_reason, url=None if env_forced else url ) def get_model_cost_map( url: str, - timeout: int = 5, + timeout: int = _IMPORT_FETCH_TIMEOUT_SECONDS, max_attempts: int = MODEL_COST_MAP_FETCH_MAX_ATTEMPTS, sleep: Callable[[float], None] = time.sleep, - rng: random.Random | None = None, - client: "_SyncGetClient | None" = None, + rng: object = None, + client: object = None, ) -> dict[str, dict[str, JsonValue]]: """ - Public entry point, returns a fresh model cost map dict picked by main's rules. A remote body is parsed and - validated by the Rust catalog, and the bundled map is returned on any failure + Public entry point, returns a fresh model cost map dict picked by main's rules. The fetch, its retries and the + validation of the body run in Rust, and the bundled map is returned on any failure. ``rng`` and ``client`` are + accepted for compatibility and ignored, since Rust owns the HTTP client and the backoff jitter """ - return _pick_model_cost_map( - url=url, timeout=timeout, max_attempts=max_attempts, sleep=sleep, rng=rng, client=client - ).served() + if rng is not None or client is not None: + warnings.warn( + "get_model_cost_map ignores rng and client, the fetch runs in Rust", DeprecationWarning, stacklevel=2 + ) + fetched, _, _ = _pick_model_cost_map( + url=url, timeout=timeout, max_attempts=max_attempts, sleep=sleep, catalog=CATALOG + ) + return fetched.served() diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 27b79ef51f4..433fc050fdf 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -211,7 +211,6 @@ if TYPE_CHECKING: from opentelemetry.trace import Span as _Span from prisma import models as prisma_models - from litellm._catalog import Fetched from litellm.integrations.opentelemetry import OpenTelemetry from litellm.proxy.health_check_utils.shared_health_check_manager import SharedHealthCheckManager @@ -5386,12 +5385,6 @@ def resolve_classifier_plugin( return resolved -def _swap_in_model_cost_map(fetched: "Fetched") -> int: - from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map - - return adopt_model_cost_map(fetched) - - def _websearch_handler_params(stored: Mapping[str, object]) -> dict[str, object]: """ Translate stored web search interception settings into handler kwargs. @@ -9091,13 +9084,11 @@ class ProxyConfig: if not is_due: return - from litellm.litellm_core_utils.get_model_cost_map import ( - ModelCostMapReloadUnavailable, - refetch_model_cost_map, - ) + from litellm._catalog import Unavailable, default_source + from litellm.litellm_core_utils.get_model_cost_map import reload_model_cost_map - reload_result = await refetch_model_cost_map(url=litellm.model_cost_map_url) - if isinstance(reload_result, ModelCostMapReloadUnavailable): + reload_result = await asyncio.to_thread(reload_model_cost_map, default_source()) + if isinstance(reload_result, Unavailable): verbose_proxy_logger.warning( "Model cost map reload failed (%s); keeping current pricing data. The revision stays " "unapplied so this pod retries on its next poll", @@ -9105,7 +9096,7 @@ class ProxyConfig: ) return - models_count = _swap_in_model_cost_map(reload_result.fetched) + models_count = reload_result.count self.model_cost_map_loaded_at = current_time await record_reload_run(prisma_client, MODEL_COST_MAP_RELOAD_PARAM_NAME, current_time) # Adopted last, so neither a failed fetch nor a failed status write is recorded @@ -20999,20 +20990,20 @@ async def reload_model_cost_map( raise HTTPException(status_code=500, detail="Database connection not available") # Immediately reload the model cost map in the current pod + from litellm._catalog import Unavailable, default_source from litellm.litellm_core_utils.get_model_cost_map import ( - ModelCostMapReloadUnavailable, get_model_cost_map_provenance, - refetch_model_cost_map, + reload_model_cost_map, ) - reload_result = await refetch_model_cost_map(url=litellm.model_cost_map_url) - if isinstance(reload_result, ModelCostMapReloadUnavailable): + reload_result = await asyncio.to_thread(reload_model_cost_map, default_source()) + if isinstance(reload_result, Unavailable): raise HTTPException( status_code=502, detail=f"Failed to reload model cost map: {reload_result.reason}. Current pricing data was kept.", ) - models_count = _swap_in_model_cost_map(reload_result.fetched) + models_count = reload_result.count current_time = utc_now() proxy_config.model_cost_map_loaded_at = current_time provenance: Final = get_model_cost_map_provenance() diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 183f6f16c1b..baafc901f94 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -1,5 +1,7 @@ +import os from asyncio import Future from collections.abc import Mapping, Sequence +from datetime import datetime from types import ModuleType from typing import Literal, Never, final @@ -70,17 +72,50 @@ class Installed: @final class Unavailable: reason: str + retry_after: float | None + def __new__(cls, reason: str, retry_after: float | None = None) -> Unavailable: ... + +@final +class Remote: + url: str + def __new__(cls, url: str) -> Remote: ... + +@final +class File: + path: str + def __new__(cls, path: str | os.PathLike[str]) -> File: ... + +@final +class Bundled: + def __new__(cls) -> Bundled: ... + +@final +class Provenance: + source: str + url: str | None + is_env_forced: bool + fallback_reason: str | None + source_revision: str + etag: str | None + @property + def loaded_at(self) -> datetime: ... @final class Catalog: @staticmethod def bound_to(module: ModuleType, attr: str) -> Catalog: ... def bundled(self) -> Fetched: ... - @staticmethod - def parse( - body: bytes, *, etag: str | None, min_model_count: int, max_shrink_ratio: float - ) -> Fetched | Unavailable: ... - def install(self, fetched: Fetched) -> Installed: ... + def fetch(self, source: Remote | File | Bundled, *, timeout: float, attempts: int) -> Fetched | Unavailable: ... + def reload(self, source: Remote | File | Bundled) -> Installed | Unavailable: ... + def provenance(self) -> Provenance | None: ... + def install( + self, + fetched: Fetched, + *, + env_forced: bool = False, + fallback_reason: str | None = None, + url: str | None = None, + ) -> Installed: ... @final class NativeClickHouseSpendConfig: diff --git a/tests/unit/_catalog/__init__.py b/tests/unit/_catalog/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/_catalog/test_fetch.py b/tests/unit/_catalog/test_fetch.py new file mode 100644 index 00000000000..b9741fddd72 --- /dev/null +++ b/tests/unit/_catalog/test_fetch.py @@ -0,0 +1,88 @@ +import contextlib +import datetime +import ipaddress +import ssl +import threading +import types +from collections.abc import Generator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.x509.oid import NameOID + +import litellm +from litellm._catalog import CATALOG, Catalog, Installed, Remote, Unavailable + + +def _self_signed(directory: Path) -> tuple[Path, Path]: + key = ec.generate_private_key(ec.SECP256R1()) + name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "127.0.0.1")]) + now = datetime.datetime.now(datetime.timezone.utc) + certificate = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(days=1)) + .not_valid_after(now + datetime.timedelta(days=1)) + .add_extension(x509.SubjectAlternativeName([x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), critical=False) + .sign(key, hashes.SHA256()) + ) + cert_path = directory / "cert.pem" + key_path = directory / "key.pem" + cert_path.write_bytes(certificate.public_bytes(serialization.Encoding.PEM)) + key_path.write_bytes( + key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()) + ) + return cert_path, key_path + + +@contextlib.contextmanager +def _https_serving(directory: Path, body: bytes) -> Generator[str]: + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args: object) -> None: + pass + + cert_path, key_path = _self_signed(directory) + httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(cert_path, key_path) + httpd.socket = context.wrap_socket(httpd.socket, server_side=True) + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + try: + yield f"https://127.0.0.1:{httpd.server_address[1]}/model_prices.json" + finally: + httpd.shutdown() + httpd.server_close() + + +@pytest.mark.parametrize(("ssl_verify", "installed"), [(False, True), (True, False)]) +def test_the_fetch_uses_the_http_settings_provider_calls_use( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ssl_verify: bool, installed: bool +): + monkeypatch.setattr(litellm, "ssl_verify", ssl_verify) + module = types.ModuleType("fake_litellm") + module.model_cost = {} + catalog = Catalog.bound_to(module, "model_cost") + body = Path(litellm.__file__).resolve().parents[1].joinpath("model_prices_and_context_window.json").read_bytes() + + with _https_serving(tmp_path, body) as url: + result = catalog.reload(Remote(url)) + + assert isinstance(result, Installed if installed else Unavailable), getattr(result, "reason", None) + + +def test_a_reload_of_an_unparseable_url_is_unavailable(): + assert isinstance(CATALOG.fetch(Remote("not a url"), timeout=1, attempts=1), Unavailable) diff --git a/tests/unit/litellm_core_utils/test_get_model_cost_map.py b/tests/unit/litellm_core_utils/test_get_model_cost_map.py index 1f7553a7207..16f61813aee 100644 --- a/tests/unit/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/unit/litellm_core_utils/test_get_model_cost_map.py @@ -1,19 +1,30 @@ """ -Tests for model-cost-map loading: the model-count integrity check (which must -count actual model entries, not reserved meta keys) and the extraction of the -``fallback_generalizations`` block out of the raw map. +Tests for model-cost-map loading through the Rust catalog: fetching, retrying, validating, installing into the bound +dict, provenance and the extraction of the ``fallback_generalizations`` block out of the raw map """ -import asyncio, hashlib, importlib, json, litellm +import asyncio +import contextlib +import functools +import hashlib +import importlib +import json +import logging import os import sys +import tempfile import threading +import types +from collections.abc import Generator +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path from typing import Final -import boto3 import pytest -from typing_extensions import NotRequired, ReadOnly, TypedDict +import litellm +from litellm._catalog import CATALOG, Bundled, Catalog, Fetched, File, Installed, Remote, Unavailable from litellm.litellm_core_utils import get_model_cost_map as get_model_cost_map_module from litellm.litellm_core_utils.fallback_generalizations import ( get_fallback_generalization_rules, @@ -21,95 +32,163 @@ from litellm.litellm_core_utils.fallback_generalizations import ( match_routing_generalization, set_fallback_generalizations, ) -from litellm._catalog import CATALOG, Catalog, Fetched, Unavailable from litellm.litellm_core_utils.get_model_cost_map import ( FALLBACK_GENERALIZATIONS_KEY, GetModelCostMap, - get_model_cost_map_provenance, + get_model_cost_map, + install_initial_model_cost_map, + reload_model_cost_map, ) +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.utils import ( + _invalidate_model_cost_lowercase_map, # pyright: ignore[reportPrivateUsage] # tests reset the lowercase index +) +from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome - -def _load_root_cost_map() -> dict: - path = os.path.join(os.path.dirname(__file__), "../../../model_prices_and_context_window.json") - with open(path) as f: - return json.load(f) +_ROOT: Final = Path(__file__).resolve().parents[3] def git_blob_id(body: bytes) -> str: return hashlib.sha1(b"blob %d\0" % len(body) + body, usedforsecurity=False).hexdigest() -def _bundled_blob_id() -> str: - path = os.path.join(os.path.dirname(__file__), "../../../model_prices_and_context_window.json") - with open(path, "rb") as f: - return git_blob_id(f.read()) +@functools.lru_cache(maxsize=1) +def _real_map_bytes() -> bytes: + return (_ROOT / "model_prices_and_context_window.json").read_bytes() -def _parse(document: dict, min_model_count: int = 50, max_shrink_ratio: float = 0.5) -> Fetched | Unavailable: - return Catalog.parse( - json.dumps(document).encode(), - etag=None, - min_model_count=min_model_count, - max_shrink_ratio=max_shrink_ratio, - ) - - -def test_the_revision_is_what_git_hash_object_prints(): - parsed = Catalog.parse( - b'{"gpt-5.4-mini": {"mode": "chat"}}\n', etag=None, min_model_count=0, max_shrink_ratio=0.0 - ) - assert isinstance(parsed, Fetched) - assert parsed.revision == "18b9a8381e13a3b38a2128f184f631f95829e987" - - -def test_the_bundled_revision_is_the_root_json_blob_id(): - assert CATALOG.bundled().revision == _bundled_blob_id() +def _load_root_cost_map() -> dict: + return json.loads(_real_map_bytes()) def _make_models(n: int) -> dict: return {f"model-{i}": {"litellm_provider": "openai", "mode": "chat"} for i in range(n)} -def test_validation_rejects_truly_shrunk_file_even_with_meta_keys(): - """Meta keys must not inflate the model count past the minimum""" - shrunk = _make_models(5) - shrunk["sample_spec"] = {"foo": "bar"} - shrunk[FALLBACK_GENERALIZATIONS_KEY] = {"rules": [{"name": "x"}]} - - assert isinstance(_parse(shrunk, max_shrink_ratio=0.0), Unavailable) +@functools.lru_cache(maxsize=1) +def _bundled_document() -> dict: + return CATALOG.bundled().document() -def test_validation_accepts_healthy_file_with_meta_keys(): - healthy = _make_models(len(CATALOG.bundled().served())) - healthy["sample_spec"] = {"foo": "bar"} - healthy[FALLBACK_GENERALIZATIONS_KEY] = {"rules": []} - - assert isinstance(_parse(healthy), Fetched) +def _with(edit) -> bytes: + document = json.loads(json.dumps(_bundled_document())) + edit(document) + return json.dumps(document).encode() -def test_validation_rejects_significant_shrink_vs_the_bundled_map(): - shrunk = _make_models(len(CATALOG.bundled().served()) // 3) +class _Server: + """A local HTTP server answering each GET with the next queued reply, the last one repeating""" - assert isinstance(_parse(shrunk), Unavailable) + def __init__(self, replies: list[tuple[int, dict[str, str], bytes]]) -> None: + self.replies = replies + self.requests: list[dict[str, str]] = [] + outer = self + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + outer.requests.append(dict(self.headers)) + status, headers, body = outer.replies[min(len(outer.requests), len(outer.replies)) - 1] + self.send_response(status) + for name, value in headers.items(): + self.send_header(name, value) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args: object) -> None: + pass + + self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.url = f"http://127.0.0.1:{self.httpd.server_address[1]}/model_prices.json" -def test_a_loaded_map_installs_its_rules_and_never_serves_the_block(): - previous = list(get_fallback_generalization_rules()) +@contextlib.contextmanager +def _serving(replies: list[tuple[int, dict[str, str], bytes]]) -> Generator[_Server]: + server = _Server(replies) + thread = threading.Thread(target=server.httpd.serve_forever, daemon=True) + thread.start() try: - raw = _make_models(2) - raw[FALLBACK_GENERALIZATIONS_KEY] = { - "rules": [ - { - "name": "rule", - "pattern": r"^widget-", - "model_info": {"litellm_provider": "openai"}, - } - ] - } - fetched = _parse(raw, min_model_count=0, max_shrink_ratio=0.0) + yield server + finally: + server.httpd.shutdown() + server.httpd.server_close() + + +def _bound_catalog() -> tuple[types.ModuleType, Catalog]: + module = types.ModuleType("fake_litellm") + module.model_cost = {} + catalog = Catalog.bound_to(module, "model_cost") + catalog.install(catalog.bundled()) + return module, catalog + + +def _file(tmp_path: Path, body: bytes) -> File: + path = tmp_path / f"map-{git_blob_id(body)[:12]}.json" + path.write_bytes(body) + return File(path) + + +@pytest.fixture(autouse=True) +def _unset_local_cost_map_env(monkeypatch): + monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False) + monkeypatch.delenv("LITELLM_OFFLINE_MODE", raising=False) + + +def _retry_threads(): + return [thread for thread in threading.enumerate() if thread.name == "litellm-model-cost-map-retry"] + + +# --------------------------------------------------------------------------- +# revision, validation and fallback rules +# --------------------------------------------------------------------------- + + +def test_the_revision_is_what_git_hash_object_prints(tmp_path): + body = _real_map_bytes() + fetched = CATALOG.fetch(_file(tmp_path, body), timeout=5, attempts=1) + assert isinstance(fetched, Fetched) + assert fetched.revision == git_blob_id(body) + + +def test_the_bundled_revision_is_the_root_json_blob_id(): + assert CATALOG.bundled().revision == git_blob_id(_real_map_bytes()) + + +@pytest.mark.parametrize( + "body", + [ + pytest.param(b'"not a dict"', id="not_an_object"), + pytest.param(b"{}", id="empty"), + pytest.param(b"[1, 2, 3]", id="list"), + pytest.param(b"null", id="null"), + pytest.param(json.dumps(_make_models(5)).encode(), id="below_minimum"), + pytest.param( + json.dumps({**_make_models(60), "sample_spec": {}, FALLBACK_GENERALIZATIONS_KEY: {"rules": []}}).encode(), + id="shrunk_with_meta_keys", + ), + ], +) +def test_a_map_that_fails_mains_whole_map_checks_is_unavailable(tmp_path, body): + assert isinstance(CATALOG.fetch(_file(tmp_path, body), timeout=5, attempts=1), Unavailable) + + +def test_a_healthy_map_with_meta_keys_is_fetched(tmp_path): + body = _with(lambda document: document.update({"sample_spec": {"foo": "bar"}})) + assert isinstance(CATALOG.fetch(_file(tmp_path, body), timeout=5, attempts=1), Fetched) + + +def test_a_loaded_map_installs_its_rules_and_never_serves_the_block(tmp_path): + previous = list(get_fallback_generalization_rules()) + rules = {"rules": [{"name": "rule", "pattern": r"^widget-", "model_info": {"litellm_provider": "openai"}}]} + try: + fetched = CATALOG.fetch( + _file(tmp_path, _with(lambda document: document.update({FALLBACK_GENERALIZATIONS_KEY: rules}))), + timeout=5, + attempts=1, + ) assert isinstance(fetched, Fetched) - get_model_cost_map_module._note_loaded(get_model_cost_map_module.ModelCostMapReloaded(fetched=fetched)) + get_model_cost_map_module._note_loaded(fetched) # pyright: ignore[reportPrivateUsage] # the load hook has no public entry assert FALLBACK_GENERALIZATIONS_KEY not in fetched.served() assert match_routing_generalization("widget-9") == "openai" @@ -117,13 +196,17 @@ def test_a_loaded_map_installs_its_rules_and_never_serves_the_block(): set_fallback_generalizations(previous) -def test_a_loaded_map_with_no_block_clears_rules(): +def test_a_loaded_map_with_no_block_clears_rules(tmp_path): previous = list(get_fallback_generalization_rules()) try: set_fallback_generalizations([{"name": "stale", "pattern": r"^x", "model_info": {"a": 1}}]) - fetched = _parse(_make_models(2), min_model_count=0, max_shrink_ratio=0.0) + fetched = CATALOG.fetch( + _file(tmp_path, _with(lambda document: document.pop(FALLBACK_GENERALIZATIONS_KEY))), + timeout=5, + attempts=1, + ) assert isinstance(fetched, Fetched) - get_model_cost_map_module._note_loaded(get_model_cost_map_module.ModelCostMapReloaded(fetched=fetched)) + get_model_cost_map_module._note_loaded(fetched) # pyright: ignore[reportPrivateUsage] # the load hook has no public entry assert match_capability_generalizations("x-1") is None finally: set_fallback_generalizations(previous) @@ -218,346 +301,171 @@ def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0(): assert "supports_adaptive_thinking" not in backup[non_adaptive], non_adaptive -def test_get_model_cost_map_stamps_loaded_at(): - """The load time feeds each pod's reload-due decision; a load that does not stamp it - would make manual reload requests race the proxy's startup""" - from datetime import datetime, timezone - - from litellm.litellm_core_utils import get_model_cost_map as module - - client, _calls = _mock_client([httpx.Response(200, content=_real_map_bytes())], client_cls=httpx.Client) - - before = datetime.now(timezone.utc) - module.get_model_cost_map(url="https://example.invalid/cost_map.json", client=client) - loaded_at = module.get_model_cost_map_loaded_at() - - assert loaded_at is not None - assert before <= loaded_at <= datetime.now(timezone.utc) - - # --------------------------------------------------------------------------- -# refetch_model_cost_map: retry/backoff behavior for runtime reloads +# reload_model_cost_map: retry/backoff behavior for runtime reloads # --------------------------------------------------------------------------- -import functools -import random -from datetime import datetime, timezone -import httpx - -from litellm.litellm_core_utils.get_model_cost_map import ( - ModelCostMapReloaded, - ModelCostMapReloadUnavailable, - refetch_model_cost_map, -) - -_URL = "https://example.invalid/model_prices.json" +def test_reload_retries_429_honoring_retry_after(): + _, catalog = _bound_catalog() + with _serving([(429, {"Retry-After": "0"}, b""), (200, {}, _real_map_bytes())]) as server: + result = reload_model_cost_map(Remote(server.url), catalog) + assert isinstance(result, Installed) + assert len(server.requests) == 2 -@functools.lru_cache(maxsize=1) -def _real_map_bytes() -> bytes: - return json.dumps(_load_root_cost_map()).encode() +def test_reload_gives_up_after_max_attempts(): + _, catalog = _bound_catalog() + with _serving([(503, {"Retry-After": "0"}, b"")]) as server: + result = reload_model_cost_map(Remote(server.url), catalog) + assert isinstance(result, Unavailable) + assert result.reason == f"HTTP 503 from {server.url} (after 3 attempts)" + assert len(server.requests) == 3 -class _SleepRecorder: - """Injected in place of asyncio.sleep so tests assert waits without real delay.""" - - def __init__(self): - self.waits = [] - - async def __call__(self, seconds: float) -> None: - self.waits.append(seconds) +@pytest.mark.parametrize("status", [404, 501]) +def test_reload_non_retryable_status_fails_immediately(status): + _, catalog = _bound_catalog() + with _serving([(status, {}, b"")]) as server: + result = reload_model_cost_map(Remote(server.url), catalog) + assert isinstance(result, Unavailable) + assert result.reason == f"HTTP {status} from {server.url}" + assert len(server.requests) == 1 -@pytest.fixture(autouse=True) -def _unset_local_cost_map_env(monkeypatch): - """CI exports LITELLM_LOCAL_MODEL_COST_MAP=True; clear it so fetch behavior is deterministic.""" - monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False) - - -def _mock_client(outcomes, client_cls=httpx.AsyncClient): - """httpx client over a MockTransport serving one outcome per request; an exception instance is raised.""" - calls = {"count": 0} - - def handler(request): - idx = min(calls["count"], len(outcomes) - 1) - calls["count"] += 1 - outcome = outcomes[idx] - if isinstance(outcome, Exception): - raise outcome - return outcome - - return client_cls(transport=httpx.MockTransport(handler)), calls - - -@pytest.mark.asyncio -async def test_refetch_retries_429_honoring_retry_after(): - """Two 429s with Retry-After then success: waits follow the header, not backoff.""" - client, calls = _mock_client( - [ - httpx.Response(429, headers={"Retry-After": "7"}), - httpx.Response(429, headers={"Retry-After": "7"}), - httpx.Response(200, content=_real_map_bytes()), - ] - ) - sleeper = _SleepRecorder() - result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) - assert isinstance(result, ModelCostMapReloaded) - assert len(result.fetched.served()) > 100 - assert calls["count"] == 3 - assert sleeper.waits == [7.0, 7.0] - - -@pytest.mark.asyncio -async def test_refetch_gives_up_after_max_attempts_with_exponential_backoff(): - """All 429 without Retry-After: exponential backoff waits, then a failure value.""" - client, calls = _mock_client([httpx.Response(429)]) - sleeper = _SleepRecorder() - result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) - assert isinstance(result, ModelCostMapReloadUnavailable) - assert "429" in result.reason - assert "after 3 attempts" in result.reason - assert calls["count"] == 3 - assert len(sleeper.waits) == 2 - assert 2.0 <= sleeper.waits[0] < 3.0 - assert 4.0 <= sleeper.waits[1] < 5.0 - - -@pytest.mark.asyncio -async def test_refetch_caps_retry_after_wait(): - """A hostile/huge Retry-After is capped so reloads never sleep unbounded.""" - client, _calls = _mock_client( - [ - httpx.Response(429, headers={"Retry-After": "9999"}), - httpx.Response(200, content=_real_map_bytes()), - ] - ) - sleeper = _SleepRecorder() - result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) - assert isinstance(result, ModelCostMapReloaded) - assert sleeper.waits == [30.0] - - -@pytest.mark.asyncio -async def test_refetch_retries_transport_errors(): - """Connection failures are transient: retried like 5xx, succeeding when the network heals.""" - client, calls = _mock_client( - [ - httpx.ConnectError("connection refused"), - httpx.Response(200, content=_real_map_bytes()), - ] - ) - sleeper = _SleepRecorder() - result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) - assert isinstance(result, ModelCostMapReloaded) - assert calls["count"] == 2 - assert len(sleeper.waits) == 1 - - -@pytest.mark.asyncio -async def test_refetch_non_retryable_status_fails_immediately(): - """A 404 is permanent: one attempt, no sleeps, failure value.""" - client, calls = _mock_client([httpx.Response(404)]) - sleeper = _SleepRecorder() - result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) - assert isinstance(result, ModelCostMapReloadUnavailable) - assert "404" in result.reason - assert calls["count"] == 1 - assert sleeper.waits == [] - - -@pytest.mark.asyncio -async def test_refetch_invalid_json_fails_immediately(): - client, calls = _mock_client([httpx.Response(200, content=b"not json")]) - sleeper = _SleepRecorder() - result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) - assert isinstance(result, ModelCostMapReloadUnavailable) +def test_reload_invalid_json_fails_immediately(): + _, catalog = _bound_catalog() + with _serving([(200, {}, b"not json")]) as server: + result = reload_model_cost_map(Remote(server.url), catalog) + assert isinstance(result, Unavailable) assert "invalid JSON" in result.reason - assert "unusable" in result.reason - assert calls["count"] == 1 - assert sleeper.waits == [] + assert len(server.requests) == 1 -@pytest.mark.asyncio -async def test_refetch_shrunk_map_fails_integrity_not_swapped_in(): - """A drastically shrunk upstream file is rejected instead of being adopted.""" - tiny = json.dumps(_make_models(60)).encode() - client, _calls = _mock_client([httpx.Response(200, content=tiny)]) - result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client) - assert isinstance(result, ModelCostMapReloadUnavailable) +def test_reload_shrunk_map_fails_integrity_and_keeps_the_map(): + module, catalog = _bound_catalog() + before = dict(module.model_cost) + with _serving([(200, {}, json.dumps(_make_models(60)).encode())]) as server: + result = reload_model_cost_map(Remote(server.url), catalog) + assert isinstance(result, Unavailable) assert "below 0.5 of reference count" in result.reason + assert module.model_cost == before -@pytest.mark.asyncio @pytest.mark.parametrize("flag", ["LITELLM_LOCAL_MODEL_COST_MAP", "LITELLM_OFFLINE_MODE"]) -async def test_refetch_respects_local_env_override(monkeypatch, flag): - """LITELLM_LOCAL_MODEL_COST_MAP=True or LITELLM_OFFLINE_MODE=True short-circuits to the bundled backup, zero HTTP.""" +def test_the_default_reload_source_respects_the_local_env_override(monkeypatch, flag): + from litellm._catalog import default_source + monkeypatch.setenv(flag, "True") - - def _fail(request): - raise AssertionError("no HTTP request should be made when local map is forced") - - result = await refetch_model_cost_map( - url=_URL, - sleep=_SleepRecorder(), - rng=random.Random(0), - client=httpx.AsyncClient(transport=httpx.MockTransport(_fail)), - ) - assert isinstance(result, ModelCostMapReloaded) - assert len(result.fetched.served()) > 100 + assert isinstance(default_source(), Bundled) -@pytest.mark.asyncio -async def test_refetch_records_the_blob_id_of_the_bytes_served_and_the_fetch_etag(): +def test_reload_records_the_blob_id_of_the_bytes_served_and_the_fetch_etag(): + _, catalog = _bound_catalog() body = _real_map_bytes() - client, _ = _mock_client([httpx.Response(200, headers={"ETag": 'W/"abc123"'}, content=body)]) - - result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client) - - assert isinstance(result, ModelCostMapReloaded) - assert result.revision == git_blob_id(body) - assert result.etag == 'W/"abc123"' - assert get_model_cost_map_provenance() == {"source_revision": git_blob_id(body), "etag": 'W/"abc123"'} + with _serving([(200, {"ETag": 'W/"abc123"'}, body)]) as server: + reload_model_cost_map(Remote(server.url), catalog) + provenance = catalog.provenance() + assert provenance is not None + assert (provenance.source_revision, provenance.etag, provenance.source, provenance.url) == ( + git_blob_id(body), + 'W/"abc123"', + "remote", + server.url, + ) -@pytest.mark.asyncio -async def test_loaded_catalog_snapshot_follows_the_fetched_map_and_ignores_later_registrations(monkeypatch): - import litellm +def test_loaded_catalog_snapshot_follows_the_fetched_map_and_ignores_later_registrations(monkeypatch): + _, catalog = _bound_catalog() + edited = _with(lambda document: document["gpt-5.4-mini"].update({"max_input_tokens": 777})) + with _serving([(200, {}, edited)]) as server: + reload_model_cost_map(Remote(server.url), catalog) - edited = json.loads(_real_map_bytes()) - edited["gpt-5.4-mini"]["max_input_tokens"] = 777 - client, _ = _mock_client([httpx.Response(200, content=json.dumps(edited).encode())]) - - result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client) - - assert isinstance(result, ModelCostMapReloaded) - monkeypatch.setattr(litellm, "model_cost", result.fetched.served()) + monkeypatch.setattr(litellm, "model_cost", dict(GetModelCostMap.loaded_model_cost_map())) litellm.register_model({"gpt-5.4-mini": {"max_input_tokens": 2048}}, persist_across_reloads=False) assert litellm.model_cost["gpt-5.4-mini"]["max_input_tokens"] == 2048 assert GetModelCostMap.loaded_model_cost_map()["gpt-5.4-mini"]["max_input_tokens"] == 777 -@pytest.mark.asyncio -async def test_loaded_catalog_fields_follow_each_fetched_map(): - added = json.loads(_real_map_bytes()) - added["gpt-5.4-mini"]["supports_litellm_qa_capability"] = True - with_field, _ = _mock_client([httpx.Response(200, content=json.dumps(added).encode())]) - without_field, _ = _mock_client([httpx.Response(200, content=_real_map_bytes())]) - - await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=with_field) +def test_loaded_catalog_fields_follow_each_fetched_map(): + _, catalog = _bound_catalog() + added = _with(lambda document: document["gpt-5.4-mini"].update({"supports_litellm_qa_capability": True})) + with _serving([(200, {}, added)]) as server: + reload_model_cost_map(Remote(server.url), catalog) fields_after_adding: Final = GetModelCostMap.loaded_model_cost_map_fields() - await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=without_field) + with _serving([(200, {}, _real_map_bytes())]) as server: + reload_model_cost_map(Remote(server.url), catalog) assert "supports_litellm_qa_capability" in fields_after_adding assert "supports_litellm_qa_capability" not in GetModelCostMap.loaded_model_cost_map_fields() assert "max_input_tokens" in GetModelCostMap.loaded_model_cost_map_fields() -@pytest.mark.asyncio -async def test_refetch_revision_follows_the_bytes_not_the_url(): - edited = json.loads(_real_map_bytes()) - edited["gpt-5.4-mini"]["input_cost_per_token"] = 0.5 - client, _ = _mock_client( - [httpx.Response(200, content=_real_map_bytes()), httpx.Response(200, content=json.dumps(edited).encode())] +def test_reload_revision_follows_the_bytes_not_the_url(): + _, catalog = _bound_catalog() + edited = _with(lambda document: document["gpt-5.4-mini"].update({"input_cost_per_token": 0.5})) + with _serving([(200, {}, _real_map_bytes()), (200, {}, edited)]) as server: + first = reload_model_cost_map(Remote(server.url), catalog) + first_revision = catalog.provenance().source_revision + second = reload_model_cost_map(Remote(server.url), catalog) + assert isinstance(first, Installed) and isinstance(second, Installed) + assert first_revision != catalog.provenance().source_revision == git_blob_id(edited) + + +def test_a_bundled_reload_reports_the_bundled_blob_id_without_an_etag(monkeypatch): + _, catalog = _bound_catalog() + with _serving([(200, {"ETag": 'W/"remote"'}, _real_map_bytes())]) as server: + reload_model_cost_map(Remote(server.url), catalog) + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + + reload_model_cost_map(Bundled(), catalog) + + provenance = catalog.provenance() + assert (provenance.source_revision, provenance.etag, provenance.source, provenance.is_env_forced) == ( + git_blob_id(_real_map_bytes()), + None, + "local", + True, ) - first = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client) - second = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client) - assert isinstance(first, ModelCostMapReloaded) and isinstance(second, ModelCostMapReloaded) - assert first.revision != second.revision - assert get_model_cost_map_provenance()["source_revision"] == second.revision +def test_reload_stamps_loaded_at_when_the_load_starts(): + _, catalog = _bound_catalog() + before = datetime.now(timezone.utc) + with _serving([(200, {}, _real_map_bytes())]) as server: + reload_model_cost_map(Remote(server.url), catalog) + loaded_at = catalog.provenance().loaded_at + assert before <= loaded_at <= datetime.now(timezone.utc) -@pytest.mark.asyncio -async def test_refetch_local_override_reports_the_bundled_blob_id_without_an_etag(monkeypatch): - remote, _ = _mock_client([httpx.Response(200, headers={"ETag": 'W/"remote"'}, content=_real_map_bytes())]) - await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=remote) - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - - result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0)) - - assert isinstance(result, ModelCostMapReloaded) - assert result.revision == _bundled_blob_id() - assert get_model_cost_map_provenance() == {"source_revision": _bundled_blob_id(), "etag": None} +def test_a_failed_reload_keeps_the_provenance_of_the_map_in_use(): + _, catalog = _bound_catalog() + kept = catalog.provenance() + with _serving([(404, {}, b"")]) as server: + reload_model_cost_map(Remote(server.url), catalog) + assert catalog.provenance().source_revision == kept.source_revision + assert catalog.provenance().loaded_at == kept.loaded_at -@pytest.mark.asyncio -async def test_refetch_stamps_loaded_at_on_remote_and_local_reloads(monkeypatch): - from litellm.litellm_core_utils import get_model_cost_map as module +def test_reload_reads_a_file_url(tmp_path): + module, catalog = _bound_catalog() + path = tmp_path / "map.json" + path.write_bytes(_with(lambda document: document.update({"only-in-the-file": {"litellm_provider": "openai"}}))) - client, _ = _mock_client([httpx.Response(200, content=_real_map_bytes())]) - before_remote = datetime.now(timezone.utc) - await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client) - remote_loaded_at = module.get_model_cost_map_loaded_at() - assert remote_loaded_at is not None - assert before_remote <= remote_loaded_at <= datetime.now(timezone.utc) + result = reload_model_cost_map(Remote(path.as_uri()), catalog) - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - before_local = datetime.now(timezone.utc) - await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0)) - local_loaded_at = module.get_model_cost_map_loaded_at() - assert local_loaded_at is not None - assert before_local <= local_loaded_at <= datetime.now(timezone.utc) + assert isinstance(result, Installed) + assert "only-in-the-file" in module.model_cost + assert catalog.provenance().source == "remote" # --------------------------------------------------------------------------- -# get_model_cost_map: the boot-time load retries transient failures like a reload does +# install_initial_model_cost_map: the import-time load # --------------------------------------------------------------------------- -from litellm.litellm_core_utils.get_model_cost_map import ( - get_model_cost_map, - get_model_cost_map_source_info, -) -from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.utils import _invalidate_model_cost_lowercase_map -from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome -from unittest.mock import MagicMock, patch - - -class _FakeS3Body: - def __init__(self, content: bytes) -> None: - self.content: Final = content - - def read(self) -> bytes: - return self.content - - -class _FakeS3GetObjectResponse(TypedDict): - Body: ReadOnly[_FakeS3Body] - ETag: NotRequired[ReadOnly[str]] - - -class _FakeS3Client: - def __init__(self, content: bytes) -> None: - self.content: Final = content - self.calls: list[tuple[str, str]] = [] - - def get_object(self, *, Bucket: str, Key: str) -> _FakeS3GetObjectResponse: - self.calls.append((Bucket, Key)) - return {"Body": _FakeS3Body(self.content), "ETag": '"fake-etag"'} - - -def test_get_model_cost_map_fetches_s3_catalog(monkeypatch: pytest.MonkeyPatch) -> None: - from litellm.litellm_core_utils import get_model_cost_map as module - - source_info = module._cost_map_source_info - for name in ("source", "url", "is_env_forced", "fallback_reason", "loaded_at", "source_revision", "etag"): - monkeypatch.setattr(source_info, name, getattr(source_info, name)) - - s3_content: Final = _make_models(len(CATALOG.bundled().served())) - s3_client: Final = _FakeS3Client(json.dumps(s3_content).encode()) - monkeypatch.setattr(boto3, "client", lambda *args, **kwargs: s3_client) - - cost_map: Final = get_model_cost_map("s3://my-bucket/prices/model_prices.json") - - assert cost_map == s3_content - assert s3_client.calls == [("my-bucket", "prices/model_prices.json")] - assert get_model_cost_map_source_info()["source"] == "remote" - class _SyncSleepRecorder: - """Injected in place of time.sleep so the boot path's waits are asserted without delay.""" + """Injected as the retry thread's first wait so the boot path is asserted without delay""" def __init__(self, block=False): self.waits = [] @@ -572,34 +480,22 @@ class _SyncSleepRecorder: self.waits.append(seconds) -def _retry_threads(): - return [thread for thread in threading.enumerate() if thread.name == "litellm-model-cost-map-retry"] - - def test_boot_load_success_does_not_start_background_retry(): - client, calls = _mock_client([httpx.Response(200, content=_real_map_bytes())], client_cls=httpx.Client) + module, catalog = _bound_catalog() sleeper = _SyncSleepRecorder() - - cost_map = get_model_cost_map( - url=_URL, - sleep=sleeper, - rng=random.Random(0), - client=client, - ) - assert calls["count"] == 1 + with _serving([(200, {}, _real_map_bytes())]) as server: + install_initial_model_cost_map(server.url, catalog, sleeper) + assert len(server.requests) == 1 assert sleeper.waits == [] assert _retry_threads() == [] - assert cost_map.keys() >= _load_root_cost_map().keys() - {"sample_spec", FALLBACK_GENERALIZATIONS_KEY} - assert get_model_cost_map_source_info()["source"] == "remote" + assert module.model_cost.keys() >= _load_root_cost_map().keys() - {"sample_spec", FALLBACK_GENERALIZATIONS_KEY} + assert catalog.provenance().source == "remote" -def test_boot_load_transient_failure_returns_local_then_background_retry_adopts_remote(monkeypatch): - import litellm +def test_boot_load_transient_failure_installs_local_then_background_retry_installs_remote(monkeypatch): from litellm import utils as litellm_utils - from litellm.litellm_core_utils import get_model_cost_map as module - original_model_cost = litellm.model_cost - monkeypatch.setattr(litellm, "model_cost", dict(original_model_cost)) + monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost)) for name, provider_models in tuple(vars(litellm).items()): if name.endswith("_models") and isinstance(provider_models, set): monkeypatch.setattr(litellm, name, set(provider_models)) @@ -609,164 +505,113 @@ def test_boot_load_transient_failure_returns_local_then_background_retry_adopts_ "_runtime_registered_model_cost", dict(litellm_utils._runtime_registered_model_cost), ) - source_info = module._cost_map_source_info - for name in ("source", "url", "is_env_forced", "fallback_reason", "loaded_at", "source_revision", "etag"): - monkeypatch.setattr(source_info, name, getattr(source_info, name)) - + catalog = Catalog.bound_to(litellm, "model_cost") held = litellm.model_cost - remote_map = _load_root_cost_map() - remote_map["claude-remote-only-test"] = {"litellm_provider": "anthropic", "mode": "chat"} - remote_map["gpt-4o"]["input_cost_per_token"] = 0.125 - client, calls = _mock_client( - [ - httpx.ConnectError("connection refused"), - httpx.Response(200, content=json.dumps(remote_map).encode()), - ], - client_cls=httpx.Client, + remote = _with( + lambda document: ( + document.update({"claude-remote-only-test": {"litellm_provider": "anthropic", "mode": "chat"}}), + document["gpt-4o"].update({"input_cost_per_token": 0.125}), + ) ) sleeper = _SyncSleepRecorder(block=True) litellm.register_model({"my-runtime-model": {"litellm_provider": "custom", "max_input_tokens": 4321}}) - cost_map = get_model_cost_map( - url=_URL, - max_attempts=3, - sleep=sleeper, - rng=random.Random(0), - client=client, - ) - - assert calls["count"] == 1 - assert sleeper.waits == [] - assert sleeper.started.wait(timeout=10) - threads = _retry_threads() - try: - assert len(threads) == 1 - assert "claude-remote-only-test" not in cost_map - assert cost_map.keys() == CATALOG.bundled().served().keys() - source = get_model_cost_map_source_info() - assert source["source"] == "local" - assert source["fallback_reason"].startswith("Remote fetch failed:") - sleeper.release.set() - for thread in threads: - thread.join(timeout=10) - assert all(not thread.is_alive() for thread in threads) - assert sleeper.waits and 2.0 <= sleeper.waits[0] < 3.0 - assert calls["count"] == 2 - assert "claude-remote-only-test" in litellm.model_cost - assert litellm.model_cost is held - assert held["gpt-4o"]["input_cost_per_token"] == 0.125 - assert "claude-remote-only-test" in litellm.anthropic_models - assert "my-runtime-model" in litellm.model_cost - source = get_model_cost_map_source_info() - assert source["source"] == "remote" - assert source["fallback_reason"] is None - finally: - sleeper.release.set() - for thread in _retry_threads(): - thread.join(timeout=10) + with _serving([(503, {"Retry-After": "0"}, b""), (200, {}, remote)]) as server: + install_initial_model_cost_map(server.url, catalog, sleeper) + assert sleeper.started.wait(timeout=10) + threads = _retry_threads() + try: + assert len(server.requests) == 1 + assert len(threads) == 1 + assert "claude-remote-only-test" not in litellm.model_cost + provenance = catalog.provenance() + assert provenance.source == "local" + assert provenance.fallback_reason.startswith("Remote fetch failed:") + sleeper.release.set() + for thread in threads: + thread.join(timeout=10) + assert all(not thread.is_alive() for thread in threads) + assert sleeper.waits == [0.0] + assert len(server.requests) == 2 + assert "claude-remote-only-test" in litellm.model_cost + assert litellm.model_cost is held + assert held["gpt-4o"]["input_cost_per_token"] == 0.125 + assert "claude-remote-only-test" in litellm.anthropic_models + assert "my-runtime-model" in litellm.model_cost + assert catalog.provenance().source == "remote" + assert catalog.provenance().fallback_reason is None + finally: + sleeper.release.set() + for thread in _retry_threads(): + thread.join(timeout=10) def test_boot_load_does_not_retry_non_retryable_failure(): - client, calls = _mock_client([httpx.Response(404)], client_cls=httpx.Client) + _, catalog = _bound_catalog() sleeper = _SyncSleepRecorder() - - get_model_cost_map( - url=_URL, - sleep=sleeper, - rng=random.Random(0), - client=client, - ) - assert calls["count"] == 1 + with _serving([(404, {}, b"")]) as server: + install_initial_model_cost_map(server.url, catalog, sleeper) + assert len(server.requests) == 1 assert sleeper.waits == [] assert _retry_threads() == [] - source = get_model_cost_map_source_info() - assert source["source"] == "local" - assert source["fallback_reason"] is not None + provenance = catalog.provenance() + assert provenance.source == "local" + assert provenance.fallback_reason == f"Remote fetch failed: HTTP 404 from {server.url}" + assert provenance.url == server.url -def test_boot_load_falls_back_when_urlsplit_rejects_the_url(): - client, calls = _mock_client([httpx.Response(200, content=_real_map_bytes())], client_cls=httpx.Client) - sleeper = _SyncSleepRecorder() - - cost_map = get_model_cost_map( - url="http://[bad/model_prices.json", - sleep=sleeper, - rng=random.Random(0), - client=client, - ) - assert calls["count"] == 0 - assert sleeper.waits == [] +def test_boot_load_falls_back_when_the_url_does_not_parse(): + module, catalog = _bound_catalog() + install_initial_model_cost_map("http://[bad/model_prices.json", catalog, _SyncSleepRecorder()) assert _retry_threads() == [] - assert cost_map.keys() >= _load_root_cost_map().keys() - {"sample_spec", FALLBACK_GENERALIZATIONS_KEY} - source = get_model_cost_map_source_info() - assert source["source"] == "local" - assert "invalid URL" in (source["fallback_reason"] or "") + assert module.model_cost.keys() >= _load_root_cost_map().keys() - {"sample_spec", FALLBACK_GENERALIZATIONS_KEY} + assert "invalid URL" in (catalog.provenance().fallback_reason or "") @pytest.mark.parametrize("flag", ["LITELLM_LOCAL_MODEL_COST_MAP", "LITELLM_OFFLINE_MODE"]) def test_boot_load_respects_local_env_override(monkeypatch, flag): - """LITELLM_LOCAL_MODEL_COST_MAP=True or LITELLM_OFFLINE_MODE=True still short-circuits to the backup with zero HTTP.""" monkeypatch.setenv(flag, "True") - - def _fail(request): - raise AssertionError("no HTTP request should be made when local map is forced") - - cost_map = get_model_cost_map( - url=_URL, - sleep=_SyncSleepRecorder(), - client=httpx.Client(transport=httpx.MockTransport(_fail)), - ) - assert len(cost_map) > 100 - assert get_model_cost_map_source_info()["is_env_forced"] is True + module, catalog = _bound_catalog() + with _serving([(200, {}, _real_map_bytes())]) as server: + install_initial_model_cost_map(server.url, catalog, _SyncSleepRecorder()) + assert server.requests == [] + assert len(module.model_cost) > 100 + assert catalog.provenance().is_env_forced is True def test_boot_load_records_the_blob_id_of_the_bytes_served_and_the_fetch_etag(): + _, catalog = _bound_catalog() body = _real_map_bytes() - client, _ = _mock_client([httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=body)], client_cls=httpx.Client) - - get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=client) - - source = get_model_cost_map_source_info() - assert source["source"] == "remote" - assert source["etag"] == 'W/"boot"' - assert source["source_revision"] == git_blob_id(body) - assert source["loaded_at"] is not None + with _serving([(200, {"ETag": 'W/"boot"'}, body)]) as server: + install_initial_model_cost_map(server.url, catalog, _SyncSleepRecorder()) + provenance = catalog.provenance() + assert (provenance.source, provenance.etag, provenance.source_revision) == ("remote", 'W/"boot"', git_blob_id(body)) -def test_boot_load_fallback_to_the_backup_reports_its_blob_id_and_drops_the_remote_etag(): - remote, _ = _mock_client( - [httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=_real_map_bytes())], client_cls=httpx.Client +def test_boot_load_fallback_to_the_bundled_map_reports_its_blob_id_and_drops_the_remote_etag(): + _, catalog = _bound_catalog() + with _serving([(200, {"ETag": 'W/"boot"'}, _real_map_bytes())]) as server: + install_initial_model_cost_map(server.url, catalog, _SyncSleepRecorder()) + with _serving([(404, {}, b"")]) as server: + install_initial_model_cost_map(server.url, catalog, _SyncSleepRecorder()) + provenance = catalog.provenance() + assert (provenance.source, provenance.etag, provenance.source_revision) == ( + "local", + None, + git_blob_id(_real_map_bytes()), ) - get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=remote) - failing, _ = _mock_client([httpx.Response(404)], client_cls=httpx.Client) - - get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=failing) - - source = get_model_cost_map_source_info() - assert source["source"] == "local" - assert source["etag"] is None - assert source["source_revision"] == _bundled_blob_id() -def test_boot_load_that_fails_the_integrity_check_reports_the_backup_not_the_rejected_fetch(): - remote, _ = _mock_client( - [httpx.Response(200, headers={"ETag": 'W/"boot"'}, content=_real_map_bytes())], client_cls=httpx.Client - ) - get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=remote) +def test_boot_load_that_fails_the_integrity_check_reports_the_bundled_map_not_the_rejected_fetch(): + _, catalog = _bound_catalog() shrunk_body = b'{"gpt-5.4-mini": {"mode": "chat", "input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06}}' - shrunk, _ = _mock_client( - [httpx.Response(200, headers={"ETag": 'W/"shrunk"'}, content=shrunk_body)], client_cls=httpx.Client - ) - - get_model_cost_map(url=_URL, sleep=_SyncSleepRecorder(), rng=random.Random(0), client=shrunk) - - source = get_model_cost_map_source_info() - assert source["source"] == "local" - assert source["fallback_reason"].startswith("Remote fetch failed:") - assert "below minimum" in source["fallback_reason"] - assert source["etag"] is None - assert source["source_revision"] == _bundled_blob_id() - assert source["source_revision"] != git_blob_id(shrunk_body) + with _serving([(200, {"ETag": 'W/"shrunk"'}, shrunk_body)]) as server: + install_initial_model_cost_map(server.url, catalog, _SyncSleepRecorder()) + provenance = catalog.provenance() + assert provenance.source == "local" + assert "below minimum" in (provenance.fallback_reason or "") + assert provenance.etag is None + assert provenance.source_revision == git_blob_id(_real_map_bytes()) @pytest.mark.parametrize( @@ -781,18 +626,25 @@ def test_boot_load_skips_remote_fetch_for_cli_processes( monkeypatch: pytest.MonkeyPatch, argv0: str, request_count: int ) -> None: monkeypatch.setattr(sys, "argv", [argv0, "--version"]) - monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False) - client, calls = _mock_client([httpx.Response(200, content=_real_map_bytes())], client_cls=httpx.Client) + _, catalog = _bound_catalog() + with _serving([(200, {}, _real_map_bytes())]) as server: + install_initial_model_cost_map(server.url, catalog, _SyncSleepRecorder()) + assert len(server.requests) == request_count + assert catalog.provenance().source == ("local" if request_count == 0 else "remote") - cost_map = get_model_cost_map(url=_URL, client=client) - assert calls["count"] == request_count - assert cost_map - source = get_model_cost_map_source_info() - if request_count == 0: - assert source["source"] == "local" - else: - assert source["source"] == "remote" +def test_get_model_cost_map_returns_the_remote_rows_without_installing_them(): + module, catalog = _bound_catalog() + remote = _with(lambda document: document.update({"only-remote": {"litellm_provider": "openai"}})) + with _serving([(200, {}, remote)]) as server: + cost_map = get_model_cost_map(server.url) + assert "only-remote" in cost_map + assert "only-remote" not in litellm.model_cost + + +def test_get_model_cost_map_warns_that_an_injected_client_is_ignored(): + with _serving([(200, {}, _real_map_bytes())]) as server, pytest.warns(DeprecationWarning): + get_model_cost_map(server.url, client=object()) @pytest.fixture() @@ -801,6 +653,7 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="session") def event_loop(): try: @@ -810,6 +663,7 @@ def event_loop(): yield loop loop.close() + @pytest.fixture(scope="function") def setup_and_teardown(event_loop): import litellm @@ -843,6 +697,7 @@ def setup_and_teardown(event_loop): if pending: event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) + _SCALAR_DEFAULTS = { "num_retries": getattr(litellm, "num_retries", None), "set_verbose": getattr(litellm, "set_verbose", False), @@ -858,99 +713,34 @@ _SCALAR_DEFAULTS = { } -@pytest.mark.parametrize( - ("body", "min_model_count", "max_shrink_ratio", "usable"), - [ - pytest.param(b'"not a dict"', 10, 0.5, False, id="not_an_object"), - pytest.param(b"{}", 10, 0.5, False, id="empty"), - pytest.param(b"[1, 2, 3]", 10, 0.5, False, id="list"), - pytest.param(b"null", 10, 0.5, False, id="null"), - pytest.param(json.dumps(_make_models(5)).encode(), 10, 0.0, False, id="below_minimum"), - pytest.param(json.dumps(_make_models(60)).encode(), 10, 0.5, False, id="shrunk"), - pytest.param(json.dumps(_make_models(60)).encode(), 10, 0.0, True, id="no_ratio_floor"), - pytest.param(json.dumps(_make_models(60)).encode(), 61, 0.0, False, id="one_below_minimum"), - ], -) -def test_parse_applies_mains_whole_map_checks(body, min_model_count, max_shrink_ratio, usable): - parsed = Catalog.parse(body, etag=None, min_model_count=min_model_count, max_shrink_ratio=max_shrink_ratio) - assert isinstance(parsed, Fetched) is usable - - -def _fetch_with_single_outcome(monkeypatch, outcome): - from litellm.litellm_core_utils import get_model_cost_map as module - - monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False) - source_info = module._cost_map_source_info - for name in ("source", "url", "is_env_forced", "fallback_reason", "loaded_at", "source_revision", "etag"): - monkeypatch.setattr(source_info, name, getattr(source_info, name)) - client, calls = _mock_client([outcome], client_cls=httpx.Client) - result = get_model_cost_map(url=_URL, max_attempts=1, client=client) - return result, calls["count"], get_model_cost_map_source_info() - - def _backup_keys(): return CATALOG.bundled().served().keys() -@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") -class TestGetModelCostMapFallback: - """Tests for get_model_cost_map fallback behavior with bad upstream.""" +@pytest.mark.parametrize( + "reply", + [ + pytest.param((200, {}, b"not json"), id="invalid_json"), + pytest.param((200, {}, b"{}"), id="empty"), + pytest.param((200, {}, b"[]"), id="not_an_object"), + pytest.param((200, {}, json.dumps(_make_models(11)).encode()), id="shrinks_dramatically"), + pytest.param((404, {}, b""), id="not_found"), + ], +) +def test_get_model_cost_map_falls_back_to_the_bundled_map(reply): + with _serving([reply]) as server: + result = get_model_cost_map(server.url, max_attempts=1) + assert len(server.requests) == 1 + assert result.keys() == _backup_keys() - def test_should_fallback_to_backup_on_invalid_json(self, monkeypatch): - """When upstream returns invalid JSON, should fall back to local backup.""" - result, fetches, source = _fetch_with_single_outcome(monkeypatch, httpx.Response(200, content=b"not json")) - assert fetches == 1 - assert result.keys() == _backup_keys() - assert source["source"] == "local" - assert source["fallback_reason"].startswith("Remote fetch failed") +def test_get_model_cost_map_uses_the_local_map_when_the_env_var_is_set(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + with _serving([(200, {}, b"{}")]) as server: + result = get_model_cost_map(server.url) + assert server.requests == [] + assert result.keys() == _backup_keys() - def test_should_fallback_to_backup_on_network_error(self, monkeypatch): - """When upstream is unreachable, should fall back to local backup.""" - result, fetches, source = _fetch_with_single_outcome(monkeypatch, httpx.ConnectError("Connection refused")) - - assert fetches == 1 - assert result.keys() == _backup_keys() - assert source["source"] == "local" - assert source["fallback_reason"].startswith("Remote fetch failed") - - def test_should_fallback_when_fetched_map_is_empty(self, monkeypatch): - """When upstream returns valid JSON but empty dict, should fall back.""" - result, fetches, source = _fetch_with_single_outcome(monkeypatch, httpx.Response(200, content=b"{}")) - - assert fetches == 1 - assert result.keys() == _backup_keys() - assert source["source"] == "local" - - def test_should_fallback_when_fetched_json_is_not_object(self, monkeypatch): - result, fetches, source = _fetch_with_single_outcome(monkeypatch, httpx.Response(200, content=b"[]")) - - assert fetches == 1 - assert result.keys() == _backup_keys() - assert source["source"] == "local" - assert source["fallback_reason"].startswith(f"Remote fetch failed: model cost map from {_URL} is unusable") - - def test_should_fallback_when_fetched_map_shrinks_dramatically(self, monkeypatch): - """When upstream returns far fewer models than backup, should fall back.""" - tiny_map = {f"model-{i}": {"litellm_provider": "test"} for i in range(11)} - result, fetches, source = _fetch_with_single_outcome( - monkeypatch, httpx.Response(200, content=json.dumps(tiny_map).encode()) - ) - - assert fetches == 1 - assert result.keys() == _backup_keys() - assert source["source"] == "local" - assert "below minimum" in source["fallback_reason"] - - def test_should_use_local_map_when_env_var_set(self): - """LITELLM_LOCAL_MODEL_COST_MAP=True should skip remote fetch entirely.""" - with patch.dict(os.environ, {"LITELLM_LOCAL_MODEL_COST_MAP": "True"}): - with patch("httpx.get") as mock_get: - result = get_model_cost_map("https://fake-url.com/model_prices.json") - mock_get.assert_not_called() - - assert isinstance(result, dict) - assert len(result) > 0 @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestBackupModelCostMapExists: @@ -967,6 +757,7 @@ class TestBackupModelCostMapExists: backup = CATALOG.bundled().served() assert len(backup) > 100, f"Backup has only {len(backup)} models, expected > 100" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestBadHostedModelCostMap: """ @@ -983,12 +774,8 @@ class TestBadHostedModelCostMap: If the hosted map is bad, get_model_cost_map falls back to the local backup. get_model_info should still work for models in the backup. """ - mock_response = MagicMock() - mock_response.raise_for_status = MagicMock() - mock_response.json.side_effect = json.JSONDecodeError("bad json", "", 0) - - with patch("httpx.get", return_value=mock_response): - fallback_map = get_model_cost_map("https://fake-url.com/bad.json") + with _serving([(200, {}, b"not json")]) as server: + fallback_map = get_model_cost_map(server.url) original = litellm.model_cost litellm.model_cost = fallback_map @@ -1009,12 +796,8 @@ class TestBadHostedModelCostMap: needing API credentials. """ # Simulate bad hosted map → fallback to backup - mock_http = MagicMock() - mock_http.raise_for_status = MagicMock() - mock_http.json.side_effect = json.JSONDecodeError("bad json", "", 0) - - with patch("httpx.get", return_value=mock_http): - fallback_map = get_model_cost_map("https://fake-url.com/bad.json") + with _serving([(200, {}, b"not json")]) as server: + fallback_map = get_model_cost_map(server.url) original = litellm.model_cost litellm.model_cost = fallback_map @@ -1031,12 +814,14 @@ class TestBadHostedModelCostMap: finally: litellm.model_cost = original + @pytest.fixture() def _vcr_outcome_gate_local_testing(request, vcr): install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="function") def isolate_litellm_state(): """ @@ -1089,6 +874,7 @@ def isolate_litellm_state(): setattr(litellm, attr, original_value) _invalidate_model_cost_lowercase_map() + _SCALAR_DEFAULTS_local_testing = { "num_retries": getattr(litellm, "num_retries", None), "num_retries_per_request": getattr(litellm, "num_retries_per_request", None), @@ -1109,6 +895,7 @@ _SCALAR_DEFAULTS_local_testing = { "api_key": getattr(litellm, "api_key", None), } + @pytest.fixture(scope="module") def setup_and_teardown_local_testing(): """ @@ -1131,6 +918,7 @@ def setup_and_teardown_local_testing(): litellm.in_memory_llm_clients_cache.flush_cache() yield + @pytest.mark.usefixtures( "_vcr_outcome_gate_local_testing", "isolate_litellm_state", @@ -1143,34 +931,17 @@ def test_get_model_cost_map(): pytest.fail(f"An exception occurred: {e}") - # --------------------------------------------------------------------------- # Install through the Rust catalog (decisions 1, 15, 23 and 25) # --------------------------------------------------------------------------- -import logging -import types - - -@functools.lru_cache(maxsize=1) -def _bundled_document() -> dict: - return CATALOG.bundled().document() - def _remote(edit) -> Fetched: - document = json.loads(json.dumps(_bundled_document())) - edit(document) - parsed = Catalog.parse(json.dumps(document).encode(), etag=None, min_model_count=50, max_shrink_ratio=0.5) - assert isinstance(parsed, Fetched) - return parsed - - -def _bound_catalog() -> tuple[types.ModuleType, Catalog]: - module = types.ModuleType("fake_litellm") - module.model_cost = {} - catalog = Catalog.bound_to(module, "model_cost") - catalog.install(catalog.bundled()) - return module, catalog + path = Path(tempfile.mkdtemp()) / "map.json" + path.write_bytes(_with(edit)) + fetched = CATALOG.fetch(File(path), timeout=5, attempts=1) + assert isinstance(fetched, Fetched) + return fetched def test_sample_spec_is_not_served_but_stays_in_the_bundled_document(): @@ -1291,7 +1062,7 @@ def test_the_local_map_is_the_embedded_one_not_the_backup_file(monkeypatch, tmp_ monkeypatch.setattr(get_model_cost_map_module, "files", lambda _package: tmp_path, raising=False) monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - cost_map = get_model_cost_map(url=_URL) + cost_map = get_model_cost_map(url="http://127.0.0.1:9/never-fetched.json") assert "only-in-the-backup" not in cost_map assert cost_map.keys() == CATALOG.bundled().served().keys() diff --git a/tests/unit/proxy/common_utils/test_config_sync_pubsub.py b/tests/unit/proxy/common_utils/test_config_sync_pubsub.py index 878e42c52b3..b809c1ce668 100644 --- a/tests/unit/proxy/common_utils/test_config_sync_pubsub.py +++ b/tests/unit/proxy/common_utils/test_config_sync_pubsub.py @@ -64,13 +64,22 @@ _EXPECTED_RESYNC_APPLIED_CONFIG_PARAM_NAMES = frozenset( _STARTUP_ONLY_CONFIG_PARAM_NAMES = ("environment_variables",) -def _reloaded(cost_map: dict) -> "ModelCostMapReloaded": - from litellm._catalog import Catalog, Fetched - from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded +from litellm.litellm_core_utils.get_model_cost_map import reload_model_cost_map as _REAL_RELOAD - parsed = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0) - assert isinstance(parsed, Fetched) - return ModelCostMapReloaded(fetched=parsed) + +def _reloading(cost_map: dict): + """A stand-in reload that installs ``cost_map`` over the bundled map, read from a file like a fetched map""" + import tempfile + from pathlib import Path + + from litellm._catalog import CATALOG, File + + def reload(source, catalog=CATALOG): + path = Path(tempfile.mkdtemp()) / "model_prices.json" + path.write_text(json.dumps({**CATALOG.bundled().document(), **cost_map})) + return _REAL_RELOAD(File(path), catalog) + + return reload class _RecordingRedisClient(Redis): @@ -864,12 +873,11 @@ async def test_model_cost_map_reload_does_not_publish_config_change() -> None: original_model_cost = litellm.model_cost.copy() _set_redis_usage_cache(_FakeRedisCache(client)) try: - from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded with patch( - "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", - new=AsyncMock( - return_value=_reloaded({"gpt-5.2": {"input_cost_per_token": 0.001}}) + "litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", + new=MagicMock( + side_effect=_reloading({"gpt-5.2": {"input_cost_per_token": 0.001}}) ), ): await ProxyConfig()._check_and_reload_model_cost_map(prisma_client=prisma_client) diff --git a/tests/unit/proxy/proxy_server/test_routes_model_cost_map.py b/tests/unit/proxy/proxy_server/test_routes_model_cost_map.py index 4f4dac169cb..404d242b593 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_cost_map.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_cost_map.py @@ -32,13 +32,22 @@ _ROOT_COST_MAP = Path(__file__).resolve().parents[4] / "model_prices_and_context # --------------------------------------------------------------------------- -def _reloaded(cost_map: dict) -> "ModelCostMapReloaded": - from litellm._catalog import Catalog, Fetched - from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded +from litellm.litellm_core_utils.get_model_cost_map import reload_model_cost_map as _REAL_RELOAD - parsed = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0) - assert isinstance(parsed, Fetched) - return ModelCostMapReloaded(fetched=parsed) + +def _reloading(cost_map: dict): + """A stand-in reload that installs ``cost_map`` over the bundled map, read from a file like a fetched map""" + import tempfile + from pathlib import Path + + from litellm._catalog import CATALOG, File + + def reload(source, catalog=CATALOG): + path = Path(tempfile.mkdtemp()) / "model_prices.json" + path.write_text(json.dumps({**CATALOG.bundled().document(), **cost_map})) + return _REAL_RELOAD(File(path), catalog) + + return reload def _attach_litellm_config(mock_prisma): @@ -73,8 +82,8 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma): fake_cost_map = {"gpt-4": {"input_cost": 0.03}, "gpt-3.5": {"input_cost": 0.002}} monkeypatch.setattr( - "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", - AsyncMock(return_value=_reloaded(fake_cost_map)), + "litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", + MagicMock(side_effect=_reloading(fake_cost_map)), ) monkeypatch.setattr("litellm.add_known_models", lambda model_cost_map=None: None) monkeypatch.setattr("litellm.model_cost", {}, raising=False) @@ -110,7 +119,6 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma): def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every_status_surface( client, auth_as, monkeypatch, mock_prisma ): - import httpx import litellm import hashlib @@ -125,11 +133,23 @@ def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every "source_revision": hashlib.sha1(b"blob %d\0" % len(body) + body, usedforsecurity=False).hexdigest(), "etag": _SERVED_ETAG, } - served = httpx.Response(200, headers={"ETag": _SERVED_ETAG}, content=body) - monkeypatch.setattr( - "litellm.litellm_core_utils.get_model_cost_map._default_reload_client", - lambda: httpx.AsyncClient(transport=httpx.MockTransport(lambda request: served)), - ) + import threading + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + self.send_response(200) + self.send_header("ETag", _SERVED_ETAG) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args: object) -> None: + pass + + httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + monkeypatch.setattr(litellm, "model_cost_map_url", f"http://127.0.0.1:{httpd.server_address[1]}/map.json") monkeypatch.setattr("litellm.add_known_models", lambda model_cost_map=None: None) monkeypatch.setattr("litellm.model_cost", {}, raising=False) @@ -155,6 +175,8 @@ def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every assert {key: status_response.json()[key] for key in expected} == expected assert public_response.status_code == 200 assert "gpt-4o" in public_response.json() + httpd.shutdown() + httpd.server_close() assert reload_body["models_count"] == len(litellm.model_cost) @@ -166,9 +188,7 @@ def test_reload_model_cost_map_fetch_failure_502_keeps_map( Regression: the endpoint used to report success after silently swapping in the stale packaged backup. """ - from litellm.litellm_core_utils.get_model_cost_map import ( - ModelCostMapReloadUnavailable, - ) + from litellm._catalog import Unavailable from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -178,9 +198,9 @@ def test_reload_model_cost_map_fetch_failure_502_keeps_map( sentinel_map = {"existing-model": {"input_cost": 0.01}} monkeypatch.setattr("litellm.model_cost", sentinel_map, raising=False) monkeypatch.setattr( - "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", - AsyncMock( - return_value=ModelCostMapReloadUnavailable( + "litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", + MagicMock( + return_value=Unavailable( reason="HTTP 429 from upstream (after 3 attempts)" ) ), @@ -415,7 +435,6 @@ def test_get_model_cost_map_reload_status_no_config_not_scheduled( client, auth_as, monkeypatch, mock_prisma ): """A row left behind by a manual reload (interval_hours=None) → not scheduled.""" - from datetime import datetime, timezone from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 947a773e4ef..3448f531273 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -33,7 +33,6 @@ from litellm._logging import turn_on_json, verbose_proxy_logger from litellm.caching.caching import RedisCache from litellm.caching.dual_cache import DualCache from litellm.caching.redis_cluster_cache import RedisClusterCache -from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded from litellm.proxy._types import ( AwsErrorTypeProxyException, LitellmUserRoles, @@ -80,13 +79,22 @@ example_embedding_result = { } -def _reloaded(cost_map: dict) -> "ModelCostMapReloaded": - from litellm._catalog import Catalog, Fetched - from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded +from litellm.litellm_core_utils.get_model_cost_map import reload_model_cost_map as _REAL_RELOAD - parsed = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0) - assert isinstance(parsed, Fetched) - return ModelCostMapReloaded(fetched=parsed) + +def _reloading(cost_map: dict): + """A stand-in reload that installs ``cost_map`` over the bundled map, read from a file like a fetched map""" + import tempfile + from pathlib import Path + + from litellm._catalog import CATALOG, File + + def reload(source, catalog=CATALOG): + path = Path(tempfile.mkdtemp()) / "model_prices.json" + path.write_text(json.dumps({**CATALOG.bundled().document(), **cost_map})) + return _REAL_RELOAD(File(path), catalog) + + return reload def mock_patch_aembedding(): @@ -4553,14 +4561,13 @@ class TestPriceDataReloadAPI: # Save the original model_cost so the endpoint's direct assignment # (litellm.model_cost = new_model_cost_map) does not contaminate # subsequent tests running in the same worker process. - from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded original_model_cost = litellm.model_cost.copy() try: with patch( - "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", - new=AsyncMock( - return_value=_reloaded({"gpt-3.5-turbo": {"input_cost_per_token": 0.001}}) + "litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", + new=MagicMock( + side_effect=_reloading({"gpt-3.5-turbo": {"input_cost_per_token": 0.001}}) ), ): # Mock the database connection @@ -4614,8 +4621,8 @@ class TestPriceDataReloadAPI: def test_reload_model_cost_map_error_handling(self, client_with_auth): """Test error handling in the reload endpoint""" with patch( - "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", - new=AsyncMock(side_effect=Exception("Network error")), + "litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", + new=MagicMock(side_effect=Exception("Network error")), ): # Mock the database connection with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: @@ -4828,13 +4835,11 @@ class TestPriceDataReloadIntegration: "gpt-4": {"input_cost_per_token": 0.03, "output_cost_per_token": 0.06}, } - from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded - original_model_cost = litellm.model_cost.copy() try: with patch( - "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", - new=AsyncMock(return_value=_reloaded(mock_cost_map)), + "litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", + new=MagicMock(side_effect=_reloading(mock_cost_map)), ): # Mock the database connection with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: @@ -4897,17 +4902,15 @@ class TestPriceDataReloadIntegration: proxy_config.model_cost_map_loaded_at = frozen_now - timedelta(minutes=1) proxy_config.model_cost_map_applied_revision = 3 - from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded - original_model_cost = litellm.model_cost.copy() try: with ( patch( - "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new_callable=AsyncMock + "litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", new_callable=MagicMock ) as mock_get_map, patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now), ): - mock_get_map.return_value = _reloaded({"gpt-3.5-turbo": {"input_cost_per_token": 0.001}} + mock_get_map.side_effect = _reloading({"gpt-3.5-turbo": {"input_cost_per_token": 0.001}} ) asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma)) @@ -4950,7 +4953,7 @@ class TestPriceDataReloadIntegration: try: with ( patch( - "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new_callable=AsyncMock + "litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", new_callable=MagicMock ) as mock_get_map, patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now), ): @@ -4987,11 +4990,11 @@ class TestPriceDataReloadIntegration: try: with ( patch( - "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new_callable=AsyncMock + "litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", new_callable=MagicMock ) as mock_get_map, patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now), ): - mock_get_map.return_value = _reloaded({"gpt-4-test": {"input_cost_per_token": 0.5}} + mock_get_map.side_effect = _reloading({"gpt-4-test": {"input_cost_per_token": 0.5}} ) asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma)) @@ -5031,11 +5034,11 @@ class TestPriceDataReloadIntegration: try: with ( patch( - "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new_callable=AsyncMock + "litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", new_callable=MagicMock ) as mock_get_map, patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now), ): - mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}} + mock_get_map.side_effect = _reloading({"gpt-4": {"input_cost_per_token": 0.001}} ) for _ in range(3): @@ -5074,11 +5077,11 @@ class TestPriceDataReloadIntegration: try: with ( patch( - "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new_callable=AsyncMock + "litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", new_callable=MagicMock ) as mock_get_map, patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now), ): - mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}} + mock_get_map.side_effect = _reloading({"gpt-4": {"input_cost_per_token": 0.001}} ) asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma)) @@ -5109,11 +5112,11 @@ class TestPriceDataReloadIntegration: try: with ( patch( - "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new_callable=AsyncMock + "litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", new_callable=MagicMock ) as mock_get_map, patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now), ): - mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}} + mock_get_map.side_effect = _reloading({"gpt-4": {"input_cost_per_token": 0.001}} ) asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma)) @@ -5149,12 +5152,12 @@ class TestPriceDataReloadIntegration: try: with ( patch( - "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", - new_callable=AsyncMock, + "litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", + new_callable=MagicMock, ) as mock_get_map, patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now), ): - mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.1}} + mock_get_map.side_effect = _reloading({"gpt-4": {"input_cost_per_token": 0.1}} ) asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma)) @@ -5174,9 +5177,7 @@ class TestPriceDataReloadIntegration: so a pod that records it as applied without the data stays mispriced until someone clicks again """ - from litellm.litellm_core_utils.get_model_cost_map import ( - ModelCostMapReloadUnavailable, - ) + from litellm._catalog import Unavailable from litellm.proxy.proxy_server import ProxyConfig proxy_config = ProxyConfig() @@ -5193,8 +5194,8 @@ class TestPriceDataReloadIntegration: original_model_cost = litellm.model_cost with ( patch( - "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", - new=AsyncMock(return_value=ModelCostMapReloadUnavailable(reason="HTTP 429 from upstream")), + "litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", + new=MagicMock(return_value=Unavailable(reason="HTTP 429 from upstream")), ), patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now), ): @@ -5219,7 +5220,6 @@ class TestPriceDataReloadIntegration: after it is swallowed by the surrounding except and would otherwise leave the metadata correct while the path is quietly broken""" from litellm import utils as litellm_utils - from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded from litellm.proxy.proxy_server import ProxyConfig proxy_config = ProxyConfig() @@ -5240,9 +5240,9 @@ class TestPriceDataReloadIntegration: with ( patch( - "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", - new=AsyncMock( - return_value=_reloaded({"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}} + "litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", + new=MagicMock( + side_effect=_reloading({"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}} ) ), ), @@ -5261,13 +5261,12 @@ class TestPriceDataReloadIntegration: litellm_utils._runtime_registered_model_cost.update(original_registry) _invalidate_model_cost_lowercase_map() - def test_swap_in_model_cost_map_counts_the_fetched_catalog_only(self): + def test_a_reload_counts_the_fetched_catalog_only(self): """The count the reload endpoints report describes the price data, so it is taken before the runtime registrations are written back into the same dict. Counting after would inflate it by however many deployments and overrides this pod happens to be carrying""" from litellm import utils as litellm_utils - from litellm.proxy.proxy_server import _swap_in_model_cost_map original_model_cost = litellm.model_cost original_registry = dict(litellm_utils._runtime_registered_model_cost) @@ -5276,9 +5275,9 @@ class TestPriceDataReloadIntegration: model_cost={"custom/deployment-model": {"litellm_provider": "custom", "max_input_tokens": 4321}} ) - models_count = _swap_in_model_cost_map(_reloaded({"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}}).fetched) + installed = _reloading({"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}})(None) - assert models_count == len(litellm.model_cost) - 1 + assert installed.count == len(litellm.model_cost) - 1 assert litellm.model_cost["custom/deployment-model"]["max_input_tokens"] == 4321 finally: litellm.model_cost = original_model_cost @@ -5305,18 +5304,16 @@ class TestPriceDataReloadIntegration: client = TestClient(app) frozen_now = datetime(2024, 1, 1, 7, 0, tzinfo=timezone.utc) - from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded - original_model_cost = litellm.model_cost.copy() try: with ( patch( - "litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new_callable=AsyncMock + "litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", new_callable=MagicMock ) as mock_get_map, patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma, patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now), ): - mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}} + mock_get_map.side_effect = _reloading({"gpt-4": {"input_cost_per_token": 0.001}} ) mock_prisma.db.litellm_config.upsert = AsyncMock( return_value=_reload_schedule_row({}, reload_revision=9) diff --git a/tests/unit/test_conftest_isolation.py b/tests/unit/test_conftest_isolation.py index 92457119a82..13295f4a68a 100644 --- a/tests/unit/test_conftest_isolation.py +++ b/tests/unit/test_conftest_isolation.py @@ -1,4 +1,5 @@ import asyncio +import json import sys from collections.abc import Iterator from dataclasses import dataclass @@ -12,8 +13,8 @@ from litellm import Router from litellm import router as litellm_router_module from litellm import utils as litellm_utils_module from litellm.caching.caching import DualCache -from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from tests.unit import conftest as unit_harness CANARY_MODEL = "conftest-isolation-canary-model" @@ -179,11 +180,14 @@ def registered_collection_models() -> Iterator[None]: litellm_router_module._live_routers.update(original_routers) -def test_collection_model_metadata_is_not_replayed_into_a_test_catalog() -> None: - from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map +def test_collection_model_metadata_is_not_replayed_into_a_test_catalog(tmp_path) -> None: + from litellm._catalog import CATALOG, File + from litellm.litellm_core_utils.get_model_cost_map import reload_model_cost_map catalog: Final = {"unit-test-catalog-model": {"litellm_provider": "openai", "input_cost_per_token": 0.001}} - adopt_model_cost_map(catalog) + path: Final = tmp_path / "model_prices.json" + path.write_text(json.dumps({**CATALOG.bundled().document(), **catalog})) + reload_model_cost_map(File(path)) assert "collection-isolation-ledger" not in litellm.model_cost assert "collection-isolation-router" not in litellm.model_cost assert "unit-test-catalog-model" in litellm.model_cost diff --git a/tests/unit/test_model_cost_aliases.py b/tests/unit/test_model_cost_aliases.py index 8d63ae658bb..105b23d378d 100644 --- a/tests/unit/test_model_cost_aliases.py +++ b/tests/unit/test_model_cost_aliases.py @@ -4,19 +4,31 @@ Tests for the ``aliases`` feature in the model cost map. The Rust catalog serves every name in an ``aliases`` list as its own key, sharing one dict with the canonical entry """ +import functools import json +import tempfile +from pathlib import Path -from litellm._catalog import Catalog, Fetched, Unavailable +from litellm._catalog import CATALOG, Fetched, File, Unavailable -def _parsed(model_cost: dict) -> Fetched | Unavailable: - return Catalog.parse(json.dumps(model_cost).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0) +@functools.lru_cache(maxsize=1) +def _bundled_document() -> dict: + return CATALOG.bundled().document() + + +def _parsed(model_cost: dict, over_bundled: bool = True) -> Fetched | Unavailable: + path = Path(tempfile.mkdtemp()) / "model_prices.json" + path.write_text(json.dumps({**_bundled_document(), **model_cost} if over_bundled else model_cost)) + return CATALOG.fetch(File(path), timeout=5, attempts=1) def _expand(model_cost: dict) -> dict: + """The rows the fetched map serves beyond the bundled ones, aliases included""" parsed = _parsed(model_cost) assert isinstance(parsed, Fetched) - return parsed.served() + bundled = _bundled_document() + return {key: row for key, row in parsed.served().items() if key not in bundled or key in model_cost} def _error_kinds(model_cost: dict) -> list[str]: @@ -247,4 +259,4 @@ class TestAliasIntegration: assert "model-with-alias" in result def test_an_empty_map_is_unavailable(self): - assert isinstance(_parsed({}), Unavailable) + assert isinstance(_parsed({}, over_bundled=False), Unavailable) diff --git a/tests/unit/test_register_model_custom_pricing.py b/tests/unit/test_register_model_custom_pricing.py index 402c0ca05fd..d9e5b775548 100644 --- a/tests/unit/test_register_model_custom_pricing.py +++ b/tests/unit/test_register_model_custom_pricing.py @@ -14,22 +14,22 @@ import itertools import json import os from collections.abc import Callable, Mapping +from pathlib import Path from typing import Final, NoReturn import pytest - import litellm -from litellm._catalog import Catalog, Fetched -from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry, adopt_model_cost_map +from litellm._catalog import File +from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry, reload_model_cost_map from litellm.main import _build_custom_pricing_entry from litellm.utils import _invalidate_model_cost_lowercase_map -def _fetched(cost_map: Mapping[str, object]) -> Fetched: - parsed: Final = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0) - assert isinstance(parsed, Fetched) - return parsed +def _reload_from(cost_map: Mapping[str, object], directory: Path) -> object: + path: Final = directory / "model_prices.json" + path.write_text(json.dumps(cost_map)) + return reload_model_cost_map(File(path)) def _snapshot_model_cost_entries(keys): @@ -1027,7 +1027,9 @@ def test_register_model_prices_a_provider_less_azure_ai_key_without_repricing_th _invalidate_model_cost_lowercase_map() -def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers(monkeypatch: pytest.MonkeyPatch) -> None: +def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: catalog_key: Final = "au.anthropic.lit-5853-reloaded-model" deployment_key: Final = f"bedrock/{catalog_key}" catalog_row: Final = { @@ -1046,12 +1048,12 @@ def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers(monk catalog_key: {**catalog_row, "input_cost_per_token": reloaded_input_cost}, } real_lookup: Final = litellm.utils.get_model_info - reloads: Final[list[int]] = [] # mutable-ok: counts reloads the racing lookup triggered + reloads: Final[list[object]] = [] # mutable-ok: counts reloads the racing lookup triggered def lookup_racing_a_price_data_reload(model: str, custom_llm_provider: str | None = None) -> object: info: Final = real_lookup(model=model, custom_llm_provider=custom_llm_provider) if not reloads: - reloads.append(adopt_model_cost_map(_fetched(reloaded_cost_map))) + reloads.append(_reload_from(reloaded_cost_map, tmp_path)) return info with pytest.MonkeyPatch.context() as lookup_patch: @@ -1363,10 +1365,10 @@ def test_update_dictionary_merges_nested_dicts_without_aliasing(): object stays untouched, and the caller's incoming nested dict is never inserted by reference into the merged result. """ - from litellm.utils import update_dictionary - from typing import cast + from litellm.utils import update_dictionary + existing_nested = cast(dict[str, object], {"hours_utc": "01:00-02:00", 1: "existing"}) existing = {"off_peak_pricing": existing_nested} incoming_nested = cast( diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 20b0055f729..7b829cd235c 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -54,8 +54,8 @@ _COPILOT_TOKEN_EXPIRES_AT: Final = 4_102_444_800 def _simulate_price_data_reload(fetched_catalog): """Drive what a price data reload does to this process's litellm state. - Mirrors `litellm.proxy.proxy_server._swap_in_model_cost_map`, which is the - one place both reload paths adopt a freshly fetched catalog; that wiring is + Mirrors `litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map`, which is + the one place both reload paths install a freshly fetched catalog; that wiring is covered in the proxy's own tests, so these exercise the replay itself without dragging the proxy in. The provider model sets that helper also repopulates are left alone, since nothing here reads them and rebuilding @@ -3858,7 +3858,7 @@ def test_every_declaring_deployment_is_named(caplog): def _simulate_price_data_reload_with_provider_sets(monkeypatch, fetched_catalog): """Like `_simulate_price_data_reload`, plus the provider model-set refresh the proxy's - `_swap_in_model_cost_map` does before replaying, so bare names in the new catalog resolve.""" + `reload_model_cost_map` does before replaying, so bare names in the new catalog resolve.""" monkeypatch.setattr(litellm, "model_cost", fetched_catalog) _invalidate_model_cost_lowercase_map() litellm.add_known_models(model_cost_map=fetched_catalog) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 2140df37434..a336a38527b 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -2454,16 +2454,34 @@ def test_register_model_with_scientific_notation(): @pytest.mark.usefixtures("isolated_openai_model_sets") @respx.mock def test_register_model_url_fetch_uses_single_attempt(monkeypatch): + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False) monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost)) before = dict(litellm.model_cost) threads_before = {thread.name for thread in threading.enumerate()} - route = respx.get("https://example.invalid/custom_pricing.json").mock(return_value=httpx.Response(503)) + requests: list[str] = [] - litellm.register_model(model_cost="https://example.invalid/custom_pricing.json") + class Unavailable(BaseHTTPRequestHandler): + def do_GET(self) -> None: + requests.append(self.path) + self.send_response(503) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *args: object) -> None: + pass + + httpd = ThreadingHTTPServer(("127.0.0.1", 0), Unavailable) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + try: + litellm.register_model(model_cost=f"http://127.0.0.1:{httpd.server_address[1]}/custom_pricing.json") + finally: + httpd.shutdown() + httpd.server_close() threads_after = {thread.name for thread in threading.enumerate()} - assert route.call_count == 1 + assert requests == ["/custom_pricing.json"] assert not (threads_after - threads_before) & {"litellm-model-cost-map-retry"} assert not any( thread.name == "litellm-model-cost-map-retry" and thread.is_alive() for thread in threading.enumerate()