mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
refactor(model-cost): fetch and reload the cost map through Rust
model-catalog-refresh fetches over HTTP, S3 and file with main's retry rules, python-bridge drives it with the GIL released, and provenance is one Rust snapshot recorded by install. The S3 client setup moves into a shared s3 crate used by cache-s3. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
713b69b8d4
commit
18d2c77042
38 changed files with 2451 additions and 1307 deletions
30
litellm-rust/Cargo.lock
generated
30
litellm-rust/Cargo.lock
generated
|
|
@ -3618,6 +3618,7 @@ dependencies = [
|
|||
"litellm-cache",
|
||||
"litellm-cache-testing",
|
||||
"litellm-http",
|
||||
"litellm-s3",
|
||||
"reqwest 0.12.28",
|
||||
"rstest",
|
||||
"serde_json",
|
||||
|
|
@ -4255,8 +4256,22 @@ dependencies = [
|
|||
name = "litellm-model-catalog-refresh"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"aws-config",
|
||||
"aws-credential-types",
|
||||
"aws-sdk-s3",
|
||||
"aws-smithy-runtime-api",
|
||||
"litellm-http",
|
||||
"litellm-model-catalog",
|
||||
"litellm-s3",
|
||||
"rand 0.8.7",
|
||||
"reqwest 0.12.28",
|
||||
"rstest",
|
||||
"serde_json",
|
||||
"sha1 0.10.7",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"url",
|
||||
"wiremock",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -4289,6 +4304,7 @@ dependencies = [
|
|||
"litellm-llms",
|
||||
"litellm-llms-types",
|
||||
"litellm-model-catalog",
|
||||
"litellm-model-catalog-refresh",
|
||||
"litellm-router-types",
|
||||
"litellm-secrets",
|
||||
"litellm-secrets-aws",
|
||||
|
|
@ -4306,7 +4322,6 @@ dependencies = [
|
|||
"serde",
|
||||
"serde_json",
|
||||
"serde_with",
|
||||
"sha1 0.10.7",
|
||||
"sqlx",
|
||||
"strum",
|
||||
"thiserror 2.0.19",
|
||||
|
|
@ -4376,6 +4391,19 @@ dependencies = [
|
|||
"serde_with",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-s3"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-sdk-s3",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"http 1.4.2",
|
||||
"litellm-http",
|
||||
"reqwest 0.12.28",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-secrets"
|
||||
version = "0.1.0"
|
||||
|
|
|
|||
|
|
@ -75,6 +75,7 @@ litellm-redis-testing = { path = "crates/redis-testing" }
|
|||
litellm-cost = { path = "crates/cost" }
|
||||
litellm-model-catalog = { path = "crates/model-catalog" }
|
||||
litellm-model-catalog-refresh = { path = "crates/model-catalog-refresh" }
|
||||
litellm-s3 = { path = "crates/s3" }
|
||||
|
||||
askama = { version = "0.16.1", default-features = false, features = ["derive", "std"] }
|
||||
tracing = "0.1"
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ repository.workspace = true
|
|||
litellm-http.workspace = true
|
||||
litellm-cache.workspace = true
|
||||
litellm-auth-aws.workspace = true
|
||||
litellm-s3.workspace = true
|
||||
aws-sdk-s3 = { version = "1.146.1", default-features = false, features = ["rustls", "rt-tokio"] }
|
||||
aws-credential-types = "1.3.0"
|
||||
aws-smithy-runtime-api = { version = "1.16.2", features = ["client", "http-1x"] }
|
||||
|
|
|
|||
|
|
@ -4,11 +4,7 @@ use std::{
|
|||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use aws_sdk_s3::{
|
||||
config::{BehaviorVersion, Region, RequestChecksumCalculation, ResponseChecksumValidation},
|
||||
error::SdkError,
|
||||
primitives::ByteStream,
|
||||
};
|
||||
use aws_sdk_s3::{error::SdkError, primitives::ByteStream};
|
||||
use aws_smithy_types::{DateTime, date_time::Format};
|
||||
use futures_util::future::try_join_all;
|
||||
use litellm_auth_aws::AwsAuthConfig;
|
||||
|
|
@ -17,7 +13,9 @@ use litellm_cache::{
|
|||
};
|
||||
use tokio::runtime::Handle;
|
||||
|
||||
use crate::{auth::S3Credentials, transport::ReqwestHttpClient};
|
||||
use litellm_s3::S3ClientConfig;
|
||||
|
||||
use crate::auth::S3Credentials;
|
||||
|
||||
pub struct S3Endpoint {
|
||||
pub url: String,
|
||||
|
|
@ -49,19 +47,16 @@ impl<C: CacheCodec> S3Cache<C> {
|
|||
runtime: Handle,
|
||||
) -> Self {
|
||||
let endpoint_url: Option<String> = config.endpoint.map(|endpoint| endpoint.url);
|
||||
let base = aws_sdk_s3::Config::builder()
|
||||
.behavior_version(BehaviorVersion::latest())
|
||||
.region(Region::new(config.region.clone()))
|
||||
.http_client(ReqwestHttpClient(http))
|
||||
.credentials_provider(S3Credentials::new(config.auth))
|
||||
.request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
|
||||
.response_checksum_validation(ResponseChecksumValidation::WhenRequired);
|
||||
let builder = match &endpoint_url {
|
||||
Some(url) => base.endpoint_url(url).force_path_style(true),
|
||||
None => base,
|
||||
};
|
||||
let client = litellm_s3::client(
|
||||
S3ClientConfig {
|
||||
region: config.region.clone(),
|
||||
endpoint: endpoint_url.clone(),
|
||||
},
|
||||
http,
|
||||
S3Credentials::new(config.auth),
|
||||
);
|
||||
Self {
|
||||
client: aws_sdk_s3::Client::from_conf(builder.build()),
|
||||
client,
|
||||
codec,
|
||||
runtime,
|
||||
bucket: config.bucket.into(),
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
mod auth;
|
||||
mod cache;
|
||||
mod transport;
|
||||
|
||||
pub use auth::S3Credentials;
|
||||
pub use cache::{S3Cache, S3CacheConfig, S3Endpoint};
|
||||
|
|
|
|||
|
|
@ -6,5 +6,23 @@ license.workspace = true
|
|||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"] }
|
||||
aws-sdk-s3 = { version = "1.146.1", default-features = false, features = ["rustls", "rt-tokio"] }
|
||||
aws-smithy-runtime-api = { version = "1.16.2", features = ["client", "http-1x"] }
|
||||
litellm-model-catalog.workspace = true
|
||||
litellm-s3.workspace = true
|
||||
rand.workspace = true
|
||||
litellm-http.workspace = true
|
||||
reqwest.workspace = true
|
||||
sha1.workspace = true
|
||||
thiserror.workspace = true
|
||||
tokio = { workspace = true, features = ["fs", "time"] }
|
||||
url.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
litellm-http = { workspace = true, features = ["test-support"] }
|
||||
aws-credential-types = "1.3.0"
|
||||
rstest.workspace = true
|
||||
serde_json.workspace = true
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
|
||||
wiremock.workspace = true
|
||||
|
|
|
|||
9
litellm-rust/crates/model-catalog-refresh/src/error.rs
Normal file
9
litellm-rust/crates/model-catalog-refresh/src/error.rs
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
use thiserror::Error;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Error)]
|
||||
pub enum SourceError {
|
||||
#[error("unsupported model cost map URL scheme {scheme}")]
|
||||
UnsupportedScheme { scheme: String },
|
||||
#[error("invalid model cost map URL: {reason}")]
|
||||
Invalid { reason: String },
|
||||
}
|
||||
25
litellm-rust/crates/model-catalog-refresh/src/file.rs
Normal file
25
litellm-rust/crates/model-catalog-refresh/src/file.rs
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
use std::io::ErrorKind;
|
||||
use std::path::Path;
|
||||
|
||||
use crate::step::Reply;
|
||||
|
||||
/// A file has no validator, so every read is a fresh 200
|
||||
pub(crate) async fn fetch(path: &Path) -> Reply {
|
||||
match tokio::fs::read(path).await {
|
||||
Ok(body) => Reply::Response {
|
||||
status: 200,
|
||||
etag: None,
|
||||
retry_after: None,
|
||||
body: Some(body),
|
||||
},
|
||||
Err(error) if error.kind() == ErrorKind::NotFound => Reply::Response {
|
||||
status: 404,
|
||||
etag: None,
|
||||
retry_after: None,
|
||||
body: None,
|
||||
},
|
||||
Err(error) => Reply::Transport {
|
||||
reason: format!("reading {} failed: {error}", path.display()),
|
||||
},
|
||||
}
|
||||
}
|
||||
82
litellm-rust/crates/model-catalog-refresh/src/http.rs
Normal file
82
litellm-rust/crates/model-catalog-refresh/src/http.rs
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use reqwest::header::{ETAG, IF_NONE_MATCH, RETRY_AFTER};
|
||||
use url::Url;
|
||||
|
||||
use crate::step::{FetchRequest, Reply};
|
||||
|
||||
pub(crate) async fn fetch(
|
||||
client: &litellm_http::Client,
|
||||
url: &Url,
|
||||
request: FetchRequest,
|
||||
) -> Reply {
|
||||
let builder = client.get(url.clone()).timeout(request.timeout);
|
||||
let builder = match &request.if_none_match {
|
||||
Some(etag) => builder.header(IF_NONE_MATCH, etag),
|
||||
None => builder,
|
||||
};
|
||||
let response = match builder.send().await {
|
||||
Ok(response) => response,
|
||||
Err(error) => {
|
||||
return Reply::Transport {
|
||||
reason: format!("fetching {url} failed: {error}"),
|
||||
};
|
||||
}
|
||||
};
|
||||
let status = response.status().as_u16();
|
||||
let etags: Vec<&str> = response
|
||||
.headers()
|
||||
.get_all(ETAG)
|
||||
.iter()
|
||||
.filter_map(|value| value.to_str().ok())
|
||||
.collect();
|
||||
let etag = (!etags.is_empty()).then(|| etags.join(", "));
|
||||
let retry_after = response
|
||||
.headers()
|
||||
.get(RETRY_AFTER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(retry_after_seconds);
|
||||
if status != 200 {
|
||||
return Reply::Response {
|
||||
status,
|
||||
etag,
|
||||
retry_after,
|
||||
body: None,
|
||||
};
|
||||
}
|
||||
match response.bytes().await {
|
||||
Ok(body) => Reply::Response {
|
||||
status,
|
||||
etag,
|
||||
retry_after,
|
||||
body: Some(body.to_vec()),
|
||||
},
|
||||
Err(error) => Reply::Transport {
|
||||
reason: format!("reading {url} failed: {error}"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Main reads `Retry-After` as a number of seconds and ignores an HTTP date or a negative number
|
||||
fn retry_after_seconds(header: &str) -> Option<Duration> {
|
||||
let seconds = header.trim().parse::<f64>().ok()?;
|
||||
(seconds.is_finite() && seconds >= 0.0)
|
||||
.then(|| Duration::try_from_secs_f64(seconds).ok())
|
||||
.flatten()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
#[case::seconds("7", Some(Duration::from_secs(7)))]
|
||||
#[case::fraction("0.5", Some(Duration::from_millis(500)))]
|
||||
#[case::negative("-1", None)]
|
||||
#[case::http_date("Wed, 21 Oct 2015 07:28:00 GMT", None)]
|
||||
fn retry_after_is_seconds(#[case] header: &str, #[case] expected: Option<Duration>) {
|
||||
assert_eq!(retry_after_seconds(header), expected);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +1,67 @@
|
|||
mod error;
|
||||
mod file;
|
||||
mod http;
|
||||
mod policy;
|
||||
mod revalidate;
|
||||
mod s3;
|
||||
mod source;
|
||||
mod step;
|
||||
|
||||
use std::future::Future;
|
||||
use std::time::Duration;
|
||||
|
||||
pub use error::*;
|
||||
pub use policy::*;
|
||||
pub use revalidate::*;
|
||||
pub use s3::*;
|
||||
pub use source::*;
|
||||
pub use step::*;
|
||||
|
||||
/// The transports a fetch can use. The HTTP client comes from the caller's pool, the S3 client is built per source
|
||||
pub struct Transports<'a> {
|
||||
pub http: &'a litellm_http::Client,
|
||||
pub s3: Option<&'a S3Source>,
|
||||
}
|
||||
|
||||
/// One attempt at `source`, never retried here
|
||||
pub async fn fetch(source: &Source, request: FetchRequest, transports: &Transports<'_>) -> Reply {
|
||||
match source {
|
||||
Source::Http(url) => http::fetch(transports.http, url, request).await,
|
||||
Source::File(path) => file::fetch(path).await,
|
||||
Source::S3(object) => match transports.s3 {
|
||||
Some(s3) => s3.fetch(request).await,
|
||||
None => {
|
||||
S3Source::from_environment(object.clone(), transports.http.clone())
|
||||
.await
|
||||
.fetch(request)
|
||||
.await
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Drives `Revalidation` to `Adopt` or `Done`, sleeping between attempts with the injected `sleep`
|
||||
pub async fn fetch_with_retry<S, F>(
|
||||
source: &Source,
|
||||
installed: &Installed,
|
||||
policy: RetryPolicy,
|
||||
seed: Option<u64>,
|
||||
transports: &Transports<'_>,
|
||||
sleep: S,
|
||||
) -> Step
|
||||
where
|
||||
S: Fn(Duration) -> F,
|
||||
F: Future<Output = ()>,
|
||||
{
|
||||
let (mut revalidation, mut step) = Revalidation::new(source, installed, policy, seed);
|
||||
loop {
|
||||
step = match step {
|
||||
Step::Fetch(request) => revalidation.on_reply(fetch(source, request, transports).await),
|
||||
Step::Sleep(delay) => {
|
||||
sleep(delay).await;
|
||||
revalidation.on_slept()
|
||||
}
|
||||
done => return done,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,3 +18,17 @@ pub struct RetryPolicy {
|
|||
/// Main's default `LITELLM_MODEL_COST_MAP_URL`
|
||||
pub const DEFAULT_REGISTRY_URL: &str =
|
||||
"https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json";
|
||||
|
||||
impl Default for RetryPolicy {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_attempts: 3,
|
||||
max_wait: Duration::from_secs(30),
|
||||
timeout: Duration::from_secs(5),
|
||||
backoff_base_seconds: 2.0,
|
||||
jitter_min_seconds: 0.0,
|
||||
jitter_max_seconds: 1.0,
|
||||
limits: IntegrityLimits::new(50, 0.5).expect("0.5 is a ratio"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
410
litellm-rust/crates/model-catalog-refresh/src/revalidate.rs
Normal file
410
litellm-rust/crates/model-catalog-refresh/src/revalidate.rs
Normal file
|
|
@ -0,0 +1,410 @@
|
|||
use std::sync::LazyLock;
|
||||
use std::time::Duration;
|
||||
|
||||
use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog};
|
||||
use rand::rngs::StdRng;
|
||||
use rand::{Rng, SeedableRng};
|
||||
use sha1::{Digest, Sha1};
|
||||
|
||||
use crate::policy::RetryPolicy;
|
||||
use crate::source::Source;
|
||||
use crate::step::{FetchRequest, Installed, Origin, Reply, Step};
|
||||
|
||||
/// Main's `RETRYABLE_FETCH_STATUS_CODES`
|
||||
const RETRYABLE_STATUSES: [u16; 5] = [429, 500, 502, 503, 504];
|
||||
|
||||
static BUNDLED_MODEL_COUNT: LazyLock<usize> = LazyLock::new(|| {
|
||||
Catalog::parse(BUNDLED_REGISTRY)
|
||||
.expect("the bundled model registry parses")
|
||||
.value
|
||||
.model_count()
|
||||
});
|
||||
|
||||
/// The fetch state machine. It asks its driver for an attempt or a sleep and never does IO itself
|
||||
#[derive(Debug)]
|
||||
pub struct Revalidation {
|
||||
source: Source,
|
||||
if_none_match: Option<String>,
|
||||
policy: RetryPolicy,
|
||||
attempts: u32,
|
||||
rng: StdRng,
|
||||
state: State,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum State {
|
||||
AwaitingReply,
|
||||
AwaitingSleep,
|
||||
Done(String, Option<Duration>),
|
||||
}
|
||||
|
||||
impl Revalidation {
|
||||
pub fn new(
|
||||
source: &Source,
|
||||
installed: &Installed,
|
||||
policy: RetryPolicy,
|
||||
seed: Option<u64>,
|
||||
) -> (Self, Step) {
|
||||
let if_none_match = match &installed.origin {
|
||||
Origin::Remote { source: from } if from == source => installed.etag.clone(),
|
||||
Origin::Remote { .. } | Origin::Bundled => None,
|
||||
};
|
||||
let revalidation = Self {
|
||||
source: source.clone(),
|
||||
if_none_match,
|
||||
policy,
|
||||
attempts: 1,
|
||||
rng: seed.map_or_else(StdRng::from_entropy, StdRng::seed_from_u64),
|
||||
state: State::AwaitingReply,
|
||||
};
|
||||
let step = revalidation.fetch();
|
||||
(revalidation, step)
|
||||
}
|
||||
|
||||
pub fn on_reply(&mut self, reply: Reply) -> Step {
|
||||
if !matches!(self.state, State::AwaitingReply) {
|
||||
return self.invalid_state("revalidation is not awaiting a reply");
|
||||
}
|
||||
match reply {
|
||||
Reply::Transport { reason } => self.retry_or_done(reason, None),
|
||||
Reply::Response {
|
||||
status: 200,
|
||||
etag,
|
||||
body,
|
||||
..
|
||||
} => self.adopt(body.unwrap_or_default(), etag),
|
||||
Reply::Response { status: 304, .. } => {
|
||||
self.finish(format!("{} is not modified", self.source))
|
||||
}
|
||||
Reply::Response {
|
||||
status,
|
||||
retry_after,
|
||||
..
|
||||
} if RETRYABLE_STATUSES.contains(&status) => {
|
||||
self.retry_or_done(format!("HTTP {status} from {}", self.source), retry_after)
|
||||
}
|
||||
Reply::Response { status, .. } => {
|
||||
self.finish(format!("HTTP {status} from {}", self.source))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn on_slept(&mut self) -> Step {
|
||||
if !matches!(self.state, State::AwaitingSleep) {
|
||||
return self.invalid_state("revalidation is not awaiting a sleep");
|
||||
}
|
||||
self.attempts += 1;
|
||||
self.state = State::AwaitingReply;
|
||||
self.fetch()
|
||||
}
|
||||
|
||||
fn fetch(&self) -> Step {
|
||||
Step::Fetch(FetchRequest {
|
||||
if_none_match: self.if_none_match.clone(),
|
||||
timeout: self.policy.timeout,
|
||||
})
|
||||
}
|
||||
|
||||
fn adopt(&mut self, body: Vec<u8>, etag: Option<String>) -> Step {
|
||||
let loaded = match Catalog::parse(&body) {
|
||||
Ok(loaded) => loaded,
|
||||
Err(error) => {
|
||||
return self.finish(format!(
|
||||
"model cost map from {} is unusable: {error}",
|
||||
self.source
|
||||
));
|
||||
}
|
||||
};
|
||||
if let Err(error) = loaded
|
||||
.value
|
||||
.check_size(*BUNDLED_MODEL_COUNT, self.policy.limits)
|
||||
{
|
||||
return self.finish(format!(
|
||||
"model cost map from {} is unusable: {error}",
|
||||
self.source
|
||||
));
|
||||
}
|
||||
let installed = Installed {
|
||||
origin: Origin::Remote {
|
||||
source: self.source.clone(),
|
||||
},
|
||||
etag,
|
||||
revision: git_blob_id(&body),
|
||||
model_count: loaded.value.model_count(),
|
||||
};
|
||||
self.state = State::Done("adopted".to_owned(), None);
|
||||
Step::Adopt {
|
||||
catalog: Box::new(loaded.value),
|
||||
installed,
|
||||
errors: loaded.errors,
|
||||
}
|
||||
}
|
||||
|
||||
fn retry_or_done(&mut self, reason: String, retry_after: Option<Duration>) -> Step {
|
||||
let delay = retry_after
|
||||
.unwrap_or_else(|| self.backoff())
|
||||
.min(self.policy.max_wait);
|
||||
if self.attempts >= self.policy.max_attempts {
|
||||
let reason = if self.policy.max_attempts > 1 {
|
||||
format!("{reason} (after {} attempts)", self.policy.max_attempts)
|
||||
} else {
|
||||
reason
|
||||
};
|
||||
self.state = State::Done(reason.clone(), Some(delay));
|
||||
return Step::Done {
|
||||
reason,
|
||||
retry_after: Some(delay),
|
||||
};
|
||||
}
|
||||
self.state = State::AwaitingSleep;
|
||||
Step::Sleep(delay)
|
||||
}
|
||||
|
||||
fn backoff(&mut self) -> Duration {
|
||||
let jitter = if self.policy.jitter_max_seconds > self.policy.jitter_min_seconds {
|
||||
self.rng
|
||||
.gen_range(self.policy.jitter_min_seconds..=self.policy.jitter_max_seconds)
|
||||
} else {
|
||||
self.policy.jitter_min_seconds
|
||||
};
|
||||
let seconds = self.policy.backoff_base_seconds.powi(self.attempts as i32) + jitter;
|
||||
Duration::try_from_secs_f64(seconds.min(self.policy.max_wait.as_secs_f64()))
|
||||
.unwrap_or(self.policy.max_wait)
|
||||
}
|
||||
|
||||
fn finish(&mut self, reason: String) -> Step {
|
||||
self.state = State::Done(reason.clone(), None);
|
||||
Step::Done {
|
||||
reason,
|
||||
retry_after: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid_state(&mut self, reason: &str) -> Step {
|
||||
if let State::Done(previous, retry_after) = &self.state {
|
||||
return Step::Done {
|
||||
reason: previous.clone(),
|
||||
retry_after: *retry_after,
|
||||
};
|
||||
}
|
||||
self.finish(reason.to_owned())
|
||||
}
|
||||
}
|
||||
|
||||
/// The digest `git hash-object` prints, which main records as the source revision
|
||||
pub fn git_blob_id(body: &[u8]) -> String {
|
||||
let mut hasher = Sha1::new();
|
||||
hasher.update(format!("blob {}\0", body.len()).as_bytes());
|
||||
hasher.update(body);
|
||||
hasher
|
||||
.finalize()
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::{fixture, rstest};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn source() -> Source {
|
||||
Source::parse("https://example.com/map.json").unwrap()
|
||||
}
|
||||
|
||||
#[fixture]
|
||||
fn bundled() -> Installed {
|
||||
Installed {
|
||||
origin: Origin::Bundled,
|
||||
etag: None,
|
||||
revision: git_blob_id(BUNDLED_REGISTRY),
|
||||
model_count: *BUNDLED_MODEL_COUNT,
|
||||
}
|
||||
}
|
||||
|
||||
fn response(status: u16) -> Reply {
|
||||
Reply::Response {
|
||||
status,
|
||||
etag: None,
|
||||
retry_after: None,
|
||||
body: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn done(step: Step) -> String {
|
||||
match step {
|
||||
Step::Done { reason, .. } => reason,
|
||||
other => panic!("expected done, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn slept(step: Step) -> Duration {
|
||||
match step {
|
||||
Step::Sleep(wait) => wait,
|
||||
other => panic!("expected a sleep, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn started(installed: &Installed) -> Revalidation {
|
||||
Revalidation::new(&source(), installed, RetryPolicy::default(), Some(0)).0
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::too_many_requests(429)]
|
||||
#[case::internal_error(500)]
|
||||
#[case::bad_gateway(502)]
|
||||
#[case::unavailable(503)]
|
||||
#[case::gateway_timeout(504)]
|
||||
fn mains_retryable_statuses_sleep_before_the_next_attempt(
|
||||
bundled: Installed,
|
||||
#[case] status: u16,
|
||||
) {
|
||||
let mut revalidation = started(&bundled);
|
||||
assert!(matches!(
|
||||
revalidation.on_reply(response(status)),
|
||||
Step::Sleep(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::not_implemented(501)]
|
||||
#[case::version_not_supported(505)]
|
||||
#[case::not_found(404)]
|
||||
fn other_statuses_are_final(bundled: Installed, #[case] status: u16) {
|
||||
let mut revalidation = started(&bundled);
|
||||
assert_eq!(
|
||||
done(revalidation.on_reply(response(status))),
|
||||
format!("HTTP {status} from https://example.com/map.json")
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::honored(Duration::from_secs(7), Duration::from_secs(7))]
|
||||
#[case::capped(Duration::from_secs(120), Duration::from_secs(30))]
|
||||
fn retry_after_sets_the_wait_up_to_the_cap(
|
||||
bundled: Installed,
|
||||
#[case] header: Duration,
|
||||
#[case] wait: Duration,
|
||||
) {
|
||||
let mut revalidation = started(&bundled);
|
||||
let reply = Reply::Response {
|
||||
status: 429,
|
||||
etag: None,
|
||||
retry_after: Some(header),
|
||||
body: None,
|
||||
};
|
||||
assert_eq!(slept(revalidation.on_reply(reply)), wait);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn backoff_grows_from_two_seconds_with_jitter(bundled: Installed) {
|
||||
let mut revalidation = started(&bundled);
|
||||
let first = slept(revalidation.on_reply(response(503)));
|
||||
revalidation.on_slept();
|
||||
let second = slept(revalidation.on_reply(response(503)));
|
||||
assert!((2.0..=3.0).contains(&first.as_secs_f64()), "{first:?}");
|
||||
assert!((4.0..=5.0).contains(&second.as_secs_f64()), "{second:?}");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn the_last_attempt_names_how_many_were_made(bundled: Installed) {
|
||||
let mut revalidation = started(&bundled);
|
||||
revalidation.on_reply(response(503));
|
||||
revalidation.on_slept();
|
||||
revalidation.on_reply(response(503));
|
||||
revalidation.on_slept();
|
||||
assert_eq!(
|
||||
done(revalidation.on_reply(Reply::Transport {
|
||||
reason: "connection refused".into()
|
||||
})),
|
||||
"connection refused (after 3 attempts)"
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_single_attempt_reports_the_wait_a_retry_should_use(bundled: Installed) {
|
||||
let policy = RetryPolicy {
|
||||
max_attempts: 1,
|
||||
..RetryPolicy::default()
|
||||
};
|
||||
let (mut revalidation, _) = Revalidation::new(&source(), &bundled, policy, Some(0));
|
||||
let reply = Reply::Response {
|
||||
status: 503,
|
||||
etag: None,
|
||||
retry_after: Some(Duration::from_secs(4)),
|
||||
body: None,
|
||||
};
|
||||
let Step::Done {
|
||||
reason,
|
||||
retry_after,
|
||||
} = revalidation.on_reply(reply)
|
||||
else {
|
||||
panic!("expected done");
|
||||
};
|
||||
assert_eq!(reason, "HTTP 503 from https://example.com/map.json");
|
||||
assert_eq!(retry_after, Some(Duration::from_secs(4)));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_body_that_is_not_a_catalog_is_final(bundled: Installed) {
|
||||
let mut revalidation = started(&bundled);
|
||||
let reply = Reply::Response {
|
||||
status: 200,
|
||||
etag: None,
|
||||
retry_after: None,
|
||||
body: Some(b"not json".to_vec()),
|
||||
};
|
||||
let reason = done(revalidation.on_reply(reply));
|
||||
assert!(reason.contains("invalid JSON"), "{reason}");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn the_bundled_body_is_adopted_with_its_revision_and_etag(bundled: Installed) {
|
||||
let mut revalidation = started(&bundled);
|
||||
let reply = Reply::Response {
|
||||
status: 200,
|
||||
etag: Some("W/\"abc\"".into()),
|
||||
retry_after: None,
|
||||
body: Some(BUNDLED_REGISTRY.to_vec()),
|
||||
};
|
||||
let Step::Adopt { installed, .. } = revalidation.on_reply(reply) else {
|
||||
panic!("expected adopt");
|
||||
};
|
||||
assert_eq!(installed.revision, bundled.revision);
|
||||
assert_eq!(installed.etag.as_deref(), Some("W/\"abc\""));
|
||||
assert_eq!(installed.origin, Origin::Remote { source: source() });
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::same_source(Origin::Remote { source: source() }, Some("v1"))]
|
||||
#[case::another_source(
|
||||
Origin::Remote { source: Source::parse("https://example.com/other.json").unwrap() },
|
||||
None
|
||||
)]
|
||||
#[case::bundled(Origin::Bundled, None)]
|
||||
fn the_etag_is_sent_only_back_to_the_source_that_gave_it(
|
||||
bundled: Installed,
|
||||
#[case] origin: Origin,
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
let installed = Installed {
|
||||
origin,
|
||||
etag: Some("v1".into()),
|
||||
..bundled
|
||||
};
|
||||
let (_, step) = Revalidation::new(&source(), &installed, RetryPolicy::default(), Some(0));
|
||||
let Step::Fetch(request) = step else {
|
||||
panic!("expected a fetch");
|
||||
};
|
||||
assert_eq!(request.if_none_match.as_deref(), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn git_blob_id_is_what_git_hash_object_prints() {
|
||||
assert_eq!(
|
||||
git_blob_id(b"{\"gpt-5.4-mini\": {\"mode\": \"chat\"}}\n"),
|
||||
"18b9a8381e13a3b38a2128f184f631f95829e987"
|
||||
);
|
||||
}
|
||||
}
|
||||
171
litellm-rust/crates/model-catalog-refresh/src/s3.rs
Normal file
171
litellm-rust/crates/model-catalog-refresh/src/s3.rs
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
use aws_config::BehaviorVersion;
|
||||
use aws_sdk_s3::config::SharedCredentialsProvider;
|
||||
use aws_sdk_s3::config::retry::RetryConfig;
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
use aws_smithy_runtime_api::http::Response;
|
||||
use litellm_s3::{ReqwestHttpClient, S3ClientConfig};
|
||||
|
||||
use crate::source::S3Object;
|
||||
use crate::step::{FetchRequest, Reply};
|
||||
|
||||
const BUCKET_REGION_HEADER: &str = "x-amz-bucket-region";
|
||||
const DEFAULT_REGION: &str = "us-east-1";
|
||||
|
||||
/// One S3 object as a catalog source. SDK retries are off, so `RetryPolicy` owns every attempt
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct S3Source {
|
||||
object: S3Object,
|
||||
http: litellm_http::Client,
|
||||
credentials: SharedCredentialsProvider,
|
||||
endpoint: Option<String>,
|
||||
client: aws_sdk_s3::Client,
|
||||
}
|
||||
|
||||
impl S3Source {
|
||||
pub fn new(
|
||||
object: S3Object,
|
||||
http: litellm_http::Client,
|
||||
credentials: SharedCredentialsProvider,
|
||||
config: S3ClientConfig,
|
||||
) -> Self {
|
||||
let endpoint = config.endpoint.clone();
|
||||
let client = without_retries(litellm_s3::client(
|
||||
config,
|
||||
http.clone(),
|
||||
credentials.clone(),
|
||||
));
|
||||
Self {
|
||||
object,
|
||||
http,
|
||||
credentials,
|
||||
endpoint,
|
||||
client,
|
||||
}
|
||||
}
|
||||
|
||||
/// The chain `boto3.client("s3")` used on main. Region from `AWS_REGION_NAME` first, then the SDK chain
|
||||
/// (`AWS_REGION`, `AWS_DEFAULT_REGION`, the profile). `AWS_ENDPOINT_URL_S3` or `AWS_ENDPOINT_URL` for
|
||||
/// S3-compatible stores
|
||||
pub async fn from_environment(object: S3Object, http: litellm_http::Client) -> Self {
|
||||
let shared = aws_config::defaults(BehaviorVersion::latest())
|
||||
.http_client(ReqwestHttpClient(http.clone()))
|
||||
.load()
|
||||
.await;
|
||||
let region = non_empty_env("AWS_REGION_NAME")
|
||||
.or_else(|| shared.region().map(ToString::to_string))
|
||||
.unwrap_or_else(|| DEFAULT_REGION.to_owned());
|
||||
let endpoint =
|
||||
non_empty_env("AWS_ENDPOINT_URL_S3").or_else(|| non_empty_env("AWS_ENDPOINT_URL"));
|
||||
let credentials = shared.credentials_provider().unwrap_or_else(|| {
|
||||
SharedCredentialsProvider::new(aws_sdk_s3::config::Credentials::new(
|
||||
"",
|
||||
"",
|
||||
None,
|
||||
None,
|
||||
"no AWS credentials found",
|
||||
))
|
||||
});
|
||||
Self::new(
|
||||
object,
|
||||
http,
|
||||
credentials,
|
||||
S3ClientConfig { region, endpoint },
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn fetch(&self, request: FetchRequest) -> Reply {
|
||||
match self.attempt(&self.client, &request).await {
|
||||
Redirect::To(region) => {
|
||||
let client = without_retries(litellm_s3::client(
|
||||
S3ClientConfig {
|
||||
region,
|
||||
endpoint: self.endpoint.clone(),
|
||||
},
|
||||
self.http.clone(),
|
||||
self.credentials.clone(),
|
||||
));
|
||||
match self.attempt(&client, &request).await {
|
||||
Redirect::To(region) => Reply::Transport {
|
||||
reason: format!("{} redirected twice, last to {region}", self.display()),
|
||||
},
|
||||
Redirect::No(reply) => reply,
|
||||
}
|
||||
}
|
||||
Redirect::No(reply) => reply,
|
||||
}
|
||||
}
|
||||
|
||||
async fn attempt(&self, client: &aws_sdk_s3::Client, request: &FetchRequest) -> Redirect {
|
||||
let call = client
|
||||
.get_object()
|
||||
.bucket(&self.object.bucket)
|
||||
.key(&self.object.key)
|
||||
.set_if_none_match(request.if_none_match.clone())
|
||||
.send();
|
||||
let output = match tokio::time::timeout(request.timeout, call).await {
|
||||
Err(_) => {
|
||||
return Redirect::No(Reply::Transport {
|
||||
reason: format!("fetching {} timed out", self.display()),
|
||||
});
|
||||
}
|
||||
Ok(Err(SdkError::ServiceError(error))) => return rejected(error.raw()),
|
||||
Ok(Err(SdkError::ResponseError(error))) => return rejected(error.raw()),
|
||||
Ok(Err(error)) => {
|
||||
return Redirect::No(Reply::Transport {
|
||||
reason: format!("fetching {} failed: {error}", self.display()),
|
||||
});
|
||||
}
|
||||
Ok(Ok(output)) => output,
|
||||
};
|
||||
let etag = output.e_tag().map(str::to_owned);
|
||||
match output.body.collect().await {
|
||||
Ok(body) => Redirect::No(Reply::Response {
|
||||
status: 200,
|
||||
etag,
|
||||
retry_after: None,
|
||||
body: Some(body.into_bytes().to_vec()),
|
||||
}),
|
||||
Err(error) => Redirect::No(Reply::Transport {
|
||||
reason: format!("reading {} failed: {error}", self.display()),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn display(&self) -> String {
|
||||
format!("s3://{}/{}", self.object.bucket, self.object.key)
|
||||
}
|
||||
}
|
||||
|
||||
enum Redirect {
|
||||
/// The bucket lives in another region, repeated once as boto3 does
|
||||
To(String),
|
||||
No(Reply),
|
||||
}
|
||||
|
||||
fn rejected(raw: &Response) -> Redirect {
|
||||
let status = raw.status().as_u16();
|
||||
let header = |name: &str| raw.headers().get(name).map(str::to_owned);
|
||||
match (status, header(BUCKET_REGION_HEADER)) {
|
||||
(301, Some(region)) => Redirect::To(region),
|
||||
_ => Redirect::No(Reply::Response {
|
||||
status,
|
||||
etag: header("etag"),
|
||||
retry_after: None,
|
||||
body: None,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn without_retries(client: aws_sdk_s3::Client) -> aws_sdk_s3::Client {
|
||||
aws_sdk_s3::Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(RetryConfig::disabled())
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
fn non_empty_env(name: &str) -> Option<String> {
|
||||
std::env::var(name).ok().filter(|value| !value.is_empty())
|
||||
}
|
||||
|
|
@ -1,7 +1,10 @@
|
|||
use std::fmt;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use url::Url;
|
||||
|
||||
use crate::error::SourceError;
|
||||
|
||||
/// Where a fetched catalog comes from
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum Source {
|
||||
|
|
@ -16,8 +19,84 @@ pub struct S3Object {
|
|||
pub key: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum SourceError {
|
||||
UnsupportedScheme { scheme: String },
|
||||
Invalid { reason: String },
|
||||
impl Source {
|
||||
pub fn parse(text: &str) -> Result<Self, SourceError> {
|
||||
let url = Url::parse(text).map_err(|error| SourceError::Invalid {
|
||||
reason: format!("invalid URL {text}: {error}"),
|
||||
})?;
|
||||
match url.scheme() {
|
||||
"http" | "https" => Ok(Self::Http(url)),
|
||||
"file" => url
|
||||
.to_file_path()
|
||||
.map(Self::File)
|
||||
.map_err(|()| SourceError::Invalid {
|
||||
reason: format!("invalid file URL {text}"),
|
||||
}),
|
||||
"s3" => {
|
||||
let bucket = url.host_str().unwrap_or_default();
|
||||
let key = url.path().trim_start_matches('/');
|
||||
if bucket.is_empty() || key.is_empty() {
|
||||
return Err(SourceError::Invalid {
|
||||
reason: format!("invalid S3 URL {text}: expected s3://bucket/key"),
|
||||
});
|
||||
}
|
||||
Ok(Self::S3(S3Object {
|
||||
bucket: bucket.to_owned(),
|
||||
key: key.to_owned(),
|
||||
}))
|
||||
}
|
||||
scheme => Err(SourceError::UnsupportedScheme {
|
||||
scheme: scheme.to_owned(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Source {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Http(url) => write!(formatter, "{url}"),
|
||||
Self::File(path) => write!(formatter, "file://{}", path.display()),
|
||||
Self::S3(object) => write!(formatter, "s3://{}/{}", object.bucket, object.key),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
#[case::https("https://example.com/map.json", Source::Http(Url::parse("https://example.com/map.json").unwrap()))]
|
||||
#[case::file("file:///tmp/map.json", Source::File(PathBuf::from("/tmp/map.json")))]
|
||||
#[case::s3(
|
||||
"s3://bucket/prices/map.json",
|
||||
Source::S3(S3Object { bucket: "bucket".into(), key: "prices/map.json".into() })
|
||||
)]
|
||||
fn parses_each_supported_scheme(#[case] text: &str, #[case] expected: Source) {
|
||||
assert_eq!(Source::parse(text), Ok(expected));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::no_key("s3://bucket")]
|
||||
#[case::no_bucket("s3:///key")]
|
||||
#[case::not_a_url("http://[bad/model_prices.json")]
|
||||
fn rejects_an_unusable_url(#[case] text: &str) {
|
||||
assert!(matches!(
|
||||
Source::parse(text),
|
||||
Err(SourceError::Invalid { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn rejects_an_unknown_scheme() {
|
||||
assert_eq!(
|
||||
Source::parse("ftp://example.com/map.json"),
|
||||
Err(SourceError::UnsupportedScheme {
|
||||
scheme: "ftp".into()
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,6 +38,8 @@ pub enum Step {
|
|||
/// 304, a final status, or retries exhausted
|
||||
Done {
|
||||
reason: String,
|
||||
/// The wait before the next attempt when the last failure was retryable, so a caller can retry later
|
||||
retry_after: Option<Duration>,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
|
|||
197
litellm-rust/crates/model-catalog-refresh/tests/sources.rs
Normal file
197
litellm-rust/crates/model-catalog-refresh/tests/sources.rs
Normal file
|
|
@ -0,0 +1,197 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use aws_credential_types::Credentials;
|
||||
use aws_sdk_s3::config::SharedCredentialsProvider;
|
||||
use litellm_model_catalog::BUNDLED_REGISTRY;
|
||||
use litellm_model_catalog_refresh::{
|
||||
Installed, Origin, RetryPolicy, S3Object, S3Source, Source, Step, Transports, fetch_with_retry,
|
||||
git_blob_id,
|
||||
};
|
||||
use litellm_s3::S3ClientConfig;
|
||||
use rstest::rstest;
|
||||
use wiremock::matchers::{header, method, path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
fn installed(origin: Origin, etag: Option<&str>) -> Installed {
|
||||
Installed {
|
||||
origin,
|
||||
etag: etag.map(str::to_owned),
|
||||
revision: git_blob_id(BUNDLED_REGISTRY),
|
||||
model_count: 0,
|
||||
}
|
||||
}
|
||||
|
||||
async fn no_sleep(_: Duration) {}
|
||||
|
||||
fn s3_source(server: &MockServer, object: S3Object) -> S3Source {
|
||||
S3Source::new(
|
||||
object,
|
||||
litellm_http::Client::plain_for_test(),
|
||||
SharedCredentialsProvider::new(Credentials::new("key", "secret", None, None, "test")),
|
||||
S3ClientConfig {
|
||||
region: "us-east-1".into(),
|
||||
endpoint: Some(server.uri()),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn http_retries_a_503_then_adopts_the_map_with_its_etag() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/map.json"))
|
||||
.respond_with(ResponseTemplate::new(503))
|
||||
.up_to_n_times(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/map.json"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("ETag", "W/\"v2\"")
|
||||
.set_body_bytes(BUNDLED_REGISTRY),
|
||||
)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let source = Source::parse(&format!("{}/map.json", server.uri())).unwrap();
|
||||
let http = litellm_http::Client::plain_for_test();
|
||||
|
||||
let step = fetch_with_retry(
|
||||
&source,
|
||||
&installed(Origin::Bundled, None),
|
||||
RetryPolicy::default(),
|
||||
Some(0),
|
||||
&Transports {
|
||||
http: &http,
|
||||
s3: None,
|
||||
},
|
||||
no_sleep,
|
||||
)
|
||||
.await;
|
||||
|
||||
let Step::Adopt { installed, .. } = step else {
|
||||
panic!("expected adopt, got {step:?}");
|
||||
};
|
||||
assert_eq!(installed.etag.as_deref(), Some("W/\"v2\""));
|
||||
assert_eq!(server.received_requests().await.unwrap().len(), 2);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn s3_retries_slow_down_and_sends_if_none_match() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/bucket/prices/map.json"))
|
||||
.and(header("if-none-match", "\"v1\""))
|
||||
.respond_with(ResponseTemplate::new(503).set_body_string(
|
||||
"<Error><Code>SlowDown</Code><Message>Please reduce your request rate.</Message></Error>",
|
||||
))
|
||||
.up_to_n_times(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/bucket/prices/map.json"))
|
||||
.and(header("if-none-match", "\"v1\""))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.insert_header("ETag", "\"v2\"")
|
||||
.set_body_bytes(BUNDLED_REGISTRY),
|
||||
)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let object = S3Object {
|
||||
bucket: "bucket".into(),
|
||||
key: "prices/map.json".into(),
|
||||
};
|
||||
let s3 = s3_source(&server, object.clone());
|
||||
let source = Source::S3(object);
|
||||
let http = litellm_http::Client::plain_for_test();
|
||||
|
||||
let step = fetch_with_retry(
|
||||
&source,
|
||||
&installed(
|
||||
Origin::Remote {
|
||||
source: source.clone(),
|
||||
},
|
||||
Some("\"v1\""),
|
||||
),
|
||||
RetryPolicy::default(),
|
||||
Some(0),
|
||||
&Transports {
|
||||
http: &http,
|
||||
s3: Some(&s3),
|
||||
},
|
||||
no_sleep,
|
||||
)
|
||||
.await;
|
||||
|
||||
let Step::Adopt { installed, .. } = step else {
|
||||
panic!("expected adopt, got {step:?}");
|
||||
};
|
||||
assert_eq!(installed.etag.as_deref(), Some("\"v2\""));
|
||||
assert_eq!(server.received_requests().await.unwrap().len(), 2);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn s3_not_found_is_final() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(404).set_body_string("<Error><Code>NoSuchKey</Code></Error>"),
|
||||
)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let object = S3Object {
|
||||
bucket: "bucket".into(),
|
||||
key: "missing.json".into(),
|
||||
};
|
||||
let s3 = s3_source(&server, object.clone());
|
||||
let http = litellm_http::Client::plain_for_test();
|
||||
|
||||
let step = fetch_with_retry(
|
||||
&Source::S3(object),
|
||||
&installed(Origin::Bundled, None),
|
||||
RetryPolicy::default(),
|
||||
Some(0),
|
||||
&Transports {
|
||||
http: &http,
|
||||
s3: Some(&s3),
|
||||
},
|
||||
no_sleep,
|
||||
)
|
||||
.await;
|
||||
|
||||
let Step::Done { reason, .. } = step else {
|
||||
panic!("expected done, got {step:?}");
|
||||
};
|
||||
assert_eq!(reason, "HTTP 404 from s3://bucket/missing.json");
|
||||
assert_eq!(server.received_requests().await.unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_file_source_is_read_without_a_transport() {
|
||||
let directory = std::env::temp_dir().join(format!("catalog-refresh-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&directory).unwrap();
|
||||
let file = directory.join("map.json");
|
||||
std::fs::write(&file, BUNDLED_REGISTRY).unwrap();
|
||||
let http = litellm_http::Client::plain_for_test();
|
||||
|
||||
let step = fetch_with_retry(
|
||||
&Source::File(file.clone()),
|
||||
&installed(Origin::Bundled, None),
|
||||
RetryPolicy::default(),
|
||||
Some(0),
|
||||
&Transports {
|
||||
http: &http,
|
||||
s3: None,
|
||||
},
|
||||
no_sleep,
|
||||
)
|
||||
.await;
|
||||
|
||||
std::fs::remove_dir_all(&directory).unwrap();
|
||||
assert!(matches!(step, Step::Adopt { .. }), "{step:?}");
|
||||
}
|
||||
|
|
@ -49,8 +49,8 @@ litellm-llms-types.workspace = true
|
|||
litellm-router-types.workspace = true
|
||||
litellm-host-python.workspace = true
|
||||
litellm-model-catalog.workspace = true
|
||||
litellm-model-catalog-refresh.workspace = true
|
||||
indexmap = "2.14.0"
|
||||
sha1.workspace = true
|
||||
litellm-token-counter = { path = "../token-counter", default-features = false }
|
||||
pyo3.workspace = true
|
||||
prost.workspace = true
|
||||
|
|
|
|||
|
|
@ -34,7 +34,9 @@ mod _native {
|
|||
#[pymodule_export]
|
||||
use crate::logger::NativeDiagnosticProcessor;
|
||||
#[pymodule_export]
|
||||
use crate::model_registry::{CatalogHandle, Fetched, Installed, RowError, Unavailable};
|
||||
use crate::model_registry::{
|
||||
Bundled, CatalogHandle, Fetched, File, Installed, Provenance, Remote, RowError, Unavailable,
|
||||
};
|
||||
#[pymodule_export]
|
||||
use crate::routes::audio_transcription::{atranscription, transcription};
|
||||
#[pymodule_export]
|
||||
|
|
@ -157,6 +159,10 @@ mod tests {
|
|||
"Fetched",
|
||||
"Installed",
|
||||
"Unavailable",
|
||||
"Remote",
|
||||
"File",
|
||||
"Bundled",
|
||||
"Provenance",
|
||||
"RowError",
|
||||
"RowErrorKind",
|
||||
"NativeClickHouseSpendConfig",
|
||||
|
|
|
|||
|
|
@ -1,19 +1,21 @@
|
|||
mod fetch;
|
||||
mod install;
|
||||
mod row_error;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, LazyLock, Mutex, PoisonError};
|
||||
use std::time::SystemTime;
|
||||
|
||||
use litellm_host_python::{release_gil, to_py};
|
||||
use litellm_model_catalog::wire::Fields;
|
||||
use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog, IntegrityLimits, Layers};
|
||||
use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog, Layers, Located};
|
||||
use litellm_model_catalog_refresh::{Source, git_blob_id};
|
||||
use pyo3::exceptions::PyTypeError;
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::pybacked::PyBackedBytes;
|
||||
use pyo3::types::{PyDict, PyModule};
|
||||
use serde::Serialize;
|
||||
use sha1::{Digest, Sha1};
|
||||
|
||||
pub(crate) use fetch::{Bundled, File, Provenance, Remote};
|
||||
use install::{Record, write_into};
|
||||
pub(crate) use row_error::{RowError, RowErrorKind, row_error};
|
||||
|
||||
|
|
@ -30,7 +32,7 @@ static BUNDLED: LazyLock<Arc<Parsed>> = LazyLock::new(|| {
|
|||
|
||||
static BUNDLED_REVISION: LazyLock<String> = LazyLock::new(|| git_blob_id(BUNDLED_REGISTRY));
|
||||
|
||||
struct Parsed {
|
||||
pub(crate) struct Parsed {
|
||||
/// What install layers over bundled, rows decision 15 rejects replaced or left out
|
||||
catalog: Arc<Catalog>,
|
||||
/// The document as written
|
||||
|
|
@ -46,6 +48,10 @@ pub(crate) struct Fetched {
|
|||
revision: String,
|
||||
#[pyo3(get)]
|
||||
etag: Option<String>,
|
||||
/// `None` for the bundled map
|
||||
source: Option<Source>,
|
||||
/// When the load that produced this map started, main's `loaded_at`
|
||||
started_at: SystemTime,
|
||||
}
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
|
|
@ -68,11 +74,15 @@ pub(crate) struct Installed {
|
|||
pub(crate) struct Unavailable {
|
||||
#[pyo3(get)]
|
||||
reason: String,
|
||||
/// Seconds to wait before retrying, set when the last reply was retryable
|
||||
#[pyo3(get)]
|
||||
retry_after: Option<f64>,
|
||||
}
|
||||
|
||||
struct State {
|
||||
layers: Layers,
|
||||
record: Option<Record>,
|
||||
provenance: Option<Provenance>,
|
||||
}
|
||||
|
||||
#[pyclass(frozen, name = "Catalog", module = "litellm.rust_bridge._native")]
|
||||
|
|
@ -117,6 +127,18 @@ impl Fetched {
|
|||
}
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl Unavailable {
|
||||
#[new]
|
||||
#[pyo3(signature = (reason, retry_after = None))]
|
||||
fn new(reason: String, retry_after: Option<f64>) -> Self {
|
||||
Self {
|
||||
reason,
|
||||
retry_after,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl CatalogHandle {
|
||||
#[staticmethod]
|
||||
|
|
@ -127,49 +149,58 @@ impl CatalogHandle {
|
|||
state: Mutex::new(State {
|
||||
layers: Layers::new(Arc::clone(&BUNDLED.catalog)),
|
||||
record: None,
|
||||
provenance: None,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn bundled(&self) -> Fetched {
|
||||
Fetched {
|
||||
parsed: Arc::clone(&BUNDLED),
|
||||
revision: BUNDLED_REVISION.clone(),
|
||||
etag: None,
|
||||
}
|
||||
bundled()
|
||||
}
|
||||
|
||||
/// Parses a fetched body and applies main's whole-map checks against the bundled model count
|
||||
#[staticmethod]
|
||||
#[pyo3(signature = (body, *, etag, min_model_count, max_shrink_ratio))]
|
||||
fn parse(
|
||||
/// Fetches and validates `source` with no install, retrying up to `attempts` times
|
||||
#[pyo3(signature = (source, *, timeout, attempts))]
|
||||
fn fetch(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
body: PyBackedBytes,
|
||||
etag: Option<String>,
|
||||
min_model_count: usize,
|
||||
max_shrink_ratio: f64,
|
||||
source: &Bound<'_, PyAny>,
|
||||
timeout: f64,
|
||||
attempts: u32,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
let parsed = release_gil(py, || {
|
||||
parse_remote(&body, min_model_count, max_shrink_ratio)
|
||||
});
|
||||
match parsed {
|
||||
Ok((parsed, revision)) => Ok(Fetched {
|
||||
parsed: Arc::new(parsed),
|
||||
revision,
|
||||
etag,
|
||||
}
|
||||
fetch::fetch(py, fetch::requested(source)?, timeout, attempts)
|
||||
}
|
||||
|
||||
/// Fetch with main's reload policy, 3 attempts of 5 s, then install
|
||||
fn reload(&self, py: Python<'_>, source: &Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
|
||||
let requested = fetch::requested(source)?;
|
||||
let env_forced = matches!(requested, fetch::Requested::Bundled);
|
||||
let fetched = fetch::fetch(py, requested, 5.0, 3)?;
|
||||
let Ok(fetched) = fetched.bind(py).cast::<Fetched>() else {
|
||||
return Ok(fetched);
|
||||
};
|
||||
Ok(self
|
||||
.install(py, fetched.get(), env_forced, None, None)?
|
||||
.into_pyobject(py)?
|
||||
.into_any()
|
||||
.unbind()),
|
||||
Err(reason) => Ok(Unavailable { reason }
|
||||
.into_pyobject(py)?
|
||||
.into_any()
|
||||
.unbind()),
|
||||
}
|
||||
.unbind())
|
||||
}
|
||||
|
||||
/// Writes the composed rows into the dict bound to the attribute, never over a field the user owns (decision 25)
|
||||
fn install(&self, py: Python<'_>, fetched: &Fetched) -> PyResult<Installed> {
|
||||
/// Where the installed map came from, `None` before the first install
|
||||
fn provenance(&self) -> Option<Provenance> {
|
||||
self.lock().provenance.clone()
|
||||
}
|
||||
|
||||
/// Writes the composed rows into the dict bound to the attribute, never over a field the user owns (decision 25).
|
||||
/// `env_forced`, `fallback_reason` and `url` are the provenance facts only the caller knows
|
||||
#[pyo3(signature = (fetched, *, env_forced = false, fallback_reason = None, url = None))]
|
||||
fn install(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
fetched: &Fetched,
|
||||
env_forced: bool,
|
||||
fallback_reason: Option<String>,
|
||||
url: Option<String>,
|
||||
) -> PyResult<Installed> {
|
||||
let bound = self.module.bind(py).getattr(self.attr.as_str())?;
|
||||
let dict = bound
|
||||
.cast::<PyDict>()
|
||||
|
|
@ -188,7 +219,12 @@ impl CatalogHandle {
|
|||
};
|
||||
let composed = release_gil(py, || layers.compose());
|
||||
let outcome = write_into(py, dict, &composed, &layers, record)?;
|
||||
self.lock().record = Some(outcome.record);
|
||||
let provenance = Provenance::of(fetched, env_forced, fallback_reason, url);
|
||||
{
|
||||
let mut state = self.lock();
|
||||
state.record = Some(outcome.record);
|
||||
state.provenance = Some(provenance);
|
||||
}
|
||||
Ok(Installed {
|
||||
count: composed.len(),
|
||||
written: outcome.written,
|
||||
|
|
@ -204,47 +240,29 @@ impl CatalogHandle {
|
|||
}
|
||||
}
|
||||
|
||||
fn parse_remote(
|
||||
body: &[u8],
|
||||
min_model_count: usize,
|
||||
max_shrink_ratio: f64,
|
||||
) -> Result<(Parsed, String), String> {
|
||||
let limits = IntegrityLimits::new(min_model_count, max_shrink_ratio)
|
||||
.ok_or_else(|| format!("invalid shrink ratio {max_shrink_ratio}"))?;
|
||||
let loaded = Catalog::parse(body).map_err(|error| error.to_string())?;
|
||||
loaded
|
||||
.value
|
||||
.check_size(BUNDLED.raw.model_count(), limits)
|
||||
.map_err(|error| error.to_string())?;
|
||||
let raw = Arc::new(loaded.value.clone());
|
||||
let errors = loaded.errors.iter().map(row_error).collect();
|
||||
let (catalog, rejections) = loaded
|
||||
.value
|
||||
.with_rejected_replaced(&loaded.errors, &BUNDLED.raw);
|
||||
let rejected = rejections
|
||||
.iter()
|
||||
.flat_map(|rejection| rejection.errors.iter().copied().map(row_error))
|
||||
.collect();
|
||||
Ok((
|
||||
Parsed {
|
||||
catalog: Arc::new(catalog),
|
||||
raw,
|
||||
errors,
|
||||
rejected,
|
||||
},
|
||||
git_blob_id(body),
|
||||
))
|
||||
pub(crate) fn bundled() -> Fetched {
|
||||
Fetched {
|
||||
parsed: Arc::clone(&BUNDLED),
|
||||
revision: BUNDLED_REVISION.clone(),
|
||||
etag: None,
|
||||
source: None,
|
||||
started_at: SystemTime::now(),
|
||||
}
|
||||
}
|
||||
|
||||
fn git_blob_id(body: &[u8]) -> String {
|
||||
let mut hasher = Sha1::new();
|
||||
hasher.update(format!("blob {}\0", body.len()).as_bytes());
|
||||
hasher.update(body);
|
||||
hasher
|
||||
.finalize()
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect()
|
||||
/// A fetched document with the rows decision 15 rejects replaced by bundled or left out
|
||||
pub(crate) fn remote(catalog: Catalog, errors: &[Located]) -> Parsed {
|
||||
let raw = Arc::new(catalog.clone());
|
||||
let (catalog, rejections) = catalog.with_rejected_replaced(errors, &BUNDLED.raw);
|
||||
Parsed {
|
||||
catalog: Arc::new(catalog),
|
||||
raw,
|
||||
errors: errors.iter().map(row_error).collect(),
|
||||
rejected: rejections
|
||||
.iter()
|
||||
.flat_map(|rejection| rejection.errors.iter().copied().map(row_error))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn served_rows(catalog: &Catalog) -> Vec<(String, String, Fields)> {
|
||||
|
|
@ -281,18 +299,3 @@ impl Serialize for WrittenDocument<'_> {
|
|||
serializer.collect_map(self.0.written())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
fn the_revision_is_the_git_blob_id_of_the_body() {
|
||||
assert_eq!(
|
||||
git_blob_id(b"hello\n"),
|
||||
"ce013625030ba8dba906f756967f9e9ca394464a"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
307
litellm-rust/crates/python-bridge/src/model_registry/fetch.rs
Normal file
307
litellm-rust/crates/python-bridge/src/model_registry/fetch.rs
Normal file
|
|
@ -0,0 +1,307 @@
|
|||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use litellm_host_python::release_gil;
|
||||
use litellm_http::media::PublicDnsResolver;
|
||||
use litellm_http::{ClientVariant, HttpClientConfig, HttpClientPool};
|
||||
use litellm_model_catalog::IntegrityLimits;
|
||||
use litellm_model_catalog_refresh::{
|
||||
Installed, Origin, RetryPolicy, Source, Step, Transports, fetch_with_retry,
|
||||
};
|
||||
use pyo3::exceptions::{PyTypeError, PyValueError};
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::PyDict;
|
||||
|
||||
use super::{BUNDLED, BUNDLED_REVISION, Fetched, Parsed, Unavailable, bundled, remote};
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct Remote {
|
||||
#[pyo3(get)]
|
||||
url: String,
|
||||
}
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct File {
|
||||
#[pyo3(get)]
|
||||
path: String,
|
||||
}
|
||||
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct Bundled;
|
||||
|
||||
/// One snapshot of where the installed map came from, replacing main's field-by-field `ModelCostMapSourceInfo`
|
||||
#[pyclass(frozen, skip_from_py_object, module = "litellm.rust_bridge._native")]
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct Provenance {
|
||||
/// `"local"` or `"remote"`, main's values
|
||||
#[pyo3(get)]
|
||||
source: &'static str,
|
||||
#[pyo3(get)]
|
||||
url: Option<String>,
|
||||
#[pyo3(get)]
|
||||
is_env_forced: bool,
|
||||
#[pyo3(get)]
|
||||
fallback_reason: Option<String>,
|
||||
loaded_at: SystemTime,
|
||||
#[pyo3(get)]
|
||||
source_revision: String,
|
||||
#[pyo3(get)]
|
||||
etag: Option<String>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl Remote {
|
||||
#[new]
|
||||
fn new(url: String) -> Self {
|
||||
Self { url }
|
||||
}
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl File {
|
||||
#[new]
|
||||
fn new(path: PathBuf) -> Self {
|
||||
Self {
|
||||
path: path.to_string_lossy().into_owned(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl Bundled {
|
||||
#[new]
|
||||
fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl Provenance {
|
||||
/// When the load that installed this map started, as an aware UTC datetime
|
||||
#[getter]
|
||||
fn loaded_at<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
|
||||
let seconds = self
|
||||
.loaded_at
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs_f64();
|
||||
let datetime = py.import("datetime")?;
|
||||
let utc = datetime.getattr("timezone")?.getattr("utc")?;
|
||||
datetime
|
||||
.getattr("datetime")?
|
||||
.call_method1("fromtimestamp", (seconds, utc))
|
||||
}
|
||||
}
|
||||
|
||||
impl Provenance {
|
||||
pub(crate) fn of(
|
||||
fetched: &Fetched,
|
||||
env_forced: bool,
|
||||
fallback_reason: Option<String>,
|
||||
url: Option<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
source: if fetched.source.is_some() {
|
||||
"remote"
|
||||
} else {
|
||||
"local"
|
||||
},
|
||||
url: url.or_else(|| fetched.source.as_ref().map(ToString::to_string)),
|
||||
is_env_forced: env_forced,
|
||||
fallback_reason,
|
||||
loaded_at: fetched.started_at,
|
||||
source_revision: fetched.revision.clone(),
|
||||
etag: fetched.etag.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What a Python `Source` names
|
||||
pub(crate) enum Requested {
|
||||
Bundled,
|
||||
Source(Source),
|
||||
Invalid(String),
|
||||
}
|
||||
|
||||
pub(crate) fn requested(source: &Bound<'_, PyAny>) -> PyResult<Requested> {
|
||||
if source.is_instance_of::<Bundled>() {
|
||||
return Ok(Requested::Bundled);
|
||||
}
|
||||
if let Ok(remote) = source.cast::<Remote>() {
|
||||
let url = &remote.get().url;
|
||||
return Ok(Source::parse(url).map_or_else(
|
||||
|error| Requested::Invalid(error.to_string()),
|
||||
Requested::Source,
|
||||
));
|
||||
}
|
||||
if let Ok(file) = source.cast::<File>() {
|
||||
return Ok(Requested::Source(Source::File(PathBuf::from(
|
||||
&file.get().path,
|
||||
))));
|
||||
}
|
||||
Err(PyTypeError::new_err(
|
||||
"source must be Remote, File or Bundled",
|
||||
))
|
||||
}
|
||||
|
||||
enum Outcome {
|
||||
Fetched {
|
||||
parsed: Parsed,
|
||||
revision: String,
|
||||
etag: Option<String>,
|
||||
},
|
||||
Unavailable {
|
||||
reason: String,
|
||||
retry_after: Option<Duration>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Fetches with the GIL released, retrying under main's policy, and validates the body before returning it
|
||||
pub(crate) fn fetch(
|
||||
py: Python<'_>,
|
||||
requested: Requested,
|
||||
timeout: f64,
|
||||
attempts: u32,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
let source = match requested {
|
||||
Requested::Bundled => return Ok(bundled().into_pyobject(py)?.into_any().unbind()),
|
||||
Requested::Invalid(reason) => return unavailable(py, reason, None),
|
||||
Requested::Source(source) => source,
|
||||
};
|
||||
let timeout = Duration::try_from_secs_f64(timeout)
|
||||
.map_err(|_| PyValueError::new_err("timeout must be a non-negative number of seconds"))?;
|
||||
let policy = RetryPolicy {
|
||||
max_attempts: attempts.max(1),
|
||||
timeout,
|
||||
limits: integrity_limits(py)?,
|
||||
..RetryPolicy::default()
|
||||
};
|
||||
let config = crate::http::call_config(py, &PyDict::new(py), false)?;
|
||||
let started_at = SystemTime::now();
|
||||
let fetched_from = source.clone();
|
||||
match release_gil(py, move || drive(&source, policy, config)) {
|
||||
Outcome::Fetched {
|
||||
parsed,
|
||||
revision,
|
||||
etag,
|
||||
} => Ok(Fetched {
|
||||
parsed: Arc::new(parsed),
|
||||
revision,
|
||||
etag,
|
||||
source: Some(fetched_from),
|
||||
started_at,
|
||||
}
|
||||
.into_pyobject(py)?
|
||||
.into_any()
|
||||
.unbind()),
|
||||
Outcome::Unavailable {
|
||||
reason,
|
||||
retry_after,
|
||||
} => unavailable(py, reason, retry_after),
|
||||
}
|
||||
}
|
||||
|
||||
fn unavailable(
|
||||
py: Python<'_>,
|
||||
reason: String,
|
||||
retry_after: Option<Duration>,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
Ok(Unavailable {
|
||||
reason,
|
||||
retry_after: retry_after.map(|wait| wait.as_secs_f64()),
|
||||
}
|
||||
.into_pyobject(py)?
|
||||
.into_any()
|
||||
.unbind())
|
||||
}
|
||||
|
||||
/// Main's `MODEL_COST_MAP_MIN_MODEL_COUNT` and `MODEL_COST_MAP_MAX_SHRINK_RATIO`, which are env-overridable
|
||||
fn integrity_limits(py: Python<'_>) -> PyResult<IntegrityLimits> {
|
||||
let constants = py.import("litellm.constants")?;
|
||||
let minimum: usize = constants
|
||||
.getattr("MODEL_COST_MAP_MIN_MODEL_COUNT")?
|
||||
.extract()?;
|
||||
let ratio: f64 = constants
|
||||
.getattr("MODEL_COST_MAP_MAX_SHRINK_RATIO")?
|
||||
.extract()?;
|
||||
IntegrityLimits::new(minimum, ratio).ok_or_else(|| {
|
||||
PyValueError::new_err("MODEL_COST_MAP_MAX_SHRINK_RATIO must be between 0 and 1")
|
||||
})
|
||||
}
|
||||
|
||||
/// A current-thread runtime of its own, so a fetch at import never starts the shared native runtime, and a client
|
||||
/// built inside it, so no pooled connection outlives the runtime that spawned it
|
||||
fn drive(source: &Source, policy: RetryPolicy, config: HttpClientConfig) -> Outcome {
|
||||
if tokio::runtime::Handle::try_current().is_ok() {
|
||||
return Outcome::Unavailable {
|
||||
reason: "the model cost map fetch cannot run from a Tokio context".to_owned(),
|
||||
retry_after: None,
|
||||
};
|
||||
}
|
||||
let runtime = match tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
{
|
||||
Ok(runtime) => runtime,
|
||||
Err(error) => {
|
||||
return Outcome::Unavailable {
|
||||
reason: format!("the model cost map fetch runtime could not start: {error}"),
|
||||
retry_after: None,
|
||||
};
|
||||
}
|
||||
};
|
||||
let step = runtime.block_on(async {
|
||||
let pool = HttpClientPool::new(Arc::new(PublicDnsResolver));
|
||||
let client = match pool.client(&config, ClientVariant::Provider) {
|
||||
Ok(client) => client,
|
||||
Err(error) => {
|
||||
return Step::Done {
|
||||
reason: format!("the HTTP client could not be built: {error}"),
|
||||
retry_after: None,
|
||||
};
|
||||
}
|
||||
};
|
||||
let installed = Installed {
|
||||
origin: Origin::Bundled,
|
||||
etag: None,
|
||||
revision: BUNDLED_REVISION.clone(),
|
||||
model_count: BUNDLED.raw.model_count(),
|
||||
};
|
||||
fetch_with_retry(
|
||||
source,
|
||||
&installed,
|
||||
policy,
|
||||
None,
|
||||
&Transports {
|
||||
http: &client,
|
||||
s3: None,
|
||||
},
|
||||
tokio::time::sleep,
|
||||
)
|
||||
.await
|
||||
});
|
||||
runtime.shutdown_timeout(Duration::from_secs(1));
|
||||
match step {
|
||||
Step::Adopt {
|
||||
catalog,
|
||||
installed,
|
||||
errors,
|
||||
} => Outcome::Fetched {
|
||||
parsed: remote(*catalog, &errors),
|
||||
revision: installed.revision,
|
||||
etag: installed.etag,
|
||||
},
|
||||
Step::Done {
|
||||
reason,
|
||||
retry_after,
|
||||
} => Outcome::Unavailable {
|
||||
reason,
|
||||
retry_after,
|
||||
},
|
||||
Step::Fetch(_) | Step::Sleep(_) => Outcome::Unavailable {
|
||||
reason: "the model cost map fetch stopped before it finished".to_owned(),
|
||||
retry_after: None,
|
||||
},
|
||||
}
|
||||
}
|
||||
15
litellm-rust/crates/s3/Cargo.toml
Normal file
15
litellm-rust/crates/s3/Cargo.toml
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
[package]
|
||||
name = "litellm-s3"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
aws-credential-types = "1.3.0"
|
||||
aws-sdk-s3 = { version = "1.146.1", default-features = false, features = ["rustls", "rt-tokio"] }
|
||||
aws-smithy-runtime-api = { version = "1.16.2", features = ["client", "http-1x"] }
|
||||
aws-smithy-types = { version = "1.6.0", features = ["http-body-1-x"] }
|
||||
http.workspace = true
|
||||
litellm-http.workspace = true
|
||||
reqwest.workspace = true
|
||||
35
litellm-rust/crates/s3/src/lib.rs
Normal file
35
litellm-rust/crates/s3/src/lib.rs
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
mod transport;
|
||||
|
||||
use aws_credential_types::provider::ProvideCredentials;
|
||||
use aws_sdk_s3::config::{
|
||||
BehaviorVersion, Region, RequestChecksumCalculation, ResponseChecksumValidation,
|
||||
};
|
||||
|
||||
pub use transport::ReqwestHttpClient;
|
||||
|
||||
/// Where an S3 client sends its requests
|
||||
pub struct S3ClientConfig {
|
||||
pub region: String,
|
||||
/// An S3-compatible endpoint (MinIO, R2, Ceph), addressed path-style
|
||||
pub endpoint: Option<String>,
|
||||
}
|
||||
|
||||
/// One way to build an S3 client over a reqwest client, shared by every S3 caller
|
||||
pub fn client(
|
||||
config: S3ClientConfig,
|
||||
http: litellm_http::Client,
|
||||
credentials: impl ProvideCredentials + 'static,
|
||||
) -> aws_sdk_s3::Client {
|
||||
let base = aws_sdk_s3::Config::builder()
|
||||
.behavior_version(BehaviorVersion::latest())
|
||||
.region(Region::new(config.region))
|
||||
.http_client(ReqwestHttpClient(http))
|
||||
.credentials_provider(credentials)
|
||||
.request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
|
||||
.response_checksum_validation(ResponseChecksumValidation::WhenRequired);
|
||||
let builder = match config.endpoint {
|
||||
Some(url) => base.endpoint_url(url).force_path_style(true),
|
||||
None => base,
|
||||
};
|
||||
aws_sdk_s3::Client::from_conf(builder.build())
|
||||
}
|
||||
|
|
@ -9,7 +9,7 @@ use aws_smithy_runtime_api::client::{
|
|||
use aws_smithy_types::body::SdkBody;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct ReqwestHttpClient(pub(crate) litellm_http::Client);
|
||||
pub struct ReqwestHttpClient(pub litellm_http::Client);
|
||||
|
||||
impl HttpClient for ReqwestHttpClient {
|
||||
fn http_connector(
|
||||
|
|
@ -1,10 +1,47 @@
|
|||
"""Internal catalog API backed by the Rust ``model-catalog`` crate. Not public, its shape may change in any release"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from typing import Final
|
||||
|
||||
from litellm.rust_bridge._native import Catalog, Fetched, Installed, RowError, Unavailable
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
from litellm.rust_bridge._native import (
|
||||
Bundled,
|
||||
Catalog,
|
||||
Fetched,
|
||||
File,
|
||||
Installed,
|
||||
Provenance,
|
||||
Remote,
|
||||
RowError,
|
||||
Unavailable,
|
||||
)
|
||||
|
||||
Source = Remote | File | Bundled
|
||||
|
||||
CATALOG: Final = Catalog.bound_to(sys.modules["litellm"], "model_cost")
|
||||
|
||||
__all__ = ["CATALOG", "Catalog", "Fetched", "Installed", "RowError", "Unavailable"]
|
||||
|
||||
def default_source() -> Source:
|
||||
"""Main's reload rule, the bundled map under offline mode or ``LITELLM_LOCAL_MODEL_COST_MAP``"""
|
||||
import litellm
|
||||
|
||||
if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
|
||||
return Bundled()
|
||||
return Remote(litellm.model_cost_map_url)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CATALOG",
|
||||
"Bundled",
|
||||
"Catalog",
|
||||
"Fetched",
|
||||
"File",
|
||||
"Installed",
|
||||
"Provenance",
|
||||
"Remote",
|
||||
"RowError",
|
||||
"Source",
|
||||
"Unavailable",
|
||||
"default_source",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -9,31 +9,23 @@ export LITELLM_LOCAL_MODEL_COST_MAP=True
|
|||
```
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
import warnings
|
||||
from collections.abc import Callable, Mapping
|
||||
from datetime import datetime
|
||||
from itertools import chain
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Protocol, cast # noqa: TID251 # adapts boto3 and botocore's dynamic client responses
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm import verbose_logger
|
||||
from litellm._catalog import CATALOG, Catalog, Fetched, Installed, Unavailable
|
||||
from litellm.constants import (
|
||||
MODEL_COST_MAP_MAX_SHRINK_RATIO,
|
||||
MODEL_COST_MAP_MIN_MODEL_COUNT,
|
||||
)
|
||||
from litellm._catalog import CATALOG, Catalog, Fetched, Installed, Remote, Source, Unavailable
|
||||
from litellm.litellm_core_utils.fallback_generalizations import (
|
||||
set_fallback_generalizations,
|
||||
)
|
||||
|
|
@ -42,12 +34,25 @@ from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
|||
FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations"
|
||||
_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
|
||||
_CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"})
|
||||
MODEL_COST_MAP_FETCH_MAX_ATTEMPTS: Final = 3
|
||||
_IMPORT_FETCH_TIMEOUT_SECONDS: Final = 5
|
||||
_litellm_import_complete = threading.Event()
|
||||
|
||||
|
||||
def _is_cli_process() -> bool:
|
||||
return Path(sys.argv[0]).stem in _CLI_ENTRYPOINT_NAMES
|
||||
|
||||
|
||||
def _local_map_forced() -> bool:
|
||||
# Note: can't use get_secret_bool here — this runs during litellm.__init__
|
||||
# before litellm._key_management_settings is set.
|
||||
return is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true"
|
||||
|
||||
|
||||
def mark_litellm_import_complete() -> None:
|
||||
_litellm_import_complete.set()
|
||||
|
||||
|
||||
class GetModelCostMap:
|
||||
"""The catalog as last loaded, for readers that want it without later registrations"""
|
||||
|
||||
|
|
@ -71,304 +76,6 @@ class GetModelCostMap:
|
|||
cls._loaded_catalog_fields = frozenset(chain.from_iterable(entry.keys() for entry in raw.values()))
|
||||
|
||||
|
||||
RETRYABLE_FETCH_STATUS_CODES: Final = frozenset({429, 500, 502, 503, 504})
|
||||
MODEL_COST_MAP_FETCH_MAX_ATTEMPTS: Final = 3
|
||||
MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS: Final = 30.0
|
||||
_litellm_import_complete = threading.Event()
|
||||
|
||||
|
||||
def mark_litellm_import_complete() -> None:
|
||||
_litellm_import_complete.set()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModelCostMapReloaded:
|
||||
fetched: Fetched
|
||||
|
||||
@property
|
||||
def revision(self) -> str:
|
||||
return self.fetched.revision
|
||||
|
||||
@property
|
||||
def etag(self) -> str | None:
|
||||
return self.fetched.etag
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModelCostMapReloadUnavailable:
|
||||
reason: str
|
||||
|
||||
|
||||
ModelCostMapReloadResult = ModelCostMapReloaded | ModelCostMapReloadUnavailable
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _FetchAttemptRetryable:
|
||||
reason: str
|
||||
retry_after_seconds: float | None
|
||||
|
||||
|
||||
def _parse_retry_after_seconds(response: httpx.Response) -> float | None:
|
||||
try:
|
||||
header: Final = response.headers["Retry-After"]
|
||||
except KeyError:
|
||||
return None
|
||||
try:
|
||||
seconds: Final = float(header)
|
||||
except ValueError:
|
||||
return None
|
||||
return seconds if seconds >= 0 else None
|
||||
|
||||
|
||||
def _retry_wait_seconds(outcome: _FetchAttemptRetryable, attempt: int, rng: random.Random) -> float:
|
||||
if outcome.retry_after_seconds is not None:
|
||||
return min(outcome.retry_after_seconds, MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS)
|
||||
return min(2.0**attempt + rng.uniform(0.0, 1.0), MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS)
|
||||
|
||||
|
||||
class _AsyncGetClient(Protocol):
|
||||
def get(self, url: str, *, timeout: float | None = None) -> Awaitable[httpx.Response]: ...
|
||||
|
||||
|
||||
class _SyncGetClient(Protocol):
|
||||
def get(self, url: str, *, timeout: float | None = None) -> httpx.Response: ...
|
||||
|
||||
|
||||
class _S3Body(Protocol):
|
||||
def read(self) -> bytes: ...
|
||||
|
||||
|
||||
class _S3GetObjectResponse(TypedDict):
|
||||
Body: ReadOnly[_S3Body]
|
||||
ETag: NotRequired[ReadOnly[str]]
|
||||
|
||||
|
||||
class _S3Client(Protocol):
|
||||
def get_object(self, *, Bucket: str, Key: str) -> _S3GetObjectResponse: ...
|
||||
|
||||
|
||||
_FetchAttemptOutcome = ModelCostMapReloaded | ModelCostMapReloadUnavailable | _FetchAttemptRetryable
|
||||
|
||||
|
||||
def _default_reload_client() -> _AsyncGetClient:
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.types.llms.custom_http import httpxSpecialProvider
|
||||
|
||||
return get_async_httpx_client(llm_provider=httpxSpecialProvider.ModelCostMap)
|
||||
|
||||
|
||||
def _default_s3_client(timeout: int) -> _S3Client:
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
|
||||
return cast( # cast-ok: boto3's generic client factory does not expose its generated S3 response type
|
||||
_S3Client,
|
||||
boto3.client( # pyright: ignore[reportUnknownMemberType] # generic factory returns Unknown service clients
|
||||
"s3",
|
||||
region_name=os.getenv("AWS_REGION_NAME") or None,
|
||||
config=Config(connect_timeout=timeout, read_timeout=timeout, retries={"total_max_attempts": 1}),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) -> _FetchAttemptOutcome:
|
||||
reason: Final = f"{type(error).__name__} fetching {url}: {error}"
|
||||
if isinstance(error, (httpx.InvalidURL, httpx.UnsupportedProtocol)):
|
||||
return ModelCostMapReloadUnavailable(reason=reason)
|
||||
return _FetchAttemptRetryable(reason=reason, retry_after_seconds=None)
|
||||
|
||||
|
||||
def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _FetchAttemptOutcome:
|
||||
parsed: Final = CATALOG.parse(
|
||||
content,
|
||||
etag=etag,
|
||||
min_model_count=MODEL_COST_MAP_MIN_MODEL_COUNT,
|
||||
max_shrink_ratio=MODEL_COST_MAP_MAX_SHRINK_RATIO,
|
||||
)
|
||||
if isinstance(parsed, Unavailable):
|
||||
return ModelCostMapReloadUnavailable(reason=f"model cost map from {url} is unusable: {parsed.reason}")
|
||||
return ModelCostMapReloaded(fetched=parsed)
|
||||
|
||||
|
||||
def _split_url(url: str) -> urllib.parse.SplitResult | ModelCostMapReloadUnavailable:
|
||||
try:
|
||||
return urllib.parse.urlsplit(url)
|
||||
except ValueError as error:
|
||||
return ModelCostMapReloadUnavailable(reason=f"invalid URL {url}: {error}")
|
||||
|
||||
|
||||
def _attempt_fetch_s3(parts: urllib.parse.SplitResult, url: str, timeout: int) -> _FetchAttemptOutcome:
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
bucket: Final = parts.netloc
|
||||
key: Final = parts.path.lstrip("/")
|
||||
if not bucket or not key:
|
||||
return ModelCostMapReloadUnavailable(reason=f"invalid S3 URL {url}: expected s3://bucket/key")
|
||||
|
||||
try:
|
||||
client: Final = _default_s3_client(timeout)
|
||||
response: Final = client.get_object(Bucket=bucket, Key=key)
|
||||
content: Final = response["Body"].read()
|
||||
except (BotoCoreError, ClientError) as error:
|
||||
return ModelCostMapReloadUnavailable(reason=f"S3 {type(error).__name__} fetching {url}: {error}")
|
||||
return _classify_fetch_content(content, url, response.get("ETag"))
|
||||
|
||||
|
||||
async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
|
||||
parts: Final = _split_url(url)
|
||||
if isinstance(parts, ModelCostMapReloadUnavailable):
|
||||
return parts
|
||||
if parts.scheme == "s3":
|
||||
return await asyncio.to_thread(_attempt_fetch_s3, parts, url, timeout)
|
||||
try:
|
||||
response: Final = await client.get(url, timeout=timeout)
|
||||
except (httpx.HTTPError, httpx.InvalidURL) as e:
|
||||
return _classify_fetch_error(e, url)
|
||||
return _classify_fetch_response(response, url)
|
||||
|
||||
|
||||
def _attempt_fetch_sync(client: _SyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
|
||||
parts: Final = _split_url(url)
|
||||
if isinstance(parts, ModelCostMapReloadUnavailable):
|
||||
return parts
|
||||
if parts.scheme == "s3":
|
||||
return _attempt_fetch_s3(parts, url, timeout)
|
||||
try:
|
||||
response: Final = client.get(url, timeout=timeout)
|
||||
except (httpx.HTTPError, httpx.InvalidURL) as e:
|
||||
return _classify_fetch_error(e, url)
|
||||
return _classify_fetch_response(response, url)
|
||||
|
||||
|
||||
def _classify_fetch_response(response: httpx.Response, url: str) -> _FetchAttemptOutcome:
|
||||
if response.status_code in RETRYABLE_FETCH_STATUS_CODES:
|
||||
return _FetchAttemptRetryable(
|
||||
reason=f"HTTP {response.status_code} from {url}",
|
||||
retry_after_seconds=_parse_retry_after_seconds(response),
|
||||
)
|
||||
if response.is_error:
|
||||
return ModelCostMapReloadUnavailable(reason=f"HTTP {response.status_code} from {url}")
|
||||
etag_values: Final = response.headers.get_list("etag")
|
||||
return _classify_fetch_content(response.content, url, ", ".join(etag_values) if etag_values else None)
|
||||
|
||||
|
||||
def _next_retry_wait(
|
||||
outcome: _FetchAttemptRetryable, attempt: int, max_attempts: int, rng: random.Random
|
||||
) -> float | ModelCostMapReloadUnavailable:
|
||||
if attempt == max_attempts:
|
||||
return ModelCostMapReloadUnavailable(reason=f"{outcome.reason} (after {max_attempts} attempts)")
|
||||
wait_seconds: Final = _retry_wait_seconds(outcome=outcome, attempt=attempt, rng=rng)
|
||||
verbose_logger.warning(
|
||||
"LiteLLM: model cost map fetch attempt %d/%d failed (%s); retrying in %.1fs",
|
||||
attempt,
|
||||
max_attempts,
|
||||
outcome.reason,
|
||||
wait_seconds,
|
||||
)
|
||||
return wait_seconds
|
||||
|
||||
|
||||
async def _fetch_remote_model_cost_map_with_retry(
|
||||
url: str,
|
||||
timeout: int,
|
||||
max_attempts: int,
|
||||
sleep: Callable[[float], Awaitable[None]],
|
||||
rng: random.Random,
|
||||
client: _AsyncGetClient,
|
||||
) -> ModelCostMapReloadResult:
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
outcome = await _attempt_fetch(client=client, url=url, timeout=timeout)
|
||||
if not isinstance(outcome, _FetchAttemptRetryable):
|
||||
return outcome
|
||||
wait_seconds = _next_retry_wait(outcome=outcome, attempt=attempt, max_attempts=max_attempts, rng=rng)
|
||||
if isinstance(wait_seconds, ModelCostMapReloadUnavailable):
|
||||
return wait_seconds
|
||||
await sleep(wait_seconds)
|
||||
return ModelCostMapReloadUnavailable(reason="model cost map fetch failed")
|
||||
|
||||
|
||||
def _fetch_remote_model_cost_map_with_retry_sync(
|
||||
url: str,
|
||||
timeout: int,
|
||||
attempts: range,
|
||||
sleep: Callable[[float], None],
|
||||
rng: random.Random,
|
||||
client: _SyncGetClient,
|
||||
) -> ModelCostMapReloadResult:
|
||||
max_attempts: Final = attempts.stop - 1
|
||||
for attempt in attempts:
|
||||
outcome = _attempt_fetch_sync(client=client, url=url, timeout=timeout)
|
||||
if not isinstance(outcome, _FetchAttemptRetryable):
|
||||
return outcome
|
||||
wait_seconds = _next_retry_wait(outcome=outcome, attempt=attempt, max_attempts=max_attempts, rng=rng)
|
||||
if isinstance(wait_seconds, ModelCostMapReloadUnavailable):
|
||||
return wait_seconds
|
||||
sleep(wait_seconds)
|
||||
return ModelCostMapReloadUnavailable(reason="model cost map fetch failed")
|
||||
|
||||
|
||||
async def refetch_model_cost_map(
|
||||
url: str,
|
||||
timeout: int = 5,
|
||||
max_attempts: int = MODEL_COST_MAP_FETCH_MAX_ATTEMPTS,
|
||||
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
|
||||
rng: random.Random | None = None,
|
||||
client: "_AsyncGetClient | None" = None,
|
||||
) -> ModelCostMapReloadResult:
|
||||
"""
|
||||
Re-fetch the model cost map for a runtime reload, retrying transient HTTP
|
||||
errors (429/5xx/transport) with Retry-After-aware backoff.
|
||||
|
||||
Unlike ``get_model_cost_map`` this never falls back to the packaged backup:
|
||||
on failure it returns ``ModelCostMapReloadUnavailable`` so callers keep the
|
||||
map they already have.
|
||||
"""
|
||||
if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
|
||||
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
|
||||
_cost_map_source_info.source = "local"
|
||||
_cost_map_source_info.url = None
|
||||
_cost_map_source_info.is_env_forced = True
|
||||
_cost_map_source_info.fallback_reason = None
|
||||
return _note_loaded(ModelCostMapReloaded(fetched=CATALOG.bundled()))
|
||||
|
||||
result: Final = await _fetch_remote_model_cost_map_with_retry(
|
||||
url=url,
|
||||
timeout=timeout,
|
||||
max_attempts=max_attempts,
|
||||
sleep=sleep,
|
||||
rng=rng if rng is not None else random.Random(),
|
||||
client=client if client is not None else _default_reload_client(),
|
||||
)
|
||||
if isinstance(result, ModelCostMapReloadUnavailable):
|
||||
verbose_logger.warning(
|
||||
"LiteLLM: model cost map reload failed: %s. Keeping the currently loaded map.",
|
||||
result.reason,
|
||||
)
|
||||
return result
|
||||
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
|
||||
_cost_map_source_info.source = "remote"
|
||||
_cost_map_source_info.url = url
|
||||
_cost_map_source_info.is_env_forced = False
|
||||
_cost_map_source_info.fallback_reason = None
|
||||
return _note_loaded(result)
|
||||
|
||||
|
||||
class ModelCostMapSourceInfo:
|
||||
"""Tracks the source of the currently loaded model cost map."""
|
||||
|
||||
source: str = "local" # "local" or "remote"
|
||||
url: str | None = None
|
||||
is_env_forced: bool = False
|
||||
fallback_reason: str | None = None
|
||||
loaded_at: "datetime | None" = None
|
||||
source_revision: str | None = None
|
||||
etag: str | None = None
|
||||
|
||||
|
||||
# Module-level singleton tracking the source of the current cost map
|
||||
_cost_map_source_info: Final = ModelCostMapSourceInfo()
|
||||
|
||||
|
||||
class CostMapProvenance(TypedDict):
|
||||
source_revision: ReadOnly[str | None]
|
||||
etag: ReadOnly[str | None]
|
||||
|
|
@ -383,9 +90,10 @@ class CostMapSourceInfo(CostMapProvenance):
|
|||
|
||||
|
||||
def get_model_cost_map_provenance() -> CostMapProvenance:
|
||||
provenance: Final = CATALOG.provenance()
|
||||
return {
|
||||
"source_revision": _cost_map_source_info.source_revision,
|
||||
"etag": _cost_map_source_info.etag,
|
||||
"source_revision": provenance.source_revision if provenance is not None else None,
|
||||
"etag": provenance.etag if provenance is not None else None,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -402,21 +110,32 @@ def get_model_cost_map_source_info() -> CostMapSourceInfo:
|
|||
- source_revision: git blob id of the loaded file's bytes
|
||||
- etag: the ETag of the remote fetch (None for the bundled backup)
|
||||
"""
|
||||
loaded_at: Final = _cost_map_source_info.loaded_at
|
||||
provenance: Final = CATALOG.provenance()
|
||||
if provenance is None:
|
||||
return {
|
||||
"source": "local",
|
||||
"url": None,
|
||||
"is_env_forced": False,
|
||||
"fallback_reason": None,
|
||||
"loaded_at": None,
|
||||
"source_revision": None,
|
||||
"etag": None,
|
||||
}
|
||||
return {
|
||||
"source": _cost_map_source_info.source,
|
||||
"url": _cost_map_source_info.url,
|
||||
"is_env_forced": _cost_map_source_info.is_env_forced,
|
||||
"fallback_reason": _cost_map_source_info.fallback_reason,
|
||||
"loaded_at": loaded_at.isoformat() if loaded_at is not None else None,
|
||||
"source_revision": _cost_map_source_info.source_revision,
|
||||
"etag": _cost_map_source_info.etag,
|
||||
"source": provenance.source,
|
||||
"url": provenance.url,
|
||||
"is_env_forced": provenance.is_env_forced,
|
||||
"fallback_reason": provenance.fallback_reason,
|
||||
"loaded_at": provenance.loaded_at.isoformat(),
|
||||
"source_revision": provenance.source_revision,
|
||||
"etag": provenance.etag,
|
||||
}
|
||||
|
||||
|
||||
def get_model_cost_map_loaded_at() -> "datetime | None":
|
||||
"""When this process last loaded its cost map, stamped at the start of every load"""
|
||||
return _cost_map_source_info.loaded_at
|
||||
provenance: Final = CATALOG.provenance()
|
||||
return provenance.loaded_at if provenance is not None else None
|
||||
|
||||
|
||||
class AliasedCostEntry(dict[str, object]):
|
||||
|
|
@ -430,14 +149,12 @@ class AliasedCostEntry(dict[str, object]):
|
|||
self.registered_fields: frozenset[str] = registered_fields
|
||||
|
||||
|
||||
def _note_loaded(loaded: ModelCostMapReloaded) -> ModelCostMapReloaded:
|
||||
_cost_map_source_info.source_revision = loaded.fetched.revision
|
||||
_cost_map_source_info.etag = loaded.fetched.etag
|
||||
set_fallback_generalizations(loaded.fetched.fallback_rules())
|
||||
def _note_loaded(fetched: Fetched) -> Fetched:
|
||||
set_fallback_generalizations(fetched.fallback_rules())
|
||||
GetModelCostMap._snapshot_loaded_catalog( # pyright: ignore[reportPrivateUsage] # same module
|
||||
loaded.fetched.served()
|
||||
fetched.served()
|
||||
)
|
||||
return loaded
|
||||
return fetched
|
||||
|
||||
|
||||
class _InstallGeneration:
|
||||
|
|
@ -450,8 +167,15 @@ def model_cost_install_generation() -> int:
|
|||
return _InstallGeneration.value
|
||||
|
||||
|
||||
def _install(fetched: Fetched, catalog: Catalog = CATALOG) -> Installed:
|
||||
installed: Final = catalog.install(fetched)
|
||||
def _install(
|
||||
fetched: Fetched,
|
||||
catalog: Catalog = CATALOG,
|
||||
*,
|
||||
env_forced: bool = False,
|
||||
fallback_reason: str | None = None,
|
||||
url: str | None = None,
|
||||
) -> Installed:
|
||||
installed: Final = catalog.install(fetched, env_forced=env_forced, fallback_reason=fallback_reason, url=url)
|
||||
_InstallGeneration.value += 1
|
||||
if installed.rejected:
|
||||
verbose_logger.warning(
|
||||
|
|
@ -461,40 +185,38 @@ def _install(fetched: Fetched, catalog: Catalog = CATALOG) -> Installed:
|
|||
return installed
|
||||
|
||||
|
||||
def adopt_model_cost_map(fetched: Fetched) -> int:
|
||||
def _refresh_derived_state() -> None:
|
||||
import litellm
|
||||
from litellm import utils
|
||||
|
||||
installed: Final = _install(fetched)
|
||||
utils._invalidate_model_cost_lowercase_map() # pyright: ignore[reportPrivateUsage] # required cache invalidation
|
||||
litellm.add_known_models(model_cost_map=litellm.model_cost)
|
||||
utils.reapply_runtime_model_cost_registrations()
|
||||
return installed.count
|
||||
|
||||
|
||||
def reload_model_cost_map(source: Source, catalog: Catalog = CATALOG) -> Installed | Unavailable:
|
||||
"""Fetch with main's reload policy, then install and refresh what Python still derives from the map. A failed
|
||||
fetch keeps the map in use"""
|
||||
fetched: Final = catalog.fetch(
|
||||
source, timeout=_IMPORT_FETCH_TIMEOUT_SECONDS, attempts=MODEL_COST_MAP_FETCH_MAX_ATTEMPTS
|
||||
)
|
||||
if isinstance(fetched, Unavailable):
|
||||
verbose_logger.warning(
|
||||
"LiteLLM: model cost map reload failed: %s. Keeping the currently loaded map.", fetched.reason
|
||||
)
|
||||
return fetched
|
||||
installed: Final = _install(_note_loaded(fetched), catalog, env_forced=_local_map_forced())
|
||||
_refresh_derived_state()
|
||||
return installed
|
||||
|
||||
|
||||
def _retry_remote_fetch_in_background(
|
||||
url: str,
|
||||
timeout: int,
|
||||
max_attempts: int,
|
||||
sleep: Callable[[float], None],
|
||||
rng: random.Random,
|
||||
client: _SyncGetClient,
|
||||
first_outcome: _FetchAttemptRetryable,
|
||||
url: str, max_attempts: int, first_wait: float, sleep: Callable[[float], None], catalog: Catalog
|
||||
) -> None:
|
||||
try:
|
||||
first_wait: Final = _next_retry_wait(outcome=first_outcome, attempt=1, max_attempts=max_attempts, rng=rng)
|
||||
if isinstance(first_wait, ModelCostMapReloadUnavailable):
|
||||
return
|
||||
sleep(first_wait)
|
||||
result: Final = _fetch_remote_model_cost_map_with_retry_sync(
|
||||
url=url,
|
||||
timeout=timeout,
|
||||
attempts=range(2, max_attempts + 1),
|
||||
sleep=sleep,
|
||||
rng=rng,
|
||||
client=client,
|
||||
)
|
||||
if isinstance(result, ModelCostMapReloadUnavailable):
|
||||
fetched: Final = catalog.fetch(Remote(url), timeout=_IMPORT_FETCH_TIMEOUT_SECONDS, attempts=max_attempts - 1)
|
||||
if isinstance(fetched, Unavailable):
|
||||
verbose_logger.warning(
|
||||
"LiteLLM: Failed to fetch remote model cost map from %s after %d attempts; keeping local backup",
|
||||
url,
|
||||
|
|
@ -502,100 +224,83 @@ def _retry_remote_fetch_in_background(
|
|||
)
|
||||
return
|
||||
_litellm_import_complete.wait()
|
||||
_note_loaded(result)
|
||||
_cost_map_source_info.source = "remote"
|
||||
_cost_map_source_info.fallback_reason = None
|
||||
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
|
||||
adopt_model_cost_map(result.fetched)
|
||||
_install(_note_loaded(fetched), catalog, url=url)
|
||||
_refresh_derived_state()
|
||||
except Exception as e: # noqa: BLE001 # a failed background retry must not kill the task; the backup stays
|
||||
verbose_logger.warning("LiteLLM: Background model cost map retry failed: %s", e)
|
||||
|
||||
|
||||
def _pick_model_cost_map(
|
||||
url: str,
|
||||
timeout: int,
|
||||
max_attempts: int,
|
||||
sleep: Callable[[float], None],
|
||||
rng: random.Random | None,
|
||||
client: "_SyncGetClient | None",
|
||||
) -> Fetched:
|
||||
url: str, timeout: int, max_attempts: int, sleep: Callable[[float], None], catalog: Catalog
|
||||
) -> tuple[Fetched, bool, str | None]:
|
||||
"""
|
||||
The map main loads at import. The bundled map when ``LITELLM_LOCAL_MODEL_COST_MAP`` is set, in offline mode or in a
|
||||
``lite`` / ``litellm-proxy`` CLI process. Otherwise one remote attempt, with a background retry on a transient
|
||||
failure, and the bundled map on any failure
|
||||
The map main loads at import, with the provenance facts install records. The bundled map when
|
||||
``LITELLM_LOCAL_MODEL_COST_MAP`` is set, in offline mode or in a ``lite`` / ``litellm-proxy`` CLI process.
|
||||
Otherwise one remote attempt, with a background retry on a transient failure, and the bundled map on any failure
|
||||
"""
|
||||
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
|
||||
# Note: can't use get_secret_bool here — this runs during litellm.__init__
|
||||
# before litellm._key_management_settings is set.
|
||||
if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process():
|
||||
_cost_map_source_info.source = "local"
|
||||
_cost_map_source_info.url = None
|
||||
_cost_map_source_info.is_env_forced = True
|
||||
_cost_map_source_info.fallback_reason = None
|
||||
return _note_loaded(ModelCostMapReloaded(fetched=CATALOG.bundled())).fetched
|
||||
|
||||
_cost_map_source_info.url = url
|
||||
_cost_map_source_info.is_env_forced = False
|
||||
|
||||
fetch_client: Final = client if client is not None else httpx
|
||||
fetch_rng: Final = rng if rng is not None else random.Random()
|
||||
outcome: Final = _attempt_fetch_sync(client=fetch_client, url=url, timeout=timeout)
|
||||
if isinstance(outcome, _FetchAttemptRetryable) and max_attempts > 1:
|
||||
if _local_map_forced() or _is_cli_process():
|
||||
return _note_loaded(catalog.bundled()), True, None
|
||||
fetched: Final = catalog.fetch(Remote(url or ""), timeout=timeout, attempts=1)
|
||||
if isinstance(fetched, Fetched):
|
||||
return _note_loaded(fetched), False, None
|
||||
if fetched.retry_after is not None and max_attempts > 1:
|
||||
threading.Thread(
|
||||
target=_retry_remote_fetch_in_background,
|
||||
kwargs={
|
||||
"url": url,
|
||||
"timeout": timeout,
|
||||
"max_attempts": max_attempts,
|
||||
"first_wait": fetched.retry_after,
|
||||
"sleep": sleep,
|
||||
"rng": fetch_rng,
|
||||
"client": fetch_client,
|
||||
"first_outcome": outcome,
|
||||
"catalog": catalog,
|
||||
},
|
||||
name="litellm-model-cost-map-retry",
|
||||
daemon=True,
|
||||
).start()
|
||||
if not isinstance(outcome, ModelCostMapReloaded):
|
||||
verbose_logger.warning(
|
||||
"LiteLLM: Failed to fetch remote model cost map from %s: %s. Falling back to local backup.",
|
||||
url,
|
||||
outcome.reason,
|
||||
)
|
||||
_cost_map_source_info.source = "local"
|
||||
_cost_map_source_info.fallback_reason = f"Remote fetch failed: {outcome.reason}"
|
||||
return _note_loaded(ModelCostMapReloaded(fetched=CATALOG.bundled())).fetched
|
||||
|
||||
_cost_map_source_info.source = "remote"
|
||||
_cost_map_source_info.fallback_reason = None
|
||||
return _note_loaded(outcome).fetched
|
||||
verbose_logger.warning(
|
||||
"LiteLLM: Failed to fetch remote model cost map from %s: %s. Falling back to local backup.",
|
||||
url,
|
||||
fetched.reason,
|
||||
)
|
||||
return _note_loaded(catalog.bundled()), False, f"Remote fetch failed: {fetched.reason}"
|
||||
|
||||
|
||||
def install_initial_model_cost_map(url: str) -> None:
|
||||
def install_initial_model_cost_map(
|
||||
url: str,
|
||||
catalog: Catalog = CATALOG,
|
||||
sleep: Callable[[float], None] = time.sleep,
|
||||
max_attempts: int = MODEL_COST_MAP_FETCH_MAX_ATTEMPTS,
|
||||
) -> Installed:
|
||||
"""Fills ``litellm.model_cost`` at import with the map main's rules pick"""
|
||||
_install(
|
||||
_pick_model_cost_map(
|
||||
url=url,
|
||||
timeout=5,
|
||||
max_attempts=MODEL_COST_MAP_FETCH_MAX_ATTEMPTS,
|
||||
sleep=time.sleep,
|
||||
rng=None,
|
||||
client=None,
|
||||
)
|
||||
fetched, env_forced, fallback_reason = _pick_model_cost_map(
|
||||
url=url,
|
||||
timeout=_IMPORT_FETCH_TIMEOUT_SECONDS,
|
||||
max_attempts=max_attempts,
|
||||
sleep=sleep,
|
||||
catalog=catalog,
|
||||
)
|
||||
return _install(
|
||||
fetched, catalog, env_forced=env_forced, fallback_reason=fallback_reason, url=None if env_forced else url
|
||||
)
|
||||
|
||||
|
||||
def get_model_cost_map(
|
||||
url: str,
|
||||
timeout: int = 5,
|
||||
timeout: int = _IMPORT_FETCH_TIMEOUT_SECONDS,
|
||||
max_attempts: int = MODEL_COST_MAP_FETCH_MAX_ATTEMPTS,
|
||||
sleep: Callable[[float], None] = time.sleep,
|
||||
rng: random.Random | None = None,
|
||||
client: "_SyncGetClient | None" = None,
|
||||
rng: object = None,
|
||||
client: object = None,
|
||||
) -> dict[str, dict[str, JsonValue]]:
|
||||
"""
|
||||
Public entry point, returns a fresh model cost map dict picked by main's rules. A remote body is parsed and
|
||||
validated by the Rust catalog, and the bundled map is returned on any failure
|
||||
Public entry point, returns a fresh model cost map dict picked by main's rules. The fetch, its retries and the
|
||||
validation of the body run in Rust, and the bundled map is returned on any failure. ``rng`` and ``client`` are
|
||||
accepted for compatibility and ignored, since Rust owns the HTTP client and the backoff jitter
|
||||
"""
|
||||
return _pick_model_cost_map(
|
||||
url=url, timeout=timeout, max_attempts=max_attempts, sleep=sleep, rng=rng, client=client
|
||||
).served()
|
||||
if rng is not None or client is not None:
|
||||
warnings.warn(
|
||||
"get_model_cost_map ignores rng and client, the fetch runs in Rust", DeprecationWarning, stacklevel=2
|
||||
)
|
||||
fetched, _, _ = _pick_model_cost_map(
|
||||
url=url, timeout=timeout, max_attempts=max_attempts, sleep=sleep, catalog=CATALOG
|
||||
)
|
||||
return fetched.served()
|
||||
|
|
|
|||
|
|
@ -211,7 +211,6 @@ if TYPE_CHECKING:
|
|||
from opentelemetry.trace import Span as _Span
|
||||
from prisma import models as prisma_models
|
||||
|
||||
from litellm._catalog import Fetched
|
||||
from litellm.integrations.opentelemetry import OpenTelemetry
|
||||
from litellm.proxy.health_check_utils.shared_health_check_manager import SharedHealthCheckManager
|
||||
|
||||
|
|
@ -5386,12 +5385,6 @@ def resolve_classifier_plugin(
|
|||
return resolved
|
||||
|
||||
|
||||
def _swap_in_model_cost_map(fetched: "Fetched") -> int:
|
||||
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
|
||||
|
||||
return adopt_model_cost_map(fetched)
|
||||
|
||||
|
||||
def _websearch_handler_params(stored: Mapping[str, object]) -> dict[str, object]:
|
||||
"""
|
||||
Translate stored web search interception settings into handler kwargs.
|
||||
|
|
@ -9091,13 +9084,11 @@ class ProxyConfig:
|
|||
if not is_due:
|
||||
return
|
||||
|
||||
from litellm.litellm_core_utils.get_model_cost_map import (
|
||||
ModelCostMapReloadUnavailable,
|
||||
refetch_model_cost_map,
|
||||
)
|
||||
from litellm._catalog import Unavailable, default_source
|
||||
from litellm.litellm_core_utils.get_model_cost_map import reload_model_cost_map
|
||||
|
||||
reload_result = await refetch_model_cost_map(url=litellm.model_cost_map_url)
|
||||
if isinstance(reload_result, ModelCostMapReloadUnavailable):
|
||||
reload_result = await asyncio.to_thread(reload_model_cost_map, default_source())
|
||||
if isinstance(reload_result, Unavailable):
|
||||
verbose_proxy_logger.warning(
|
||||
"Model cost map reload failed (%s); keeping current pricing data. The revision stays "
|
||||
"unapplied so this pod retries on its next poll",
|
||||
|
|
@ -9105,7 +9096,7 @@ class ProxyConfig:
|
|||
)
|
||||
return
|
||||
|
||||
models_count = _swap_in_model_cost_map(reload_result.fetched)
|
||||
models_count = reload_result.count
|
||||
self.model_cost_map_loaded_at = current_time
|
||||
await record_reload_run(prisma_client, MODEL_COST_MAP_RELOAD_PARAM_NAME, current_time)
|
||||
# Adopted last, so neither a failed fetch nor a failed status write is recorded
|
||||
|
|
@ -20999,20 +20990,20 @@ async def reload_model_cost_map(
|
|||
raise HTTPException(status_code=500, detail="Database connection not available")
|
||||
|
||||
# Immediately reload the model cost map in the current pod
|
||||
from litellm._catalog import Unavailable, default_source
|
||||
from litellm.litellm_core_utils.get_model_cost_map import (
|
||||
ModelCostMapReloadUnavailable,
|
||||
get_model_cost_map_provenance,
|
||||
refetch_model_cost_map,
|
||||
reload_model_cost_map,
|
||||
)
|
||||
|
||||
reload_result = await refetch_model_cost_map(url=litellm.model_cost_map_url)
|
||||
if isinstance(reload_result, ModelCostMapReloadUnavailable):
|
||||
reload_result = await asyncio.to_thread(reload_model_cost_map, default_source())
|
||||
if isinstance(reload_result, Unavailable):
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
detail=f"Failed to reload model cost map: {reload_result.reason}. Current pricing data was kept.",
|
||||
)
|
||||
|
||||
models_count = _swap_in_model_cost_map(reload_result.fetched)
|
||||
models_count = reload_result.count
|
||||
current_time = utc_now()
|
||||
proxy_config.model_cost_map_loaded_at = current_time
|
||||
provenance: Final = get_model_cost_map_provenance()
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import os
|
||||
from asyncio import Future
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import datetime
|
||||
from types import ModuleType
|
||||
from typing import Literal, Never, final
|
||||
|
||||
|
|
@ -70,17 +72,50 @@ class Installed:
|
|||
@final
|
||||
class Unavailable:
|
||||
reason: str
|
||||
retry_after: float | None
|
||||
def __new__(cls, reason: str, retry_after: float | None = None) -> Unavailable: ...
|
||||
|
||||
@final
|
||||
class Remote:
|
||||
url: str
|
||||
def __new__(cls, url: str) -> Remote: ...
|
||||
|
||||
@final
|
||||
class File:
|
||||
path: str
|
||||
def __new__(cls, path: str | os.PathLike[str]) -> File: ...
|
||||
|
||||
@final
|
||||
class Bundled:
|
||||
def __new__(cls) -> Bundled: ...
|
||||
|
||||
@final
|
||||
class Provenance:
|
||||
source: str
|
||||
url: str | None
|
||||
is_env_forced: bool
|
||||
fallback_reason: str | None
|
||||
source_revision: str
|
||||
etag: str | None
|
||||
@property
|
||||
def loaded_at(self) -> datetime: ...
|
||||
|
||||
@final
|
||||
class Catalog:
|
||||
@staticmethod
|
||||
def bound_to(module: ModuleType, attr: str) -> Catalog: ...
|
||||
def bundled(self) -> Fetched: ...
|
||||
@staticmethod
|
||||
def parse(
|
||||
body: bytes, *, etag: str | None, min_model_count: int, max_shrink_ratio: float
|
||||
) -> Fetched | Unavailable: ...
|
||||
def install(self, fetched: Fetched) -> Installed: ...
|
||||
def fetch(self, source: Remote | File | Bundled, *, timeout: float, attempts: int) -> Fetched | Unavailable: ...
|
||||
def reload(self, source: Remote | File | Bundled) -> Installed | Unavailable: ...
|
||||
def provenance(self) -> Provenance | None: ...
|
||||
def install(
|
||||
self,
|
||||
fetched: Fetched,
|
||||
*,
|
||||
env_forced: bool = False,
|
||||
fallback_reason: str | None = None,
|
||||
url: str | None = None,
|
||||
) -> Installed: ...
|
||||
|
||||
@final
|
||||
class NativeClickHouseSpendConfig:
|
||||
|
|
|
|||
0
tests/unit/_catalog/__init__.py
Normal file
0
tests/unit/_catalog/__init__.py
Normal file
88
tests/unit/_catalog/test_fetch.py
Normal file
88
tests/unit/_catalog/test_fetch.py
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
import contextlib
|
||||
import datetime
|
||||
import ipaddress
|
||||
import ssl
|
||||
import threading
|
||||
import types
|
||||
from collections.abc import Generator
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import ec
|
||||
from cryptography.x509.oid import NameOID
|
||||
|
||||
import litellm
|
||||
from litellm._catalog import CATALOG, Catalog, Installed, Remote, Unavailable
|
||||
|
||||
|
||||
def _self_signed(directory: Path) -> tuple[Path, Path]:
|
||||
key = ec.generate_private_key(ec.SECP256R1())
|
||||
name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "127.0.0.1")])
|
||||
now = datetime.datetime.now(datetime.timezone.utc)
|
||||
certificate = (
|
||||
x509.CertificateBuilder()
|
||||
.subject_name(name)
|
||||
.issuer_name(name)
|
||||
.public_key(key.public_key())
|
||||
.serial_number(x509.random_serial_number())
|
||||
.not_valid_before(now - datetime.timedelta(days=1))
|
||||
.not_valid_after(now + datetime.timedelta(days=1))
|
||||
.add_extension(x509.SubjectAlternativeName([x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), critical=False)
|
||||
.sign(key, hashes.SHA256())
|
||||
)
|
||||
cert_path = directory / "cert.pem"
|
||||
key_path = directory / "key.pem"
|
||||
cert_path.write_bytes(certificate.public_bytes(serialization.Encoding.PEM))
|
||||
key_path.write_bytes(
|
||||
key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption())
|
||||
)
|
||||
return cert_path, key_path
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _https_serving(directory: Path, body: bytes) -> Generator[str]:
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None:
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *args: object) -> None:
|
||||
pass
|
||||
|
||||
cert_path, key_path = _self_signed(directory)
|
||||
httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
context.load_cert_chain(cert_path, key_path)
|
||||
httpd.socket = context.wrap_socket(httpd.socket, server_side=True)
|
||||
thread = threading.Thread(target=httpd.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
yield f"https://127.0.0.1:{httpd.server_address[1]}/model_prices.json"
|
||||
finally:
|
||||
httpd.shutdown()
|
||||
httpd.server_close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("ssl_verify", "installed"), [(False, True), (True, False)])
|
||||
def test_the_fetch_uses_the_http_settings_provider_calls_use(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ssl_verify: bool, installed: bool
|
||||
):
|
||||
monkeypatch.setattr(litellm, "ssl_verify", ssl_verify)
|
||||
module = types.ModuleType("fake_litellm")
|
||||
module.model_cost = {}
|
||||
catalog = Catalog.bound_to(module, "model_cost")
|
||||
body = Path(litellm.__file__).resolve().parents[1].joinpath("model_prices_and_context_window.json").read_bytes()
|
||||
|
||||
with _https_serving(tmp_path, body) as url:
|
||||
result = catalog.reload(Remote(url))
|
||||
|
||||
assert isinstance(result, Installed if installed else Unavailable), getattr(result, "reason", None)
|
||||
|
||||
|
||||
def test_a_reload_of_an_unparseable_url_is_unavailable():
|
||||
assert isinstance(CATALOG.fetch(Remote("not a url"), timeout=1, attempts=1), Unavailable)
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -64,13 +64,22 @@ _EXPECTED_RESYNC_APPLIED_CONFIG_PARAM_NAMES = frozenset(
|
|||
_STARTUP_ONLY_CONFIG_PARAM_NAMES = ("environment_variables",)
|
||||
|
||||
|
||||
def _reloaded(cost_map: dict) -> "ModelCostMapReloaded":
|
||||
from litellm._catalog import Catalog, Fetched
|
||||
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
|
||||
from litellm.litellm_core_utils.get_model_cost_map import reload_model_cost_map as _REAL_RELOAD
|
||||
|
||||
parsed = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0)
|
||||
assert isinstance(parsed, Fetched)
|
||||
return ModelCostMapReloaded(fetched=parsed)
|
||||
|
||||
def _reloading(cost_map: dict):
|
||||
"""A stand-in reload that installs ``cost_map`` over the bundled map, read from a file like a fetched map"""
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from litellm._catalog import CATALOG, File
|
||||
|
||||
def reload(source, catalog=CATALOG):
|
||||
path = Path(tempfile.mkdtemp()) / "model_prices.json"
|
||||
path.write_text(json.dumps({**CATALOG.bundled().document(), **cost_map}))
|
||||
return _REAL_RELOAD(File(path), catalog)
|
||||
|
||||
return reload
|
||||
|
||||
|
||||
class _RecordingRedisClient(Redis):
|
||||
|
|
@ -864,12 +873,11 @@ async def test_model_cost_map_reload_does_not_publish_config_change() -> None:
|
|||
original_model_cost = litellm.model_cost.copy()
|
||||
_set_redis_usage_cache(_FakeRedisCache(client))
|
||||
try:
|
||||
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
|
||||
|
||||
with patch(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
|
||||
new=AsyncMock(
|
||||
return_value=_reloaded({"gpt-5.2": {"input_cost_per_token": 0.001}})
|
||||
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map",
|
||||
new=MagicMock(
|
||||
side_effect=_reloading({"gpt-5.2": {"input_cost_per_token": 0.001}})
|
||||
),
|
||||
):
|
||||
await ProxyConfig()._check_and_reload_model_cost_map(prisma_client=prisma_client)
|
||||
|
|
|
|||
|
|
@ -32,13 +32,22 @@ _ROOT_COST_MAP = Path(__file__).resolve().parents[4] / "model_prices_and_context
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _reloaded(cost_map: dict) -> "ModelCostMapReloaded":
|
||||
from litellm._catalog import Catalog, Fetched
|
||||
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
|
||||
from litellm.litellm_core_utils.get_model_cost_map import reload_model_cost_map as _REAL_RELOAD
|
||||
|
||||
parsed = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0)
|
||||
assert isinstance(parsed, Fetched)
|
||||
return ModelCostMapReloaded(fetched=parsed)
|
||||
|
||||
def _reloading(cost_map: dict):
|
||||
"""A stand-in reload that installs ``cost_map`` over the bundled map, read from a file like a fetched map"""
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from litellm._catalog import CATALOG, File
|
||||
|
||||
def reload(source, catalog=CATALOG):
|
||||
path = Path(tempfile.mkdtemp()) / "model_prices.json"
|
||||
path.write_text(json.dumps({**CATALOG.bundled().document(), **cost_map}))
|
||||
return _REAL_RELOAD(File(path), catalog)
|
||||
|
||||
return reload
|
||||
|
||||
|
||||
def _attach_litellm_config(mock_prisma):
|
||||
|
|
@ -73,8 +82,8 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma):
|
|||
|
||||
fake_cost_map = {"gpt-4": {"input_cost": 0.03}, "gpt-3.5": {"input_cost": 0.002}}
|
||||
monkeypatch.setattr(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
|
||||
AsyncMock(return_value=_reloaded(fake_cost_map)),
|
||||
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map",
|
||||
MagicMock(side_effect=_reloading(fake_cost_map)),
|
||||
)
|
||||
monkeypatch.setattr("litellm.add_known_models", lambda model_cost_map=None: None)
|
||||
monkeypatch.setattr("litellm.model_cost", {}, raising=False)
|
||||
|
|
@ -110,7 +119,6 @@ def test_reload_model_cost_map_happy(client, auth_as, monkeypatch, mock_prisma):
|
|||
def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every_status_surface(
|
||||
client, auth_as, monkeypatch, mock_prisma
|
||||
):
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
import hashlib
|
||||
|
|
@ -125,11 +133,23 @@ def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every
|
|||
"source_revision": hashlib.sha1(b"blob %d\0" % len(body) + body, usedforsecurity=False).hexdigest(),
|
||||
"etag": _SERVED_ETAG,
|
||||
}
|
||||
served = httpx.Response(200, headers={"ETag": _SERVED_ETAG}, content=body)
|
||||
monkeypatch.setattr(
|
||||
"litellm.litellm_core_utils.get_model_cost_map._default_reload_client",
|
||||
lambda: httpx.AsyncClient(transport=httpx.MockTransport(lambda request: served)),
|
||||
)
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None:
|
||||
self.send_response(200)
|
||||
self.send_header("ETag", _SERVED_ETAG)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *args: object) -> None:
|
||||
pass
|
||||
|
||||
httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
threading.Thread(target=httpd.serve_forever, daemon=True).start()
|
||||
monkeypatch.setattr(litellm, "model_cost_map_url", f"http://127.0.0.1:{httpd.server_address[1]}/map.json")
|
||||
monkeypatch.setattr("litellm.add_known_models", lambda model_cost_map=None: None)
|
||||
monkeypatch.setattr("litellm.model_cost", {}, raising=False)
|
||||
|
||||
|
|
@ -155,6 +175,8 @@ def test_reload_model_cost_map_surfaces_the_blob_id_of_the_bytes_served_on_every
|
|||
assert {key: status_response.json()[key] for key in expected} == expected
|
||||
assert public_response.status_code == 200
|
||||
assert "gpt-4o" in public_response.json()
|
||||
httpd.shutdown()
|
||||
httpd.server_close()
|
||||
assert reload_body["models_count"] == len(litellm.model_cost)
|
||||
|
||||
|
||||
|
|
@ -166,9 +188,7 @@ def test_reload_model_cost_map_fetch_failure_502_keeps_map(
|
|||
Regression: the endpoint used to report success after silently swapping in
|
||||
the stale packaged backup.
|
||||
"""
|
||||
from litellm.litellm_core_utils.get_model_cost_map import (
|
||||
ModelCostMapReloadUnavailable,
|
||||
)
|
||||
from litellm._catalog import Unavailable
|
||||
from litellm.proxy import proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
|
||||
|
|
@ -178,9 +198,9 @@ def test_reload_model_cost_map_fetch_failure_502_keeps_map(
|
|||
sentinel_map = {"existing-model": {"input_cost": 0.01}}
|
||||
monkeypatch.setattr("litellm.model_cost", sentinel_map, raising=False)
|
||||
monkeypatch.setattr(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
|
||||
AsyncMock(
|
||||
return_value=ModelCostMapReloadUnavailable(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map",
|
||||
MagicMock(
|
||||
return_value=Unavailable(
|
||||
reason="HTTP 429 from upstream (after 3 attempts)"
|
||||
)
|
||||
),
|
||||
|
|
@ -415,7 +435,6 @@ def test_get_model_cost_map_reload_status_no_config_not_scheduled(
|
|||
client, auth_as, monkeypatch, mock_prisma
|
||||
):
|
||||
"""A row left behind by a manual reload (interval_hours=None) → not scheduled."""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from litellm.proxy import proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
|
|
|
|||
|
|
@ -33,7 +33,6 @@ from litellm._logging import turn_on_json, verbose_proxy_logger
|
|||
from litellm.caching.caching import RedisCache
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.caching.redis_cluster_cache import RedisClusterCache
|
||||
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
|
||||
from litellm.proxy._types import (
|
||||
AwsErrorTypeProxyException,
|
||||
LitellmUserRoles,
|
||||
|
|
@ -80,13 +79,22 @@ example_embedding_result = {
|
|||
}
|
||||
|
||||
|
||||
def _reloaded(cost_map: dict) -> "ModelCostMapReloaded":
|
||||
from litellm._catalog import Catalog, Fetched
|
||||
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
|
||||
from litellm.litellm_core_utils.get_model_cost_map import reload_model_cost_map as _REAL_RELOAD
|
||||
|
||||
parsed = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0)
|
||||
assert isinstance(parsed, Fetched)
|
||||
return ModelCostMapReloaded(fetched=parsed)
|
||||
|
||||
def _reloading(cost_map: dict):
|
||||
"""A stand-in reload that installs ``cost_map`` over the bundled map, read from a file like a fetched map"""
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from litellm._catalog import CATALOG, File
|
||||
|
||||
def reload(source, catalog=CATALOG):
|
||||
path = Path(tempfile.mkdtemp()) / "model_prices.json"
|
||||
path.write_text(json.dumps({**CATALOG.bundled().document(), **cost_map}))
|
||||
return _REAL_RELOAD(File(path), catalog)
|
||||
|
||||
return reload
|
||||
|
||||
|
||||
def mock_patch_aembedding():
|
||||
|
|
@ -4553,14 +4561,13 @@ class TestPriceDataReloadAPI:
|
|||
# Save the original model_cost so the endpoint's direct assignment
|
||||
# (litellm.model_cost = new_model_cost_map) does not contaminate
|
||||
# subsequent tests running in the same worker process.
|
||||
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
|
||||
|
||||
original_model_cost = litellm.model_cost.copy()
|
||||
try:
|
||||
with patch(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
|
||||
new=AsyncMock(
|
||||
return_value=_reloaded({"gpt-3.5-turbo": {"input_cost_per_token": 0.001}})
|
||||
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map",
|
||||
new=MagicMock(
|
||||
side_effect=_reloading({"gpt-3.5-turbo": {"input_cost_per_token": 0.001}})
|
||||
),
|
||||
):
|
||||
# Mock the database connection
|
||||
|
|
@ -4614,8 +4621,8 @@ class TestPriceDataReloadAPI:
|
|||
def test_reload_model_cost_map_error_handling(self, client_with_auth):
|
||||
"""Test error handling in the reload endpoint"""
|
||||
with patch(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
|
||||
new=AsyncMock(side_effect=Exception("Network error")),
|
||||
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map",
|
||||
new=MagicMock(side_effect=Exception("Network error")),
|
||||
):
|
||||
# Mock the database connection
|
||||
with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma:
|
||||
|
|
@ -4828,13 +4835,11 @@ class TestPriceDataReloadIntegration:
|
|||
"gpt-4": {"input_cost_per_token": 0.03, "output_cost_per_token": 0.06},
|
||||
}
|
||||
|
||||
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
|
||||
|
||||
original_model_cost = litellm.model_cost.copy()
|
||||
try:
|
||||
with patch(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
|
||||
new=AsyncMock(return_value=_reloaded(mock_cost_map)),
|
||||
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map",
|
||||
new=MagicMock(side_effect=_reloading(mock_cost_map)),
|
||||
):
|
||||
# Mock the database connection
|
||||
with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma:
|
||||
|
|
@ -4897,17 +4902,15 @@ class TestPriceDataReloadIntegration:
|
|||
proxy_config.model_cost_map_loaded_at = frozen_now - timedelta(minutes=1)
|
||||
proxy_config.model_cost_map_applied_revision = 3
|
||||
|
||||
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
|
||||
|
||||
original_model_cost = litellm.model_cost.copy()
|
||||
try:
|
||||
with (
|
||||
patch(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new_callable=AsyncMock
|
||||
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", new_callable=MagicMock
|
||||
) as mock_get_map,
|
||||
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
|
||||
):
|
||||
mock_get_map.return_value = _reloaded({"gpt-3.5-turbo": {"input_cost_per_token": 0.001}}
|
||||
mock_get_map.side_effect = _reloading({"gpt-3.5-turbo": {"input_cost_per_token": 0.001}}
|
||||
)
|
||||
|
||||
asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma))
|
||||
|
|
@ -4950,7 +4953,7 @@ class TestPriceDataReloadIntegration:
|
|||
try:
|
||||
with (
|
||||
patch(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new_callable=AsyncMock
|
||||
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", new_callable=MagicMock
|
||||
) as mock_get_map,
|
||||
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
|
||||
):
|
||||
|
|
@ -4987,11 +4990,11 @@ class TestPriceDataReloadIntegration:
|
|||
try:
|
||||
with (
|
||||
patch(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new_callable=AsyncMock
|
||||
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", new_callable=MagicMock
|
||||
) as mock_get_map,
|
||||
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
|
||||
):
|
||||
mock_get_map.return_value = _reloaded({"gpt-4-test": {"input_cost_per_token": 0.5}}
|
||||
mock_get_map.side_effect = _reloading({"gpt-4-test": {"input_cost_per_token": 0.5}}
|
||||
)
|
||||
|
||||
asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma))
|
||||
|
|
@ -5031,11 +5034,11 @@ class TestPriceDataReloadIntegration:
|
|||
try:
|
||||
with (
|
||||
patch(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new_callable=AsyncMock
|
||||
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", new_callable=MagicMock
|
||||
) as mock_get_map,
|
||||
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
|
||||
):
|
||||
mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}}
|
||||
mock_get_map.side_effect = _reloading({"gpt-4": {"input_cost_per_token": 0.001}}
|
||||
)
|
||||
|
||||
for _ in range(3):
|
||||
|
|
@ -5074,11 +5077,11 @@ class TestPriceDataReloadIntegration:
|
|||
try:
|
||||
with (
|
||||
patch(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new_callable=AsyncMock
|
||||
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", new_callable=MagicMock
|
||||
) as mock_get_map,
|
||||
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
|
||||
):
|
||||
mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}}
|
||||
mock_get_map.side_effect = _reloading({"gpt-4": {"input_cost_per_token": 0.001}}
|
||||
)
|
||||
|
||||
asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma))
|
||||
|
|
@ -5109,11 +5112,11 @@ class TestPriceDataReloadIntegration:
|
|||
try:
|
||||
with (
|
||||
patch(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new_callable=AsyncMock
|
||||
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", new_callable=MagicMock
|
||||
) as mock_get_map,
|
||||
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
|
||||
):
|
||||
mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}}
|
||||
mock_get_map.side_effect = _reloading({"gpt-4": {"input_cost_per_token": 0.001}}
|
||||
)
|
||||
|
||||
asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma))
|
||||
|
|
@ -5149,12 +5152,12 @@ class TestPriceDataReloadIntegration:
|
|||
try:
|
||||
with (
|
||||
patch(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
|
||||
new_callable=AsyncMock,
|
||||
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map",
|
||||
new_callable=MagicMock,
|
||||
) as mock_get_map,
|
||||
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
|
||||
):
|
||||
mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.1}}
|
||||
mock_get_map.side_effect = _reloading({"gpt-4": {"input_cost_per_token": 0.1}}
|
||||
)
|
||||
|
||||
asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma))
|
||||
|
|
@ -5174,9 +5177,7 @@ class TestPriceDataReloadIntegration:
|
|||
so a pod that records it as applied without the data stays mispriced until someone
|
||||
clicks again
|
||||
"""
|
||||
from litellm.litellm_core_utils.get_model_cost_map import (
|
||||
ModelCostMapReloadUnavailable,
|
||||
)
|
||||
from litellm._catalog import Unavailable
|
||||
from litellm.proxy.proxy_server import ProxyConfig
|
||||
|
||||
proxy_config = ProxyConfig()
|
||||
|
|
@ -5193,8 +5194,8 @@ class TestPriceDataReloadIntegration:
|
|||
original_model_cost = litellm.model_cost
|
||||
with (
|
||||
patch(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
|
||||
new=AsyncMock(return_value=ModelCostMapReloadUnavailable(reason="HTTP 429 from upstream")),
|
||||
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map",
|
||||
new=MagicMock(return_value=Unavailable(reason="HTTP 429 from upstream")),
|
||||
),
|
||||
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
|
||||
):
|
||||
|
|
@ -5219,7 +5220,6 @@ class TestPriceDataReloadIntegration:
|
|||
after it is swallowed by the surrounding except and would otherwise
|
||||
leave the metadata correct while the path is quietly broken"""
|
||||
from litellm import utils as litellm_utils
|
||||
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
|
||||
from litellm.proxy.proxy_server import ProxyConfig
|
||||
|
||||
proxy_config = ProxyConfig()
|
||||
|
|
@ -5240,9 +5240,9 @@ class TestPriceDataReloadIntegration:
|
|||
|
||||
with (
|
||||
patch(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map",
|
||||
new=AsyncMock(
|
||||
return_value=_reloaded({"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}}
|
||||
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map",
|
||||
new=MagicMock(
|
||||
side_effect=_reloading({"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}}
|
||||
)
|
||||
),
|
||||
),
|
||||
|
|
@ -5261,13 +5261,12 @@ class TestPriceDataReloadIntegration:
|
|||
litellm_utils._runtime_registered_model_cost.update(original_registry)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
def test_swap_in_model_cost_map_counts_the_fetched_catalog_only(self):
|
||||
def test_a_reload_counts_the_fetched_catalog_only(self):
|
||||
"""The count the reload endpoints report describes the price data, so it
|
||||
is taken before the runtime registrations are written back into the same
|
||||
dict. Counting after would inflate it by however many deployments and
|
||||
overrides this pod happens to be carrying"""
|
||||
from litellm import utils as litellm_utils
|
||||
from litellm.proxy.proxy_server import _swap_in_model_cost_map
|
||||
|
||||
original_model_cost = litellm.model_cost
|
||||
original_registry = dict(litellm_utils._runtime_registered_model_cost)
|
||||
|
|
@ -5276,9 +5275,9 @@ class TestPriceDataReloadIntegration:
|
|||
model_cost={"custom/deployment-model": {"litellm_provider": "custom", "max_input_tokens": 4321}}
|
||||
)
|
||||
|
||||
models_count = _swap_in_model_cost_map(_reloaded({"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}}).fetched)
|
||||
installed = _reloading({"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}})(None)
|
||||
|
||||
assert models_count == len(litellm.model_cost) - 1
|
||||
assert installed.count == len(litellm.model_cost) - 1
|
||||
assert litellm.model_cost["custom/deployment-model"]["max_input_tokens"] == 4321
|
||||
finally:
|
||||
litellm.model_cost = original_model_cost
|
||||
|
|
@ -5305,18 +5304,16 @@ class TestPriceDataReloadIntegration:
|
|||
client = TestClient(app)
|
||||
frozen_now = datetime(2024, 1, 1, 7, 0, tzinfo=timezone.utc)
|
||||
|
||||
from litellm.litellm_core_utils.get_model_cost_map import ModelCostMapReloaded
|
||||
|
||||
original_model_cost = litellm.model_cost.copy()
|
||||
try:
|
||||
with (
|
||||
patch(
|
||||
"litellm.litellm_core_utils.get_model_cost_map.refetch_model_cost_map", new_callable=AsyncMock
|
||||
"litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map", new_callable=MagicMock
|
||||
) as mock_get_map,
|
||||
patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma,
|
||||
patch("litellm.proxy.proxy_server.utc_now", return_value=frozen_now),
|
||||
):
|
||||
mock_get_map.return_value = _reloaded({"gpt-4": {"input_cost_per_token": 0.001}}
|
||||
mock_get_map.side_effect = _reloading({"gpt-4": {"input_cost_per_token": 0.001}}
|
||||
)
|
||||
mock_prisma.db.litellm_config.upsert = AsyncMock(
|
||||
return_value=_reload_schedule_row({}, reload_revision=9)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
|
|
@ -12,8 +13,8 @@ from litellm import Router
|
|||
from litellm import router as litellm_router_module
|
||||
from litellm import utils as litellm_utils_module
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from tests.unit import conftest as unit_harness
|
||||
|
||||
CANARY_MODEL = "conftest-isolation-canary-model"
|
||||
|
|
@ -179,11 +180,14 @@ def registered_collection_models() -> Iterator[None]:
|
|||
litellm_router_module._live_routers.update(original_routers)
|
||||
|
||||
|
||||
def test_collection_model_metadata_is_not_replayed_into_a_test_catalog() -> None:
|
||||
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
|
||||
def test_collection_model_metadata_is_not_replayed_into_a_test_catalog(tmp_path) -> None:
|
||||
from litellm._catalog import CATALOG, File
|
||||
from litellm.litellm_core_utils.get_model_cost_map import reload_model_cost_map
|
||||
|
||||
catalog: Final = {"unit-test-catalog-model": {"litellm_provider": "openai", "input_cost_per_token": 0.001}}
|
||||
adopt_model_cost_map(catalog)
|
||||
path: Final = tmp_path / "model_prices.json"
|
||||
path.write_text(json.dumps({**CATALOG.bundled().document(), **catalog}))
|
||||
reload_model_cost_map(File(path))
|
||||
assert "collection-isolation-ledger" not in litellm.model_cost
|
||||
assert "collection-isolation-router" not in litellm.model_cost
|
||||
assert "unit-test-catalog-model" in litellm.model_cost
|
||||
|
|
|
|||
|
|
@ -4,19 +4,31 @@ Tests for the ``aliases`` feature in the model cost map.
|
|||
The Rust catalog serves every name in an ``aliases`` list as its own key, sharing one dict with the canonical entry
|
||||
"""
|
||||
|
||||
import functools
|
||||
import json
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from litellm._catalog import Catalog, Fetched, Unavailable
|
||||
from litellm._catalog import CATALOG, Fetched, File, Unavailable
|
||||
|
||||
|
||||
def _parsed(model_cost: dict) -> Fetched | Unavailable:
|
||||
return Catalog.parse(json.dumps(model_cost).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0)
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _bundled_document() -> dict:
|
||||
return CATALOG.bundled().document()
|
||||
|
||||
|
||||
def _parsed(model_cost: dict, over_bundled: bool = True) -> Fetched | Unavailable:
|
||||
path = Path(tempfile.mkdtemp()) / "model_prices.json"
|
||||
path.write_text(json.dumps({**_bundled_document(), **model_cost} if over_bundled else model_cost))
|
||||
return CATALOG.fetch(File(path), timeout=5, attempts=1)
|
||||
|
||||
|
||||
def _expand(model_cost: dict) -> dict:
|
||||
"""The rows the fetched map serves beyond the bundled ones, aliases included"""
|
||||
parsed = _parsed(model_cost)
|
||||
assert isinstance(parsed, Fetched)
|
||||
return parsed.served()
|
||||
bundled = _bundled_document()
|
||||
return {key: row for key, row in parsed.served().items() if key not in bundled or key in model_cost}
|
||||
|
||||
|
||||
def _error_kinds(model_cost: dict) -> list[str]:
|
||||
|
|
@ -247,4 +259,4 @@ class TestAliasIntegration:
|
|||
assert "model-with-alias" in result
|
||||
|
||||
def test_an_empty_map_is_unavailable(self):
|
||||
assert isinstance(_parsed({}), Unavailable)
|
||||
assert isinstance(_parsed({}, over_bundled=False), Unavailable)
|
||||
|
|
|
|||
|
|
@ -14,22 +14,22 @@ import itertools
|
|||
import json
|
||||
import os
|
||||
from collections.abc import Callable, Mapping
|
||||
from pathlib import Path
|
||||
from typing import Final, NoReturn
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
import litellm
|
||||
from litellm._catalog import Catalog, Fetched
|
||||
from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry, adopt_model_cost_map
|
||||
from litellm._catalog import File
|
||||
from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry, reload_model_cost_map
|
||||
from litellm.main import _build_custom_pricing_entry
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map
|
||||
|
||||
|
||||
def _fetched(cost_map: Mapping[str, object]) -> Fetched:
|
||||
parsed: Final = Catalog.parse(json.dumps(cost_map).encode(), etag=None, min_model_count=0, max_shrink_ratio=0.0)
|
||||
assert isinstance(parsed, Fetched)
|
||||
return parsed
|
||||
def _reload_from(cost_map: Mapping[str, object], directory: Path) -> object:
|
||||
path: Final = directory / "model_prices.json"
|
||||
path.write_text(json.dumps(cost_map))
|
||||
return reload_model_cost_map(File(path))
|
||||
|
||||
|
||||
def _snapshot_model_cost_entries(keys):
|
||||
|
|
@ -1027,7 +1027,9 @@ def test_register_model_prices_a_provider_less_azure_ai_key_without_repricing_th
|
|||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
catalog_key: Final = "au.anthropic.lit-5853-reloaded-model"
|
||||
deployment_key: Final = f"bedrock/{catalog_key}"
|
||||
catalog_row: Final = {
|
||||
|
|
@ -1046,12 +1048,12 @@ def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers(monk
|
|||
catalog_key: {**catalog_row, "input_cost_per_token": reloaded_input_cost},
|
||||
}
|
||||
real_lookup: Final = litellm.utils.get_model_info
|
||||
reloads: Final[list[int]] = [] # mutable-ok: counts reloads the racing lookup triggered
|
||||
reloads: Final[list[object]] = [] # mutable-ok: counts reloads the racing lookup triggered
|
||||
|
||||
def lookup_racing_a_price_data_reload(model: str, custom_llm_provider: str | None = None) -> object:
|
||||
info: Final = real_lookup(model=model, custom_llm_provider=custom_llm_provider)
|
||||
if not reloads:
|
||||
reloads.append(adopt_model_cost_map(_fetched(reloaded_cost_map)))
|
||||
reloads.append(_reload_from(reloaded_cost_map, tmp_path))
|
||||
return info
|
||||
|
||||
with pytest.MonkeyPatch.context() as lookup_patch:
|
||||
|
|
@ -1363,10 +1365,10 @@ def test_update_dictionary_merges_nested_dicts_without_aliasing():
|
|||
object stays untouched, and the caller's incoming nested dict is never
|
||||
inserted by reference into the merged result.
|
||||
"""
|
||||
from litellm.utils import update_dictionary
|
||||
|
||||
from typing import cast
|
||||
|
||||
from litellm.utils import update_dictionary
|
||||
|
||||
existing_nested = cast(dict[str, object], {"hours_utc": "01:00-02:00", 1: "existing"})
|
||||
existing = {"off_peak_pricing": existing_nested}
|
||||
incoming_nested = cast(
|
||||
|
|
|
|||
|
|
@ -54,8 +54,8 @@ _COPILOT_TOKEN_EXPIRES_AT: Final = 4_102_444_800
|
|||
def _simulate_price_data_reload(fetched_catalog):
|
||||
"""Drive what a price data reload does to this process's litellm state.
|
||||
|
||||
Mirrors `litellm.proxy.proxy_server._swap_in_model_cost_map`, which is the
|
||||
one place both reload paths adopt a freshly fetched catalog; that wiring is
|
||||
Mirrors `litellm.litellm_core_utils.get_model_cost_map.reload_model_cost_map`, which is
|
||||
the one place both reload paths install a freshly fetched catalog; that wiring is
|
||||
covered in the proxy's own tests, so these exercise the replay itself
|
||||
without dragging the proxy in. The provider model sets that helper also
|
||||
repopulates are left alone, since nothing here reads them and rebuilding
|
||||
|
|
@ -3858,7 +3858,7 @@ def test_every_declaring_deployment_is_named(caplog):
|
|||
|
||||
def _simulate_price_data_reload_with_provider_sets(monkeypatch, fetched_catalog):
|
||||
"""Like `_simulate_price_data_reload`, plus the provider model-set refresh the proxy's
|
||||
`_swap_in_model_cost_map` does before replaying, so bare names in the new catalog resolve."""
|
||||
`reload_model_cost_map` does before replaying, so bare names in the new catalog resolve."""
|
||||
monkeypatch.setattr(litellm, "model_cost", fetched_catalog)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
litellm.add_known_models(model_cost_map=fetched_catalog)
|
||||
|
|
|
|||
|
|
@ -2454,16 +2454,34 @@ def test_register_model_with_scientific_notation():
|
|||
@pytest.mark.usefixtures("isolated_openai_model_sets")
|
||||
@respx.mock
|
||||
def test_register_model_url_fetch_uses_single_attempt(monkeypatch):
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False)
|
||||
monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost))
|
||||
before = dict(litellm.model_cost)
|
||||
threads_before = {thread.name for thread in threading.enumerate()}
|
||||
route = respx.get("https://example.invalid/custom_pricing.json").mock(return_value=httpx.Response(503))
|
||||
requests: list[str] = []
|
||||
|
||||
litellm.register_model(model_cost="https://example.invalid/custom_pricing.json")
|
||||
class Unavailable(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None:
|
||||
requests.append(self.path)
|
||||
self.send_response(503)
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
|
||||
def log_message(self, *args: object) -> None:
|
||||
pass
|
||||
|
||||
httpd = ThreadingHTTPServer(("127.0.0.1", 0), Unavailable)
|
||||
threading.Thread(target=httpd.serve_forever, daemon=True).start()
|
||||
try:
|
||||
litellm.register_model(model_cost=f"http://127.0.0.1:{httpd.server_address[1]}/custom_pricing.json")
|
||||
finally:
|
||||
httpd.shutdown()
|
||||
httpd.server_close()
|
||||
|
||||
threads_after = {thread.name for thread in threading.enumerate()}
|
||||
assert route.call_count == 1
|
||||
assert requests == ["/custom_pricing.json"]
|
||||
assert not (threads_after - threads_before) & {"litellm-model-cost-map-retry"}
|
||||
assert not any(
|
||||
thread.name == "litellm-model-cost-map-retry" and thread.is_alive() for thread in threading.enumerate()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue