diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index db559619526..e60cb676cc5 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -247,6 +247,8 @@ checksum = "47712fde1909402600ccfbb26e47d482d2e58bb9e9e603d9f17e67cc435a6319" dependencies = [ "aws-credential-types", "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", "aws-smithy-http", @@ -258,11 +260,14 @@ dependencies = [ "aws-types", "bytes", "fastrand", + "hex", "http 1.4.2", + "sha1 0.10.7", "time", "tokio", "tracing", "url", + "zeroize", ] [[package]] @@ -417,6 +422,58 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-sdk-sso" +version = "1.111.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65700d75f6da78c89a61a49f1539979400a4609d3329376b1dc2133d7a1b2b90" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.2", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.113.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb1147c254af7b6cee3ace90ae7da6f21366ce9a9b899035fa8a0fdf586eb75a" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.2", + "regex-lite", + "tracing", +] + [[package]] name = "aws-sdk-sts" version = "1.108.0" @@ -3596,16 +3653,13 @@ version = "0.1.0" dependencies = [ "aws-credential-types", "aws-sdk-s3", - "aws-smithy-runtime-api", "aws-smithy-types", - "aws-types", "futures-util", - "http 1.4.2", "litellm-auth-aws", "litellm-cache", "litellm-cache-testing", "litellm-http", - "reqwest 0.12.28", + "litellm-s3", "rstest", "serde_json", "tokio", @@ -4230,9 +4284,12 @@ name = "litellm-model-catalog-refresh" version = "0.1.0" dependencies = [ "async-trait", + "aws-config", + "aws-sdk-s3", "axum", "litellm-http", "litellm-model-catalog", + "litellm-s3", "nix", "rcgen", "reqwest 0.12.28", @@ -4245,6 +4302,7 @@ dependencies = [ "tokio-util", "tracing", "url", + "wiremock", ] [[package]] @@ -4339,6 +4397,18 @@ dependencies = [ "serde_with", ] +[[package]] +name = "litellm-s3" +version = "0.1.0" +dependencies = [ + "aws-credential-types", + "aws-sdk-s3", + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.4.2", + "reqwest 0.12.28", +] + [[package]] name = "litellm-secrets" version = "0.1.0" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 5cad504aca4..f3ed8d4ea87 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -57,6 +57,7 @@ litellm-cache-azure-blob = { path = "crates/cache-azure-blob" } litellm-cache-memory = { path = "crates/cache-memory" } litellm-cache-redis = { path = "crates/cache-redis" } litellm-cache-s3 = { path = "crates/cache-s3" } +litellm-s3 = { path = "crates/s3" } litellm-cache-gcs = { path = "crates/cache-gcs" } litellm-cache-disk = { path = "crates/cache-disk" } litellm-cache-redis-semantic = { path = "crates/cache-redis-semantic" } diff --git a/litellm-rust/crates/cache-s3/Cargo.toml b/litellm-rust/crates/cache-s3/Cargo.toml index eb3a2fff1ac..ecf9966477f 100644 --- a/litellm-rust/crates/cache-s3/Cargo.toml +++ b/litellm-rust/crates/cache-s3/Cargo.toml @@ -7,16 +7,13 @@ repository.workspace = true [dependencies] litellm-http.workspace = true +litellm-s3.workspace = true litellm-cache.workspace = true litellm-auth-aws.workspace = true aws-sdk-s3 = { version = "1.146.1", default-features = false, features = ["rustls", "rt-tokio"] } aws-credential-types = "1.3.0" -aws-smithy-runtime-api = { version = "1.16.2", features = ["client", "http-1x"] } aws-smithy-types = { version = "1.6.0", features = ["http-body-1-x"] } -aws-types = "1.6.0" futures-util.workspace = true -http.workspace = true -reqwest.workspace = true tokio.workspace = true [dev-dependencies] diff --git a/litellm-rust/crates/cache-s3/src/cache.rs b/litellm-rust/crates/cache-s3/src/cache.rs index ced948e80c6..d22c0768fa8 100644 --- a/litellm-rust/crates/cache-s3/src/cache.rs +++ b/litellm-rust/crates/cache-s3/src/cache.rs @@ -4,11 +4,7 @@ use std::{ time::{Duration, SystemTime}, }; -use aws_sdk_s3::{ - config::{BehaviorVersion, Region, RequestChecksumCalculation, ResponseChecksumValidation}, - error::SdkError, - primitives::ByteStream, -}; +use aws_sdk_s3::{error::SdkError, primitives::ByteStream}; use aws_smithy_types::{DateTime, date_time::Format}; use futures_util::future::try_join_all; use litellm_auth_aws::AwsAuthConfig; @@ -17,7 +13,7 @@ use litellm_cache::{ }; use tokio::runtime::Handle; -use crate::{auth::S3Credentials, transport::ReqwestHttpClient}; +use crate::auth::S3Credentials; pub struct S3Endpoint { pub url: String, @@ -49,19 +45,16 @@ impl S3Cache { runtime: Handle, ) -> Self { let endpoint_url: Option = config.endpoint.map(|endpoint| endpoint.url); - let base = aws_sdk_s3::Config::builder() - .behavior_version(BehaviorVersion::latest()) - .region(Region::new(config.region.clone())) - .http_client(ReqwestHttpClient(http)) - .credentials_provider(S3Credentials::new(config.auth)) - .request_checksum_calculation(RequestChecksumCalculation::WhenRequired) - .response_checksum_validation(ResponseChecksumValidation::WhenRequired); - let builder = match &endpoint_url { - Some(url) => base.endpoint_url(url).force_path_style(true), - None => base, - }; + let client = litellm_s3::client( + litellm_s3::S3ClientConfig { + region: config.region.clone(), + endpoint: endpoint_url.clone(), + }, + (*http).clone(), + S3Credentials::new(config.auth), + ); Self { - client: aws_sdk_s3::Client::from_conf(builder.build()), + client, codec, runtime, bucket: config.bucket.into(), diff --git a/litellm-rust/crates/cache-s3/src/lib.rs b/litellm-rust/crates/cache-s3/src/lib.rs index 9be0210aad3..9e722913148 100644 --- a/litellm-rust/crates/cache-s3/src/lib.rs +++ b/litellm-rust/crates/cache-s3/src/lib.rs @@ -1,6 +1,5 @@ mod auth; mod cache; -mod transport; pub use auth::S3Credentials; pub use cache::{S3Cache, S3CacheConfig, S3Endpoint}; diff --git a/litellm-rust/crates/model-catalog-refresh/Cargo.toml b/litellm-rust/crates/model-catalog-refresh/Cargo.toml index 5d090a39906..2df793f65b4 100644 --- a/litellm-rust/crates/model-catalog-refresh/Cargo.toml +++ b/litellm-rust/crates/model-catalog-refresh/Cargo.toml @@ -7,8 +7,11 @@ repository.workspace = true [dependencies] async-trait = "0.1" +aws-config = { version = "1.9.0", default-features = false, features = ["rt-tokio", "sso", "credentials-process"] } +aws-sdk-s3 = { version = "1.146.1", default-features = false, features = ["rustls", "rt-tokio"] } litellm-http.workspace = true litellm-model-catalog.workspace = true +litellm-s3.workspace = true reqwest.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["fs"] } @@ -22,6 +25,7 @@ rcgen = "0.14.10" rstest.workspace = true rustls.workspace = true tempfile.workspace = true +wiremock.workspace = true tokio-rustls = { version = "0.26", default-features = false, features = ["ring"] } [target.'cfg(unix)'.dev-dependencies] diff --git a/litellm-rust/crates/model-catalog-refresh/src/lib.rs b/litellm-rust/crates/model-catalog-refresh/src/lib.rs index 5fbb9271d9a..cbb68cacd89 100644 --- a/litellm-rust/crates/model-catalog-refresh/src/lib.rs +++ b/litellm-rust/crates/model-catalog-refresh/src/lib.rs @@ -1,8 +1,10 @@ mod error; mod process_local; mod refresher; +mod s3; mod source; pub use error::{SourceError, Stale}; pub use refresher::{Adopted, Checked, Install, Refresher}; +pub use s3::{S3Object, S3Source}; pub use source::{DEFAULT_SOURCE, Fetch, FileSource, HttpSource, Source}; diff --git a/litellm-rust/crates/model-catalog-refresh/src/s3.rs b/litellm-rust/crates/model-catalog-refresh/src/s3.rs new file mode 100644 index 00000000000..9adcab5c254 --- /dev/null +++ b/litellm-rust/crates/model-catalog-refresh/src/s3.rs @@ -0,0 +1,746 @@ +#![allow( + clippy::disallowed_types, + reason = "the fork-local client uses HttpClientConfig without the shared pool" +)] + +use std::sync::Arc; + +use async_trait::async_trait; +use aws_sdk_s3::{ + config::{BehaviorVersion, Region, retry::RetryConfig, timeout::TimeoutConfig}, + error::SdkError, + operation::get_object::{GetObjectError, GetObjectOutput}, +}; +use litellm_http::HttpClientConfig; +use litellm_model_catalog::{FetchRequest, Reply, SourceId}; +use litellm_s3::{ReqwestHttpClient, S3ClientConfig}; +use tokio::sync::OnceCell; + +use crate::{Fetch, SourceError, process_local::ProcessLocal}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct S3Object { + pub bucket: String, + pub key: String, + source: String, +} + +impl S3Object { + pub(crate) fn parse(value: &str) -> Result { + let (bucket, key) = value + .split_once("://") + .and_then(|(_, rest)| rest.split(['?', '#']).next()) + .and_then(|rest| rest.split_once('/')) + .unwrap_or_default(); + let key = key.trim_start_matches('/'); + if bucket.is_empty() || key.is_empty() { + return Err(SourceError::Invalid { + reason: "expected s3://bucket/key".to_owned(), + }); + } + Ok(Self { + bucket: bucket.to_owned(), + key: key.to_owned(), + source: value.to_owned(), + }) + } + + pub(crate) fn id(&self) -> SourceId { + SourceId(self.source.clone()) + } +} + +struct Connection { + http: reqwest::Client, + client: OnceCell, +} + +type Environment = dyn Fn(&str) -> Option + Send + Sync; + +pub struct S3Source { + object: S3Object, + config: HttpClientConfig, + env: Arc, + connection: ProcessLocal>, +} + +impl S3Source { + pub fn new(object: S3Object, config: &HttpClientConfig) -> Result { + Self::with_env(object, config, |name| std::env::var(name).ok()) + } + + fn with_env( + object: S3Object, + config: &HttpClientConfig, + env: impl Fn(&str) -> Option + Send + Sync + 'static, + ) -> Result { + let source = Self { + object, + config: config.clone(), + env: Arc::new(env), + connection: ProcessLocal::new(), + }; + source.connection()?; + Ok(source) + } + + fn connection(&self) -> Result<&Connection, litellm_http::Error> { + self.connection + .get_or_init(|| { + let http = reqwest::ClientBuilder::try_from(&self.config)? + .redirect(reqwest::redirect::Policy::none()) + .build()?; + Ok(Connection { + http, + client: OnceCell::new(), + }) + }) + .as_ref() + .map_err(Clone::clone) + } + + async fn client(&self, connection: &Connection) -> aws_sdk_s3::Client { + connection + .client + .get_or_init(|| async { + let loader = aws_config::defaults(BehaviorVersion::latest()) + .http_client(ReqwestHttpClient(connection.http.clone())) + .retry_config(RetryConfig::disabled()); + let loader = match (self.env)("AWS_REGION_NAME").filter(|s| !s.is_empty()) { + Some(region) => loader.region(Region::new(region)), + None => loader, + }; + let defaults = loader.load().await; + litellm_s3::client( + S3ClientConfig { + region: defaults + .region() + .map_or("us-east-1", Region::as_ref) + .to_owned(), + endpoint: (self.env)("AWS_ENDPOINT_URL_S3") + .filter(|s| !s.is_empty()) + .or_else(|| (self.env)("AWS_ENDPOINT_URL").filter(|s| !s.is_empty())), + }, + connection.http.clone(), + defaults + .credentials_provider() + .expect("the default AWS loader enables credentials"), + ) + }) + .await + .clone() + } + + async fn get( + &self, + client: &aws_sdk_s3::Client, + request: &FetchRequest, + ) -> Result>> { + client + .get_object() + .bucket(&self.object.bucket) + .key(&self.object.key) + .set_if_none_match(request.if_none_match.clone()) + .customize() + .config_override( + aws_sdk_s3::config::Builder::new() + .retry_config(RetryConfig::disabled()) + .timeout_config( + TimeoutConfig::builder() + .operation_timeout(request.timeout) + .build(), + ), + ) + .send() + .await + .map_err(Box::new) + } + + async fn fetch_inner(&self, request: &FetchRequest) -> Reply { + let connection = match self.connection() { + Ok(connection) => connection, + Err(_) => { + return Reply::Failed { + reason: "could not configure S3 transport".to_owned(), + }; + } + }; + let client = self.client(connection).await; + let output = self.get(&client, request).await; + let region = output + .as_ref() + .err() + .and_then(|error| redirect_region(error, &client)); + let output = if let Some(region) = region { + let redirected = aws_sdk_s3::Client::from_conf( + client + .config() + .to_builder() + .region(Region::new(region)) + .build(), + ); + self.get(&redirected, request).await + } else { + output + }; + match output { + Ok(output) => { + let validator = output.e_tag; + match output.body.collect().await { + Ok(body) => Reply::Fetched { + body: body.into_bytes(), + validator, + }, + Err(_) => retry("could not read S3 model catalog body"), + } + } + Err(error) => classify(*error), + } + } +} + +#[async_trait] +impl Fetch for S3Source { + async fn fetch(&self, request: FetchRequest) -> Reply { + tokio::time::timeout(request.timeout, self.fetch_inner(&request)) + .await + .unwrap_or_else(|_| retry("S3 model catalog fetch timed out")) + } +} + +fn redirect_region( + error: &SdkError, + client: &aws_sdk_s3::Client, +) -> Option { + let response = error.raw_response()?; + if !matches!(response.status().as_u16(), 301 | 302 | 307 | 400) { + return None; + } + let region = response.headers().get("x-amz-bucket-region")?; + (!region.is_empty() && client.config().region().map(Region::as_ref) != Some(region)) + .then(|| region.to_owned()) +} + +fn classify(error: SdkError) -> Reply { + if let Some(response) = error.raw_response() { + let status = response.status().as_u16(); + if status == 304 { + return Reply::NotModified; + } + let throttled = error.as_service_error().is_some_and(|error| { + matches!( + error.meta().code(), + Some( + "SlowDown" + | "Throttling" + | "ThrottlingException" + | "RequestLimitExceeded" + | "RequestThrottled" + | "RequestThrottledException" + | "TooManyRequestsException" + | "RequestTimeout" + ) + ) + }); + let reason = format!("S3 HTTP {status} fetching model catalog"); + return if status >= 500 || matches!(status, 408 | 429) || throttled { + Reply::Retry { + after: None, + reason, + } + } else { + Reply::Failed { reason } + }; + } + match error { + SdkError::TimeoutError(_) => retry("S3 model catalog fetch timed out"), + SdkError::DispatchFailure(error) if error.is_io() || error.is_timeout() => { + retry("S3 model catalog connection failed") + } + _ => Reply::Failed { + reason: "S3 model catalog credentials or request configuration failed".to_owned(), + }, + } +} + +fn retry(reason: &str) -> Reply { + Reply::Retry { + after: None, + reason: reason.to_owned(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{process::Command, time::Duration}; + + use litellm_http::{EnvironmentProxies, HttpSettings, Resolution}; + use rstest::{fixture, rstest}; + use tokio::{ + io::{AsyncBufReadExt, AsyncWriteExt, BufReader}, + net::TcpListener, + }; + use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{header, header_regex, method, path}, + }; + + const KEYS: [(&str, &str); 2] = [ + ("AWS_ACCESS_KEY_ID", "test-access-key"), + ("AWS_SECRET_ACCESS_KEY", "test-secret-key"), + ]; + + fn isolated(env: &[(&str, &str)]) -> bool { + if std::env::var_os("S3_TEST_CHILD").is_some() { + return true; + } + let directory = tempfile::tempdir().unwrap(); + let config = directory.path().join("config"); + let credentials = directory.path().join("credentials"); + std::fs::write( + &config, + "[profile selected]\nregion = eu-north-1\n\ + [profile process]\nregion = eu-north-1\n\ + credential_process = /bin/echo '{\"Version\":1,\"AccessKeyId\":\"process-key\",\"SecretAccessKey\":\"test-secret-key\"}'\n\ + [profile broken]\ncredential_process = /nonexistent/s3-test-command\n", + ).unwrap(); + std::fs::write( + &credentials, + "[selected]\naws_access_key_id = profile-key\naws_secret_access_key = test-secret-key\n", + ).unwrap(); + let output = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + std::thread::current().name().unwrap(), + "--nocapture", + ]) + .env_clear() + .env("S3_TEST_CHILD", "1") + .env("AWS_EC2_METADATA_DISABLED", "true") + .env("AWS_CONFIG_FILE", config) + .env("AWS_SHARED_CREDENTIALS_FILE", credentials) + .env("HOME", directory.path()) + .envs(env.iter().copied()) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + false + } + + #[fixture] + fn config() -> HttpClientConfig { + Resolution::from(&HttpSettings::default()).config + } + + #[fixture] + fn request() -> FetchRequest { + FetchRequest { + if_none_match: Some("\"old\"".to_owned()), + timeout: Duration::from_secs(3), + } + } + + fn source(config: &HttpClientConfig, endpoint: String) -> S3Source { + S3Source::with_env( + S3Object::parse("s3://bucket/key").unwrap(), + config, + move |name| match name { + "AWS_ENDPOINT_URL_S3" => Some(endpoint.clone()), + "AWS_REGION_NAME" => Some("auto".to_owned()), + _ => None, + }, + ) + .unwrap() + } + + #[rstest] + #[case::object(200, "{}", "fetched")] + #[case::not_modified(304, "", "not_modified")] + #[case::server_error(500, "InternalError", "retry")] + #[case::slow_down(503, "SlowDown", "retry")] + #[case::throttling(400, "Throttling", "retry")] + #[case::rate_limited(429, "TooManyRequests", "retry")] + #[case::forbidden(403, "AccessDenied", "failed")] + #[case::missing_key(404, "NoSuchKey", "failed")] + #[case::missing_bucket(404, "NoSuchBucket", "failed")] + #[case::bad_request(400, "InvalidArgument", "failed")] + #[tokio::test] + async fn status_mapping( + config: HttpClientConfig, + request: FetchRequest, + #[case] status: u16, + #[case] body: &str, + #[case] expected: &str, + ) { + if !isolated(&KEYS) { + return; + } + let server = MockServer::start().await; + let payload = if status >= 400 { + format!("{body}test error") + } else { + body.to_owned() + }; + Mock::given(method("GET")) + .and(path("/bucket/key")) + .and(header("if-none-match", "\"old\"")) + .and(header_regex( + "authorization", + r"Credential=test-access-key/\d+/auto/s3/aws4_request", + )) + .respond_with( + ResponseTemplate::new(status) + .set_body_string(payload) + .insert_header("etag", "\"new\""), + ) + .expect(1) + .mount(&server) + .await; + let reply = source(&config, server.uri()).fetch(request).await; + match (expected, &reply) { + ("fetched", Reply::Fetched { body, validator }) => { + assert_eq!(body.as_ref(), b"{}"); + assert_eq!(validator.as_deref(), Some("\"new\"")); + } + ("not_modified", Reply::NotModified) + | ("retry", Reply::Retry { after: None, .. }) + | ("failed", Reply::Failed { .. }) => {} + _ => panic!("expected {expected}, got {reply:?}"), + } + assert_eq!(server.received_requests().await.unwrap().len(), 1); + server.verify().await; + } + + #[rstest] + #[case::absent(&[])] + #[case::broken_process(&[("AWS_PROFILE", "broken")])] + #[tokio::test] + async fn missing_credentials_send_nothing( + config: HttpClientConfig, + request: FetchRequest, + #[case] env: &[(&str, &str)], + ) { + if !isolated(env) { + return; + } + let server = MockServer::start().await; + let reply = source(&config, server.uri()).fetch(request).await; + assert!(matches!(reply, Reply::Failed { .. }), "{reply:?}"); + assert!(server.received_requests().await.unwrap().is_empty()); + } + + #[rstest] + #[case::env_region(&[("AWS_REGION", "ap-south-1"), ("AWS_DEFAULT_REGION", "us-east-2")], "ap-south-1")] + #[case::default_region(&[("AWS_DEFAULT_REGION", "us-east-2")], "us-east-2")] + #[case::litellm_override(&[("AWS_REGION_NAME", "auto"), ("AWS_REGION", "ap-south-1")], "auto")] + #[case::profile(&[("AWS_PROFILE", "selected")], "eu-north-1")] + #[case::fallback(&[], "us-east-1")] + #[tokio::test] + async fn region_precedence( + config: HttpClientConfig, + request: FetchRequest, + #[case] env: &[(&str, &str)], + #[case] region: &str, + ) { + let vars = [KEYS.as_slice(), env].concat(); + if !isolated(&vars) { + return; + } + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/bucket/key")) + .and(header_regex( + "authorization", + &format!(r"/{region}/s3/aws4_request"), + )) + .respond_with(ResponseTemplate::new(200).set_body_string("{}")) + .expect(1) + .mount(&server) + .await; + let endpoint = server.uri(); + let source = S3Source::with_env( + S3Object::parse("s3://bucket/key").unwrap(), + &config, + move |name| { + if name == "AWS_ENDPOINT_URL_S3" { + Some(endpoint.clone()) + } else { + std::env::var(name).ok() + } + }, + ) + .unwrap(); + assert!(matches!(source.fetch(request).await, Reply::Fetched { .. })); + server.verify().await; + } + + #[rstest] + #[case::shared_file("selected", "profile-key")] + #[case::credential_process("process", "process-key")] + #[tokio::test] + async fn profile_credentials( + config: HttpClientConfig, + request: FetchRequest, + #[case] profile: &str, + #[case] key: &str, + ) { + if !isolated(&[ + ("AWS_PROFILE", profile), + ("AWS_PROFILE_NAME", "ignored"), + ("AWS_ROLE_NAME", "ignored"), + ]) { + return; + } + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/bucket/key")) + .and(header_regex( + "authorization", + &format!(r"Credential={key}/"), + )) + .respond_with(ResponseTemplate::new(200).set_body_string("{}")) + .expect(1) + .mount(&server) + .await; + let reply = source(&config, server.uri()).fetch(request).await; + assert!(matches!(reply, Reply::Fetched { .. }), "{reply:?}"); + server.verify().await; + } + + #[rstest] + #[case::service_endpoint(true)] + #[case::global_endpoint(false)] + #[tokio::test] + async fn endpoint_precedence( + config: HttpClientConfig, + request: FetchRequest, + #[case] service_endpoint: bool, + ) { + if !isolated(&KEYS) { + return; + } + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/bucket/key")) + .respond_with(ResponseTemplate::new(200).set_body_string("{}")) + .expect(1) + .mount(&server) + .await; + let endpoint = server.uri(); + let source = S3Source::with_env( + S3Object::parse("s3://bucket/key").unwrap(), + &config, + move |name| match name { + "AWS_ENDPOINT_URL_S3" if service_endpoint => Some(endpoint.clone()), + "AWS_ENDPOINT_URL" if service_endpoint => Some("http://127.0.0.1:1".to_owned()), + "AWS_ENDPOINT_URL" => Some(endpoint.clone()), + _ => None, + }, + ) + .unwrap(); + assert!(matches!(source.fetch(request).await, Reply::Fetched { .. })); + server.verify().await; + } + + #[rstest] + #[case::redirect(301)] + #[case::wrong_signing_region(400)] + #[tokio::test] + async fn follows_region_once( + config: HttpClientConfig, + request: FetchRequest, + #[case] status: u16, + ) { + if !isolated(&KEYS) { + return; + } + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/bucket/key")) + .and(header_regex("authorization", "/auto/s3/aws4_request")) + .respond_with( + ResponseTemplate::new(status).insert_header("x-amz-bucket-region", "eu-west-1"), + ) + .expect(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/bucket/key")) + .and(header("if-none-match", "\"old\"")) + .and(header_regex("authorization", "/eu-west-1/s3/aws4_request")) + .respond_with(ResponseTemplate::new(200).set_body_string("{}")) + .expect(1) + .mount(&server) + .await; + let reply = source(&config, server.uri()).fetch(request).await; + assert!(matches!(reply, Reply::Fetched { .. }), "{reply:?}"); + assert_eq!(server.received_requests().await.unwrap().len(), 2); + server.verify().await; + } + + #[rstest] + #[tokio::test] + async fn redirect_is_bounded(config: HttpClientConfig, request: FetchRequest) { + if !isolated(&KEYS) { + return; + } + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/bucket/key")) + .respond_with( + ResponseTemplate::new(301).insert_header("x-amz-bucket-region", "eu-west-1"), + ) + .expect(2) + .mount(&server) + .await; + assert!(matches!( + source(&config, server.uri()).fetch(request).await, + Reply::Failed { .. } + )); + server.verify().await; + } + + #[rstest] + #[tokio::test] + async fn timeout_does_not_retry_inside_sdk(config: HttpClientConfig) { + if !isolated(&KEYS) { + return; + } + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/bucket/key")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string("{}") + .set_delay(Duration::from_secs(2)), + ) + .expect(1) + .mount(&server) + .await; + let reply = source(&config, server.uri()) + .fetch(FetchRequest { + if_none_match: None, + timeout: Duration::from_millis(200), + }) + .await; + assert!( + matches!(reply, Reply::Retry { after: None, .. }), + "{reply:?}" + ); + server.verify().await; + } + + #[rstest] + #[tokio::test] + async fn refused_connection_is_retry(config: HttpClientConfig, request: FetchRequest) { + if !isolated(&KEYS) { + return; + } + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + drop(listener); + let reply = source(&config, endpoint).fetch(request).await; + assert!( + matches!(reply, Reply::Retry { after: None, .. }), + "{reply:?}" + ); + } + + #[rstest] + #[case::cut_off(false)] + #[case::stalled(true)] + #[tokio::test] + async fn incomplete_body_is_retry(config: HttpClientConfig, #[case] stall: bool) { + if !isolated(&KEYS) { + return; + } + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let (sent, received) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut reader = BufReader::new(stream); + let mut request = String::new(); + reader.read_line(&mut request).await.unwrap(); + sent.send(request).unwrap(); + let mut stream = reader.into_inner(); + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 1000\r\nConnection: close\r\n\r\n{") + .await + .unwrap(); + if stall { + tokio::time::sleep(Duration::from_secs(2)).await; + } + }); + let reply = source(&config, endpoint) + .fetch(FetchRequest { + if_none_match: None, + timeout: Duration::from_millis(200), + }) + .await; + assert!( + matches!(reply, Reply::Retry { after: None, .. }), + "{reply:?}" + ); + let line = received.await.unwrap(); + assert_eq!(line.split_whitespace().next(), Some("GET")); + assert_eq!( + line.split_whitespace().nth(1).unwrap().split('?').next(), + Some("/bucket/key") + ); + if stall { + server.abort(); + } else { + server.await.unwrap(); + } + } + + #[rstest] + #[tokio::test] + async fn respects_http_proxy(config: HttpClientConfig, request: FetchRequest) { + if !isolated(&KEYS) { + return; + } + let server = MockServer::start().await; + let config = HttpClientConfig { + proxies: EnvironmentProxies::from_environment(&|name: &str| { + (name == "HTTP_PROXY").then(|| server.uri()) + }), + ..config + }; + Mock::given(method("GET")) + .and(path("/bucket/key")) + .respond_with(ResponseTemplate::new(200).set_body_string("{}")) + .expect(1) + .mount(&server) + .await; + assert!(matches!( + source(&config, "http://unresolvable.invalid".to_owned()) + .fetch(request) + .await, + Reply::Fetched { .. } + )); + server.verify().await; + } + + #[rstest] + #[case::ordinary("s3://bucket/key", "bucket", "key")] + #[case::nested("s3://bucket/folder/key.json", "bucket", "folder/key.json")] + #[case::spelling("S3://Bucket/a%2Fb?query#fragment", "Bucket", "a%2Fb")] + #[case::literal_dots("s3://bucket/folder/../key", "bucket", "folder/../key")] + fn source_keeps_original_id(#[case] value: &str, #[case] bucket: &str, #[case] key: &str) { + let source = crate::Source::parse(value).unwrap(); + assert_eq!(source.id().0, value); + let crate::Source::S3(object) = source else { + panic!("expected S3"); + }; + assert_eq!(object.bucket, bucket); + assert_eq!(object.key, key); + } +} diff --git a/litellm-rust/crates/model-catalog-refresh/src/source.rs b/litellm-rust/crates/model-catalog-refresh/src/source.rs index 0da220451c9..6aa4f112ad6 100644 --- a/litellm-rust/crates/model-catalog-refresh/src/source.rs +++ b/litellm-rust/crates/model-catalog-refresh/src/source.rs @@ -11,7 +11,7 @@ use litellm_model_catalog::{FetchRequest, Reply, SourceId}; use reqwest::header::{ETAG, IF_NONE_MATCH, RETRY_AFTER}; use url::Url; -use crate::{SourceError, process_local::ProcessLocal}; +use crate::{S3Object, S3Source, SourceError, process_local::ProcessLocal}; pub const DEFAULT_SOURCE: &str = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"; @@ -20,6 +20,7 @@ pub const DEFAULT_SOURCE: &str = pub enum Source { Http(Url), File(PathBuf), + S3(S3Object), } impl Source { @@ -29,6 +30,7 @@ impl Source { })?; match url.scheme() { "http" | "https" => Ok(Self::Http(url)), + "s3" => S3Object::parse(value).map(Self::S3), "file" => url .to_file_path() .map(Self::File) @@ -44,6 +46,7 @@ impl Source { pub fn id(&self) -> SourceId { SourceId(match self { Self::Http(url) => url.to_string(), + Self::S3(object) => return object.id(), Self::File(path) => Url::from_file_path(path).map_or_else( |()| format!("file://{}", path.display()), |url| url.to_string(), @@ -55,6 +58,7 @@ impl Source { match self { Self::Http(url) => Ok(Arc::new(HttpSource::new(url.clone(), http)?)), Self::File(path) => Ok(Arc::new(FileSource::new(path.clone()))), + Self::S3(object) => Ok(Arc::new(S3Source::new(object.clone(), http)?)), } } } @@ -272,12 +276,19 @@ mod tests { #[case::http("http://example.test/map", Ok("http"))] #[case::file("file:///etc/litellm/my.json", Ok("file"))] #[case::escaped_file("file:///etc/litellm/my%20map.json", Ok("file"))] - #[case::s3("s3://bucket/key", Err("s3"))] + #[case::s3("s3://bucket/key", Ok("s3"))] + #[case::s3_missing_key("s3://bucket", Err("s3-invalid"))] + #[case::s3_empty_key("s3://bucket/", Err("s3-invalid"))] + #[case::s3_missing_bucket("s3:///key", Err("s3-invalid"))] #[case::ftp("ftp://example.test/map", Err("ftp"))] #[case::invalid("not a URL", Err("invalid"))] #[case::missing_host("http://", Err("invalid"))] fn source_parsing(#[case] value: &str, #[case] expected: Result<&str, &str>) { match (Source::parse(value), expected) { + (Ok(source @ Source::S3(_)), Ok("s3")) => assert_eq!(source.id().0, value), + (Err(SourceError::Invalid { reason }), Err("s3-invalid")) => { + assert_eq!(reason, "expected s3://bucket/key") + } (Ok(Source::Http(url)), Ok("http")) => assert_eq!(url.as_str(), value), (Ok(source @ Source::File(_)), Ok("file")) => { assert_eq!(Source::parse(&source.id().0).unwrap(), source); @@ -291,6 +302,12 @@ mod tests { } } + #[rstest] + fn s3_source_is_accepted() { + let value = "s3://bucket/key"; + assert_eq!(Source::parse(value).unwrap().id().0, value); + } + #[rstest] #[case::ok(200, Some("\"new\""), None, "fetched", None)] #[case::other_success(201, None, None, "fetched", None)] diff --git a/litellm-rust/crates/s3/Cargo.toml b/litellm-rust/crates/s3/Cargo.toml new file mode 100644 index 00000000000..c21bbf0867f --- /dev/null +++ b/litellm-rust/crates/s3/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "litellm-s3" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +aws-sdk-s3 = { version = "1.146.1", default-features = false, features = ["rustls", "rt-tokio"] } +aws-credential-types = "1.3.0" +aws-smithy-runtime-api = { version = "1.16.2", features = ["client", "http-1x"] } +aws-smithy-types = { version = "1.6.0", features = ["http-body-1-x"] } +http.workspace = true +reqwest.workspace = true diff --git a/litellm-rust/crates/s3/src/lib.rs b/litellm-rust/crates/s3/src/lib.rs new file mode 100644 index 00000000000..27ef1afd5db --- /dev/null +++ b/litellm-rust/crates/s3/src/lib.rs @@ -0,0 +1,37 @@ +#![allow( + clippy::disallowed_types, + reason = "accepts clients built from HttpClientConfig, including fork-local clients" +)] + +mod transport; + +use aws_credential_types::provider::ProvideCredentials; +use aws_sdk_s3::config::{ + BehaviorVersion, Region, RequestChecksumCalculation, ResponseChecksumValidation, +}; + +pub use transport::ReqwestHttpClient; + +pub struct S3ClientConfig { + pub region: String, + pub endpoint: Option, +} + +pub fn client( + config: S3ClientConfig, + http: reqwest::Client, + credentials: impl ProvideCredentials + 'static, +) -> aws_sdk_s3::Client { + let base = aws_sdk_s3::Config::builder() + .behavior_version(BehaviorVersion::latest()) + .region(Region::new(config.region)) + .http_client(ReqwestHttpClient(http)) + .credentials_provider(credentials) + .request_checksum_calculation(RequestChecksumCalculation::WhenRequired) + .response_checksum_validation(ResponseChecksumValidation::WhenRequired); + let builder = match config.endpoint { + Some(url) => base.endpoint_url(url).force_path_style(true), + None => base, + }; + aws_sdk_s3::Client::from_conf(builder.build()) +} diff --git a/litellm-rust/crates/cache-s3/src/transport.rs b/litellm-rust/crates/s3/src/transport.rs similarity index 95% rename from litellm-rust/crates/cache-s3/src/transport.rs rename to litellm-rust/crates/s3/src/transport.rs index eabc54ac9e2..0967589f44c 100644 --- a/litellm-rust/crates/cache-s3/src/transport.rs +++ b/litellm-rust/crates/s3/src/transport.rs @@ -9,7 +9,7 @@ use aws_smithy_runtime_api::client::{ use aws_smithy_types::body::SdkBody; #[derive(Clone, Debug)] -pub(crate) struct ReqwestHttpClient(pub(crate) litellm_http::Client); +pub struct ReqwestHttpClient(pub reqwest::Client); impl HttpClient for ReqwestHttpClient { fn http_connector(