diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 00cfd8c6f28..84415820d89 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -1911,15 +1911,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "envy" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f47e0157f2cb54f5ae1bd371b30a2ae4311e1c028f575cd4e81de7353215965" -dependencies = [ - "serde", -] - [[package]] name = "equivalent" version = "1.0.2" @@ -3381,9 +3372,11 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", + "tempfile", "thiserror 2.0.19", "tokio", "veil", + "wiremock", ] [[package]] @@ -3460,6 +3453,7 @@ dependencies = [ "litellm-cache-response", "litellm-cache-testing", "litellm-http", + "litellm-layered", "reqwest 0.12.28", "rstest", "serde_json", @@ -3494,6 +3488,7 @@ dependencies = [ "litellm-cache", "litellm-cache-testing", "litellm-http", + "litellm-layered", "percent-encoding", "reqwest 0.12.28", "rstest", @@ -3540,6 +3535,7 @@ version = "0.1.0" dependencies = [ "litellm-cache", "litellm-cache-testing", + "litellm-layered", "r2d2", "redis", "redis-test", @@ -3598,6 +3594,7 @@ dependencies = [ "litellm-cache", "litellm-cache-testing", "litellm-http", + "litellm-layered", "reqwest 0.12.28", "rstest", "serde_json", @@ -3732,11 +3729,11 @@ version = "0.1.0" dependencies = [ "axum", "base64 0.22.1", - "envy", "futures-util", "http-body-util", "litellm-auth-types", "litellm-config", + "litellm-core-utils", "litellm-gateway-auth", "litellm-gateway-inference", "litellm-gateway-mcp", @@ -3759,6 +3756,7 @@ dependencies = [ "tower-sessions-moka-store", "tracing", "uuid", + "wiremock", ] [[package]] @@ -3803,6 +3801,7 @@ dependencies = [ "litellm-inference-ocr", "litellm-inference-responses", "litellm-inference-transcription", + "litellm-layered", "litellm-llms", "litellm-llms-types", "litellm-router", @@ -4037,6 +4036,7 @@ dependencies = [ "litellm-http", "litellm-inference", "litellm-inference-testing", + "litellm-layered", "litellm-llms", "litellm-llms-types", "litellm-router-types", @@ -4184,6 +4184,7 @@ dependencies = [ "litellm-framer", "litellm-host", "litellm-http", + "litellm-layered", "litellm-llms-types", "litellm-python-compat", "litellm-router-types", diff --git a/litellm-rust/clippy.toml b/litellm-rust/clippy.toml index 18663cfdffb..c459f9235e8 100644 --- a/litellm-rust/clippy.toml +++ b/litellm-rust/clippy.toml @@ -19,6 +19,10 @@ disallowed-methods = [ { path = "sqlx::query_as_with", reason = "use sqlx::query_as! or query_file_as! so the SQL is checked against the migrated schema" }, { path = "sqlx::query_scalar_with", reason = "use sqlx::query_scalar! so the SQL is checked against the migrated schema" }, { path = "sqlx::raw_sql", reason = "raw_sql is unchecked; use the checked query macros" }, + { path = "std::env::var", reason = "take a Lookup: Env in the Rust gateway, ProcessEnvironment in python-bridge" }, + { path = "std::env::var_os", reason = "take a Lookup: Env in the Rust gateway, ProcessEnvironment in python-bridge" }, + { path = "std::env::vars", reason = "take a Lookup: Env in the Rust gateway, ProcessEnvironment in python-bridge" }, + { path = "std::env::vars_os", reason = "take a Lookup: Env in the Rust gateway, ProcessEnvironment in python-bridge" }, ] # Every outbound client comes from litellm_http::HttpClientPool so it honors the host's TLS, diff --git a/litellm-rust/crates/auth-aws/Cargo.toml b/litellm-rust/crates/auth-aws/Cargo.toml index 65b411ffaef..014f9341d81 100644 --- a/litellm-rust/crates/auth-aws/Cargo.toml +++ b/litellm-rust/crates/auth-aws/Cargo.toml @@ -25,6 +25,8 @@ aws-smithy-runtime-api = "1.13.0" [dev-dependencies] rstest.workspace = true +tempfile.workspace = true +wiremock.workspace = true litellm-http = { workspace = true, features = ["test-support"] } reqwest.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/auth-aws/src/aws.rs b/litellm-rust/crates/auth-aws/src/aws.rs index 8fdbba21711..34199cec5fa 100644 --- a/litellm-rust/crates/auth-aws/src/aws.rs +++ b/litellm-rust/crates/auth-aws/src/aws.rs @@ -17,10 +17,11 @@ use litellm_auth_types::AwsParams; use super::Error; use super::constants::{ - AWS_ACCESS_KEY_ID, AWS_EXTERNAL_ID, AWS_PROFILE_NAME, AWS_REGION_NAME, AWS_ROLE_ARN, - AWS_ROLE_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_NAME, AWS_SESSION_TOKEN, - AWS_SIGNED_HEADER_NAMES, AWS_STS_ENDPOINT, AWS_WEB_IDENTITY_TOKEN, AWS_WEB_IDENTITY_TOKEN_FILE, - DEFAULT_BEDROCK_REGION, DEFAULT_SESSION_NAME_PREFIX, SIGV4_COMPUTED_HEADER_NAMES, + AWS_ACCESS_KEY_ID, AWS_EXTERNAL_ID, AWS_PROFILE, AWS_PROFILE_NAME, AWS_REGION_NAME, + AWS_ROLE_ARN, AWS_ROLE_NAME, AWS_ROLE_SESSION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_NAME, + AWS_SESSION_TOKEN, AWS_SIGNED_HEADER_NAMES, AWS_STS_ENDPOINT, AWS_WEB_IDENTITY_TOKEN, + AWS_WEB_IDENTITY_TOKEN_FILE, DEFAULT_BEDROCK_REGION, DEFAULT_SESSION_NAME_PREFIX, + SIGV4_COMPUTED_HEADER_NAMES, }; const STATIC_CREDENTIALS_TTL: Duration = Duration::from_secs(3600 - 60); @@ -274,46 +275,26 @@ impl AwsAuthService { .map_err(|error| Error::AwsProfile(error.to_string())) } AwsAuthFlow::AssumeRole { role, session_name } => { - if is_already_running_as_role(&role, &resolved).await? { - let ambient_flow = AwsAuthFlow::DefaultChain; - let key = cache_key(&resolved, &ambient_flow); - if let Some(credentials) = self.get_cached_credentials(&key) { - return Ok(credentials); - } - let provider = - aws_config::default_provider::credentials::DefaultCredentialsChain::builder() - .build() - .await; - let credentials = provider - .provide_credentials() - .await - .map_err(|error| Error::AwsDefaultChain(error.to_string()))?; - self.set_cached_credentials( - key, - credentials.clone(), - credential_cache_ttl(&ambient_flow).unwrap_or(AMBIENT_CREDENTIALS_TTL), - ); - return Ok(credentials); + if Box::pin(self.ambient_is_role(&role, &resolved, env_lookup)).await { + return Box::pin(self.ambient_credentials(&resolved, env_lookup)).await; } - let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); - if let Some(region) = resolved.region_name.clone() { - loader = loader.region(aws_types::region::Region::new(region)); - } - if let Some(endpoint) = resolved.sts_endpoint.clone() { - loader = loader.endpoint_url(endpoint); - } - if let (Some(access_key_id), Some(secret_access_key)) = - (resolved.access_key_id, resolved.secret_access_key) - { - loader = loader.credentials_provider(Credentials::new( + let source = match ( + resolved.access_key_id.clone(), + resolved.secret_access_key.clone(), + ) { + (Some(access_key_id), Some(secret_access_key)) => Credentials::new( access_key_id, secret_access_key, - resolved.session_token, + resolved.session_token.clone(), None, "litellm-role-source", - )); - } - let sdk_config = loader.load().await; + ), + _ => Box::pin(self.ambient_credentials(&resolved, env_lookup)).await?, + }; + let sdk_config = sts_config(&resolved) + .credentials_provider(source) + .load() + .await; let builder = aws_config::sts::AssumeRoleProvider::builder(role); let builder = match session_name { Some(name) => builder.session_name(name), @@ -334,92 +315,175 @@ impl AwsAuthService { role, session_name, } => { - let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); - if let Some(region) = resolved.region_name { - loader = loader.region(aws_types::region::Region::new(region)); - } - if let Some(endpoint) = resolved.sts_endpoint { - loader = loader.endpoint_url(endpoint); - } - let sdk_config = loader.load().await; - let client = aws_sdk_sts::Client::new(&sdk_config); - let response = client - .assume_role_with_web_identity() - .role_arn(role) - .role_session_name(session_name) - .web_identity_token(token) - .send() - .await - .map_err(|error| Error::AwsWebIdentity(error.to_string()))?; - let credentials = response - .credentials() - .ok_or(Error::AwsMissingWebIdentityCredentials)?; - let expiration = SystemTime::try_from(*credentials.expiration()) - .map_err(|error| Error::AwsWebIdentityExpiration(error.to_string()))?; - Ok(Credentials::new( - credentials.access_key_id(), - credentials.secret_access_key(), - Some(credentials.session_token().to_string()), - Some(expiration), - "litellm-web-identity", + Box::pin(web_identity_credentials( + &resolved, + role, + session_name, + token, )) + .await } AwsAuthFlow::DefaultChain => { - let key = cache_key(&resolved, &AwsAuthFlow::DefaultChain); - if let Some(credentials) = self.get_cached_credentials(&key) { - return Ok(credentials); - } - let provider = - aws_config::default_provider::credentials::DefaultCredentialsChain::builder() - .build() - .await; - let credentials = provider - .provide_credentials() - .await - .map_err(|error| Error::AwsDefaultChain(error.to_string()))?; - self.set_cached_credentials( - key, - credentials.clone(), - credential_cache_ttl(&AwsAuthFlow::DefaultChain) - .unwrap_or(AMBIENT_CREDENTIALS_TTL), - ); - Ok(credentials) + Box::pin(self.ambient_credentials(&resolved, env_lookup)).await } } } + + async fn ambient_credentials( + &self, + resolved: &AwsAuthConfig, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + ) -> Result { + let key = cache_key(resolved, &AwsAuthFlow::DefaultChain); + if let Some(credentials) = self.get_cached_credentials(&key) { + return Ok(credentials); + } + let credentials = match ambient(env_lookup) { + Ambient::Keys(credentials) => credentials, + Ambient::WebIdentityFile { + role_arn, + token_file, + session_name, + } => { + let token = std::fs::read_to_string(&token_file) + .map_err(|error| Error::AwsWebIdentity(format!("{token_file}: {error}")))?; + web_identity_credentials(resolved, role_arn, session_name, token).await? + } + Ambient::Chain { profile_name } => { + let builder = + aws_config::default_provider::credentials::DefaultCredentialsChain::builder(); + let builder = match profile_name { + Some(name) => builder.profile_name(&name), + None => builder, + }; + builder + .build() + .await + .provide_credentials() + .await + .map_err(|error| Error::AwsDefaultChain(error.to_string()))? + } + }; + self.set_cached_credentials( + key, + credentials.clone(), + credential_cache_ttl(&AwsAuthFlow::DefaultChain).unwrap_or(AMBIENT_CREDENTIALS_TTL), + ); + Ok(credentials) + } + + async fn ambient_is_role( + &self, + role: &str, + resolved: &AwsAuthConfig, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + ) -> bool { + if role_identity(role).is_none() { + return false; + } + if let Ambient::WebIdentityFile { role_arn, .. } = ambient(env_lookup) { + return same_role_arns(role, &role_arn); + } + let Ok(credentials) = self.ambient_credentials(resolved, env_lookup).await else { + return false; + }; + let sdk_config = sts_config(resolved) + .credentials_provider(credentials) + .load() + .await; + aws_sdk_sts::Client::new(&sdk_config) + .get_caller_identity() + .send() + .await + .is_ok_and(|response| { + response + .arn() + .is_some_and(|caller| same_role_arns(role, caller)) + }) + } } -async fn is_already_running_as_role(role: &str, config: &AwsAuthConfig) -> Result { - if role_identity(role).is_none() { - return Ok(false); - } - if let (Ok(current_role), Ok(token_file)) = ( - std::env::var(AWS_ROLE_ARN), - std::env::var(AWS_WEB_IDENTITY_TOKEN_FILE), - ) && !token_file.is_empty() - { - return Ok(same_role_arns(role, ¤t_role)); - } +/// What the SDK's default credentials chain would find, read through the injected lookup so a +/// config overlay reaches it. Only sources with no environment form are left to the SDK chain. +enum Ambient { + Keys(Credentials), + WebIdentityFile { + role_arn: String, + token_file: String, + session_name: String, + }, + Chain { + profile_name: Option, + }, +} - let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); - if let Some(region) = config.region_name.clone() { - loader = loader.region(aws_types::region::Region::new(region)); +fn ambient(env_lookup: &(dyn Fn(&str) -> Option + Sync)) -> Ambient { + if let (Some(access_key_id), Some(secret_access_key)) = ( + env_lookup(AWS_ACCESS_KEY_ID), + env_lookup(AWS_SECRET_ACCESS_KEY), + ) { + return Ambient::Keys(Credentials::new( + access_key_id, + secret_access_key, + env_lookup(AWS_SESSION_TOKEN), + None, + "litellm-environment", + )); } - if let Some(endpoint) = config.sts_endpoint.clone() { - loader = loader.endpoint_url(endpoint); + if let (Some(role_arn), Some(token_file)) = ( + env_lookup(AWS_ROLE_ARN), + env_lookup(AWS_WEB_IDENTITY_TOKEN_FILE).filter(|file| !file.is_empty()), + ) { + return Ambient::WebIdentityFile { + role_arn, + token_file, + session_name: env_lookup(AWS_ROLE_SESSION_NAME).unwrap_or_else(default_session_name), + }; } - let sdk_config = loader.load().await; - let response = match aws_sdk_sts::Client::new(&sdk_config) - .get_caller_identity() + Ambient::Chain { + profile_name: env_lookup(AWS_PROFILE), + } +} + +fn sts_config(resolved: &AwsAuthConfig) -> aws_config::ConfigLoader { + let loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); + let loader = match resolved.region_name.clone() { + Some(region) => loader.region(aws_types::region::Region::new(region)), + None => loader, + }; + match resolved.sts_endpoint.clone() { + Some(endpoint) => loader.endpoint_url(endpoint), + None => loader, + } +} + +async fn web_identity_credentials( + resolved: &AwsAuthConfig, + role: String, + session_name: String, + token: String, +) -> Result { + let sdk_config = sts_config(resolved).load().await; + let response = aws_sdk_sts::Client::new(&sdk_config) + .assume_role_with_web_identity() + .role_arn(role) + .role_session_name(session_name) + .web_identity_token(token) .send() .await - { - Ok(response) => response, - Err(_) => return Ok(false), - }; - Ok(response - .arn() - .is_some_and(|caller| same_role_arns(role, caller))) + .map_err(|error| Error::AwsWebIdentity(error.to_string()))?; + let credentials = response + .credentials() + .ok_or(Error::AwsMissingWebIdentityCredentials)?; + let expiration = SystemTime::try_from(*credentials.expiration()) + .map_err(|error| Error::AwsWebIdentityExpiration(error.to_string()))?; + Ok(Credentials::new( + credentials.access_key_id(), + credentials.secret_access_key(), + Some(credentials.session_token().to_string()), + Some(expiration), + "litellm-web-identity", + )) } fn default_session_name() -> String { @@ -829,6 +893,66 @@ mod tests { assert_eq!(credentials.session_token(), None); } + #[tokio::test] + async fn lookup_keys_are_the_ambient_credentials_without_a_region() { + let env = |name: &str| match name { + AWS_ACCESS_KEY_ID => Some("lookup-ak".to_owned()), + AWS_SECRET_ACCESS_KEY => Some("lookup-sk".to_owned()), + _ => None, + }; + let credentials = AwsAuthService::default() + .resolve_credentials(AwsAuthConfig::default(), &env) + .await + .unwrap(); + assert_eq!( + (credentials.access_key_id(), credentials.secret_access_key()), + ("lookup-ak", "lookup-sk") + ); + } + + #[tokio::test] + async fn a_role_the_lookup_web_identity_names_is_assumed_with_that_token() { + let role = "arn:aws:iam::123456789012:role/A"; + let sts = wiremock::MockServer::start().await; + wiremock::Mock::given(wiremock::matchers::body_string_contains( + "Action=AssumeRoleWithWebIdentity", + )) + .and(wiremock::matchers::body_string_contains( + "WebIdentityToken=lookup-token", + )) + .respond_with(wiremock::ResponseTemplate::new(200).set_body_string( + "\ + sts-aksts-sk\ + sts-token2099-01-01T00:00:00Z\ + ", + )) + .mount(&sts) + .await; + let token_file = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(token_file.path(), "lookup-token").unwrap(); + let token_path = token_file.path().to_string_lossy().into_owned(); + let env = |name: &str| match name { + AWS_ROLE_ARN => Some(role.to_owned()), + AWS_WEB_IDENTITY_TOKEN_FILE => Some(token_path.clone()), + _ => None, + }; + + let credentials = AwsAuthService::default() + .resolve_credentials( + AwsAuthConfig { + role_name: Some(role.into()), + region_name: Some("us-east-1".into()), + sts_endpoint: Some(sts.uri()), + ..Default::default() + }, + &env, + ) + .await + .unwrap(); + + assert_eq!(credentials.access_key_id(), "sts-ak"); + } + #[test] fn cache_policy_matches_python_flows() { assert_eq!( @@ -1063,6 +1187,7 @@ mod tests { #[ignore] #[tokio::test] + #[allow(clippy::disallowed_methods)] // the live test reads its own credentials async fn live_bedrock_invoke_model_returns_200() -> Result<(), Box> { let access_key_id = std::env::var("AWS_BEDROCK_TEST_ACCESS_KEY_ID")?; let secret_access_key = std::env::var("AWS_BEDROCK_TEST_SECRET_ACCESS_KEY")?; diff --git a/litellm-rust/crates/auth-aws/src/constants.rs b/litellm-rust/crates/auth-aws/src/constants.rs index 789fde156bd..860b4ab7724 100644 --- a/litellm-rust/crates/auth-aws/src/constants.rs +++ b/litellm-rust/crates/auth-aws/src/constants.rs @@ -12,6 +12,8 @@ pub const AWS_PROFILE_NAME: &str = AwsParams::PROFILE_NAME.env[0]; pub const AWS_ROLE_NAME: &str = AwsParams::ROLE_NAME.env[0]; pub const AWS_WEB_IDENTITY_TOKEN: &str = AwsParams::WEB_IDENTITY_TOKEN.env[0]; pub const AWS_ROLE_ARN: &str = "AWS_ROLE_ARN"; +pub const AWS_ROLE_SESSION_NAME: &str = "AWS_ROLE_SESSION_NAME"; +pub const AWS_PROFILE: &str = "AWS_PROFILE"; pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = "AWS_WEB_IDENTITY_TOKEN_FILE"; pub const AWS_STS_ENDPOINT: &str = AwsParams::STS_ENDPOINT.env[0]; pub const AWS_EXTERNAL_ID: &str = AwsParams::EXTERNAL_ID.env[0]; diff --git a/litellm-rust/crates/cache-azure-blob/Cargo.toml b/litellm-rust/crates/cache-azure-blob/Cargo.toml index c28cb90d84a..5b202794636 100644 --- a/litellm-rust/crates/cache-azure-blob/Cargo.toml +++ b/litellm-rust/crates/cache-azure-blob/Cargo.toml @@ -10,6 +10,7 @@ litellm-http.workspace = true litellm-auth-azure.workspace = true litellm-auth-types.workspace = true litellm-cache.workspace = true +litellm-layered.workspace = true async-trait = "0.1" azure_core = "1.1.0" diff --git a/litellm-rust/crates/cache-azure-blob/src/cache.rs b/litellm-rust/crates/cache-azure-blob/src/cache.rs index c5c1fdd8ab9..6ac42b07ba7 100644 --- a/litellm-rust/crates/cache-azure-blob/src/cache.rs +++ b/litellm-rust/crates/cache-azure-blob/src/cache.rs @@ -10,9 +10,11 @@ use azure_storage_blob::{ models::{BlobClientUploadOptions, StorageErrorCode}, }; use futures_util::{TryStreamExt, future::try_join_all}; +use litellm_auth_azure::AzureAuthService; use litellm_cache::{ BaseCache, BatchCache, CacheCodec, DisconnectCache, Error, ExactCacheContext, FlushCache, }; +use litellm_layered::SharedLookup; use tokio::runtime::Handle; use url::Url; @@ -34,11 +36,15 @@ impl AzureBlobCache { http: litellm_http::Client, codec: C, runtime: Handle, + env: SharedLookup, ) -> Result { Self::connect_with_options( account_url, container, - Some(Arc::new(AzureBlobCredential::default())), + Some(Arc::new(AzureBlobCredential::new( + AzureAuthService::default(), + env, + ))), ClientOptions { transport: Some(Transport::new(Arc::new(ReqwestTransport(http)))), ..ClientOptions::default() diff --git a/litellm-rust/crates/cache-azure-blob/src/credential.rs b/litellm-rust/crates/cache-azure-blob/src/credential.rs index d1a3d0e44ec..a51f08cd154 100644 --- a/litellm-rust/crates/cache-azure-blob/src/credential.rs +++ b/litellm-rust/crates/cache-azure-blob/src/credential.rs @@ -1,6 +1,5 @@ use std::{ fmt, - sync::Arc, time::{Duration, SystemTime}, }; @@ -11,15 +10,14 @@ use azure_core::{ }; use litellm_auth_azure::{AzureAuthInputs, AzureAuthService}; use litellm_auth_types::ResolvedCredential; +use litellm_layered::SharedLookup; const STATIC_TOKEN_LIFETIME: Duration = Duration::from_secs(300); const LLM_TOKEN_ENV: &str = "AZURE_AD_TOKEN"; -type EnvLookup = Arc Option + Send + Sync>; - pub struct AzureBlobCredential { service: AzureAuthService, - env_lookup: EnvLookup, + env_lookup: SharedLookup, } impl fmt::Debug for AzureBlobCredential { @@ -28,17 +26,8 @@ impl fmt::Debug for AzureBlobCredential { } } -impl Default for AzureBlobCredential { - fn default() -> Self { - Self::new( - AzureAuthService::default(), - Arc::new(|name| std::env::var(name).ok()), - ) - } -} - impl AzureBlobCredential { - pub fn new(service: AzureAuthService, env_lookup: EnvLookup) -> Self { + pub fn new(service: AzureAuthService, env_lookup: SharedLookup) -> Self { Self { service, env_lookup, @@ -54,7 +43,11 @@ impl TokenCredential for AzureBlobCredential { _options: Option>, ) -> azure_core::Result { let env_lookup = &self.env_lookup; - let lookup = move |name: &str| (name != LLM_TOKEN_ENV).then(|| env_lookup(name)).flatten(); + let lookup = move |name: &str| { + (name != LLM_TOKEN_ENV) + .then(|| env_lookup.get(name)) + .flatten() + }; let credential = self .service .get_azure_ad_token( diff --git a/litellm-rust/crates/cache-gcs/Cargo.toml b/litellm-rust/crates/cache-gcs/Cargo.toml index 91630879cbe..4e6f3dd8cd1 100644 --- a/litellm-rust/crates/cache-gcs/Cargo.toml +++ b/litellm-rust/crates/cache-gcs/Cargo.toml @@ -11,6 +11,7 @@ futures-util.workspace = true litellm-auth-gcp.workspace = true litellm-auth-types.workspace = true litellm-cache.workspace = true +litellm-layered.workspace = true percent-encoding.workspace = true reqwest.workspace = true tokio.workspace = true @@ -18,6 +19,7 @@ tokio.workspace = true [dev-dependencies] litellm-http = { workspace = true, features = ["test-support"] } litellm-cache-testing.workspace = true +litellm-layered.workspace = true rstest.workspace = true serde_json.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/cache-gcs/src/cache.rs b/litellm-rust/crates/cache-gcs/src/cache.rs index bad81573cb4..734e49f8074 100644 --- a/litellm-rust/crates/cache-gcs/src/cache.rs +++ b/litellm-rust/crates/cache-gcs/src/cache.rs @@ -6,6 +6,7 @@ use litellm_cache::{ FlushCache, }; use litellm_http::Client; +use litellm_layered::SharedLookup; use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_encode}; use crate::{GcpTokenSource, TokenSource}; @@ -53,8 +54,11 @@ pub struct GcsCache { } impl GcsCache { - pub fn new(config: GcsConfig, client: Client, codec: S) -> Self { - let token = Arc::new(GcpTokenSource::new(config.path_service_account.clone())); + pub fn new(config: GcsConfig, client: Client, codec: S, env: SharedLookup) -> Self { + let token = Arc::new(GcpTokenSource::new( + config.path_service_account.clone(), + env, + )); Self::with_token_source(config, client, codec, token) } diff --git a/litellm-rust/crates/cache-gcs/src/token.rs b/litellm-rust/crates/cache-gcs/src/token.rs index adb601c276c..a4ffc9fb931 100644 --- a/litellm-rust/crates/cache-gcs/src/token.rs +++ b/litellm-rust/crates/cache-gcs/src/token.rs @@ -3,6 +3,7 @@ use std::{future::Future, pin::Pin}; use litellm_auth_gcp::{VertexAuth, VertexConfig}; use litellm_auth_types::{InputSource, SecretValue, Sourced}; use litellm_cache::Error; +use litellm_layered::SharedLookup; pub trait TokenSource: Send + Sync + 'static { fn bearer_token(&self) -> Pin> + Send + '_>>; @@ -11,15 +12,17 @@ pub trait TokenSource: Send + Sync + 'static { pub struct GcpTokenSource { auth: VertexAuth, config: VertexConfig, + env: SharedLookup, } impl GcpTokenSource { - pub fn new(path_service_account: Option) -> Self { + pub fn new(path_service_account: Option, env: SharedLookup) -> Self { let credentials = path_service_account .map(|path| Sourced::new(SecretValue::new(path), InputSource::Deployment)); Self { auth: VertexAuth::default(), config: VertexConfig::new(credentials, None, None), + env, } } } @@ -28,7 +31,7 @@ impl TokenSource for GcpTokenSource { fn bearer_token(&self) -> Pin> + Send + '_>> { Box::pin(async move { self.auth - .access_token(&self.config, &|name| std::env::var(name).ok()) + .access_token(&self.config, &|name| self.env.get(name)) .await .map_err(|_| Error::Unavailable) }) diff --git a/litellm-rust/crates/cache-gcs/tests/cache.rs b/litellm-rust/crates/cache-gcs/tests/cache.rs index 096b691ea16..8cf250711de 100644 --- a/litellm-rust/crates/cache-gcs/tests/cache.rs +++ b/litellm-rust/crates/cache-gcs/tests/cache.rs @@ -99,6 +99,7 @@ async fn cache_exposes_its_configuration(#[future(awt)] server: MockServer) { }, litellm_http::Client::plain_for_test(), litellm_cache::JsonCodec::::new(), + std::sync::Arc::new(litellm_layered::Env::default()), ); assert_eq!(cache.bucket_name(), "bucket"); assert_eq!(cache.key_prefix(), "folder/"); diff --git a/litellm-rust/crates/cache-redis-semantic/tests/cache.rs b/litellm-rust/crates/cache-redis-semantic/tests/cache.rs index fa4ac00e767..30b56bc4f04 100644 --- a/litellm-rust/crates/cache-redis-semantic/tests/cache.rs +++ b/litellm-rust/crates/cache-redis-semantic/tests/cache.rs @@ -1,3 +1,4 @@ +#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment mod support; use std::time::Duration; diff --git a/litellm-rust/crates/cache-redis/Cargo.toml b/litellm-rust/crates/cache-redis/Cargo.toml index a234286a338..f8dfc97f757 100644 --- a/litellm-rust/crates/cache-redis/Cargo.toml +++ b/litellm-rust/crates/cache-redis/Cargo.toml @@ -7,6 +7,7 @@ repository.workspace = true [dependencies] litellm-cache.workspace = true +litellm-layered.workspace = true redis = { version = "1.7.0", features = ["cluster", "tls-rustls"] } r2d2 = "0.8.10" tokio.workspace = true diff --git a/litellm-rust/crates/cache-redis/src/cache.rs b/litellm-rust/crates/cache-redis/src/cache.rs index ebcf0b6916b..6ea753d0210 100644 --- a/litellm-rust/crates/cache-redis/src/cache.rs +++ b/litellm-rust/crates/cache-redis/src/cache.rs @@ -4,6 +4,7 @@ use std::{ }; use litellm_cache::{BatchEntry, CacheCodec, Error}; +use litellm_layered::Lookup; use crate::{ connection::{ConnectionRef, Connections}, @@ -20,11 +21,17 @@ pub struct RedisCache { pub(crate) topology: RedisTopology, /// The server's major version, read from `INFO` once, like Python's `redis_version`. pub(crate) major_version: Arc>, + default_major_version: u32, } impl RedisCache { - pub fn new(url: &str, default_ttl: Option, codec: S) -> Result { - Self::connect(url, &RedisTopology::Standalone, default_ttl, codec) + pub fn new( + url: &str, + default_ttl: Option, + codec: S, + env: &(impl Lookup + ?Sized), + ) -> Result { + Self::connect(url, &RedisTopology::Standalone, default_ttl, codec, env) } pub fn connect( @@ -32,6 +39,7 @@ impl RedisCache { topology: &RedisTopology, default_ttl: Option, codec: S, + env: &(impl Lookup + ?Sized), ) -> Result { let connections = Connections::open(url, topology)?; Ok(Self { @@ -41,6 +49,7 @@ impl RedisCache { namespace: None, topology: topology.clone(), major_version: Arc::default(), + default_major_version: default_major_version(env), }) } } @@ -50,7 +59,12 @@ where S: CacheCodec, C: redis::ConnectionLike + Send + 'static, { - pub fn with_connection(connection: C, default_ttl: Option, codec: S) -> Self { + pub fn with_connection( + connection: C, + default_ttl: Option, + codec: S, + env: &(impl Lookup + ?Sized), + ) -> Self { Self { connections: Arc::new(Connections::fixed(connection)), default_ttl: default_ttl.unwrap_or(DEFAULT_TTL), @@ -58,6 +72,7 @@ where namespace: None, topology: RedisTopology::Standalone, major_version: Arc::default(), + default_major_version: default_major_version(env), } } @@ -109,7 +124,7 @@ where let version = info .ok() .and_then(|info| parse_major_version(&info)) - .unwrap_or_else(default_major_version); + .unwrap_or(self.default_major_version); *self.major_version.get_or_init(|| version) } @@ -184,9 +199,9 @@ fn parse_major_version(info: &str) -> Option { } } -fn default_major_version() -> u32 { - std::env::var("DEFAULT_REDIS_MAJOR_VERSION") - .ok() +/// `DEFAULT_REDIS_MAJOR_VERSION`, the version assumed when `INFO` does not answer. +fn default_major_version(env: &(impl Lookup + ?Sized)) -> u32 { + env.get("DEFAULT_REDIS_MAJOR_VERSION") .and_then(|value| value.parse().ok()) .unwrap_or(7) } diff --git a/litellm-rust/crates/cache-redis/tests/cache.rs b/litellm-rust/crates/cache-redis/tests/cache.rs index 70baeffd572..05ba09757bd 100644 --- a/litellm-rust/crates/cache-redis/tests/cache.rs +++ b/litellm-rust/crates/cache-redis/tests/cache.rs @@ -25,11 +25,11 @@ fn mock(commands: Vec) -> MockRedisConnection { } fn tagged(commands: Vec) -> Mocked { - RedisCache::with_connection(mock(commands), None, TaggedByteCodec(42)) + RedisCache::with_connection(mock(commands), None, TaggedByteCodec(42), &|_: &str| None) } fn json_cache(commands: Vec) -> Mocked> { - RedisCache::with_connection(mock(commands), None, JsonCodec::new()) + RedisCache::with_connection(mock(commands), None, JsonCodec::new(), &|_: &str| None) } fn team(commands: Vec) -> Mocked> { @@ -55,7 +55,15 @@ fn scan(pattern: &str, cursor: u64, count: usize, reply: redis::Value) -> MockCm #[rstest] fn constructor_rejects_invalid_urls() { - assert!(RedisCache::new("not a redis url", None, JsonCodec::::new()).is_err()); + assert!( + RedisCache::new( + "not a redis url", + None, + JsonCodec::::new(), + &|_: &str| None + ) + .is_err() + ); } #[rstest] @@ -144,6 +152,7 @@ async fn async_operations_preserve_codec_ttl_and_missing_values(context: ExactCa ]), Some(Duration::from_secs(9)), TaggedByteCodec(42), + &|_: &str| None, ); cache .batch_cache_write("counter", 7, context.clone()) @@ -342,6 +351,7 @@ async fn ttl_pipeline_keeps_each_entry_ttl_and_defaults_missing_ones() { )]), Some(Duration::from_secs(300)), JsonCodec::::new(), + &|_: &str| None, ) .with_namespace(Some("ns".into())); @@ -616,6 +626,35 @@ async fn counted_lpop_assumes_redis_7_when_info_fails() { ); } +#[rstest] +#[tokio::test] +async fn counted_lpop_pops_one_at_a_time_when_info_fails_and_the_env_names_an_older_version() { + let single_pop = || { + let mut pipeline = redis::pipe(); + pipeline.cmd("LPOP").arg("team:queue"); + pipeline + }; + let cache = RedisCache::with_connection( + mock(vec![ + MockCmd::new( + redis::cmd("INFO"), + Err::(redis::RedisError::from((redis::ErrorKind::Io, "down"))), + ), + MockCmd::with_values(single_pop(), Ok(vec![redis_test::redis_value!("a")])), + MockCmd::with_values(single_pop(), Ok(vec![redis_test::redis_value!("b")])), + ]), + None, + JsonCodec::::new(), + &litellm_layered::Env::of(&[("DEFAULT_REDIS_MAJOR_VERSION", "5")]), + ) + .with_namespace(Some("team".into())); + + assert_eq!( + cache.async_lpop("queue", Some(2)).await.unwrap(), + RedisLpopResult::Values(vec![b"a".to_vec(), b"b".to_vec()]) + ); +} + fn push_and_trim(start: i64) -> redis::Pipeline { let mut pipeline = redis::pipe(); pipeline @@ -924,6 +963,7 @@ async fn increments_keep_an_existing_ttl_in_one_atomic_script(context: ExactCach mock(vec![increment_script(2.5), increment_script(2.5)]), None, JsonCodec::::new(), + &|_: &str| None, ); assert_eq!( @@ -1153,6 +1193,12 @@ async fn disconnect_keeps_a_caller_owned_connection_usable() { #[rstest] #[tokio::test] async fn disconnect_drains_an_idle_pool_without_connecting() { - let cache = RedisCache::new("redis://127.0.0.1:1", None, JsonCodec::::new()).unwrap(); + let cache = RedisCache::new( + "redis://127.0.0.1:1", + None, + JsonCodec::::new(), + &|_: &str| None, + ) + .unwrap(); cache.disconnect().await.unwrap(); } diff --git a/litellm-rust/crates/cache-redis/tests/cluster.rs b/litellm-rust/crates/cache-redis/tests/cluster.rs index c1a9a70a5d2..82ccbb7adb6 100644 --- a/litellm-rust/crates/cache-redis/tests/cluster.rs +++ b/litellm-rust/crates/cache-redis/tests/cluster.rs @@ -64,6 +64,7 @@ fn constructor_rejects_unusable_cluster_configs( &RedisTopology::Cluster { startup_nodes }, None, JsonCodec::new(), + &|_: &str| None, ) .err(); assert!(matches!(error, Some(Error::Unavailable))); diff --git a/litellm-rust/crates/cache-redis/tests/support/mod.rs b/litellm-rust/crates/cache-redis/tests/support/mod.rs index bff1e618582..4c527230f88 100644 --- a/litellm-rust/crates/cache-redis/tests/support/mod.rs +++ b/litellm-rust/crates/cache-redis/tests/support/mod.rs @@ -1,3 +1,4 @@ +#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment #![allow(dead_code)] use std::{ @@ -183,8 +184,10 @@ impl redis::ConnectionLike for FakeRedis { pub type JsonCache = RedisCache, C>; pub fn fake_cache(namespace: &str) -> JsonCache { - RedisCache::with_connection(FakeRedis::default(), None, JsonCodec::new()) - .with_namespace(Some(namespace.into())) + RedisCache::with_connection(FakeRedis::default(), None, JsonCodec::new(), &|_: &str| { + None + }) + .with_namespace(Some(namespace.into())) } /// Startup nodes from `LITELLM_TEST_REDIS_CLUSTER_NODES` (`host:port,host:port`); tests that @@ -224,8 +227,14 @@ pub fn cluster_cache( ) -> Option> { let topology = cluster_topology()?; Some( - RedisCache::connect(&cluster_url(), &topology, Some(default_ttl), codec) - .expect("cluster connection") - .with_namespace(Some(unique_namespace(label))), + RedisCache::connect( + &cluster_url(), + &topology, + Some(default_ttl), + codec, + &|_: &str| None, + ) + .expect("cluster connection") + .with_namespace(Some(unique_namespace(label))), ) } diff --git a/litellm-rust/crates/cache-response/tests/support/mod.rs b/litellm-rust/crates/cache-response/tests/support/mod.rs index b992a937b70..31b8bebc1eb 100644 --- a/litellm-rust/crates/cache-response/tests/support/mod.rs +++ b/litellm-rust/crates/cache-response/tests/support/mod.rs @@ -34,7 +34,7 @@ pub fn keyed(key: &str) -> ResponseCacheRequest { pub fn redis(commands: Vec, namespace: Option<&str>) -> ResponseCache { let connection = MockRedisConnection::new(commands).assert_all_commands_consumed(); ResponseCache::new(Arc::new( - RedisCache::with_connection(connection, None, ResponseCacheCodec) + RedisCache::with_connection(connection, None, ResponseCacheCodec, &|_: &str| None) .with_namespace(namespace.map(str::to_owned)), )) } diff --git a/litellm-rust/crates/cache-s3/Cargo.toml b/litellm-rust/crates/cache-s3/Cargo.toml index eb3a2fff1ac..af49e463bf2 100644 --- a/litellm-rust/crates/cache-s3/Cargo.toml +++ b/litellm-rust/crates/cache-s3/Cargo.toml @@ -9,6 +9,7 @@ repository.workspace = true litellm-http.workspace = true litellm-cache.workspace = true litellm-auth-aws.workspace = true +litellm-layered.workspace = true aws-sdk-s3 = { version = "1.146.1", default-features = false, features = ["rustls", "rt-tokio"] } aws-credential-types = "1.3.0" aws-smithy-runtime-api = { version = "1.16.2", features = ["client", "http-1x"] } @@ -22,6 +23,7 @@ tokio.workspace = true [dev-dependencies] litellm-http = { workspace = true, features = ["test-support"] } litellm-cache-testing.workspace = true +litellm-layered.workspace = true rstest.workspace = true wiremock.workspace = true serde_json.workspace = true diff --git a/litellm-rust/crates/cache-s3/src/auth.rs b/litellm-rust/crates/cache-s3/src/auth.rs index f4f06e5371d..62aa10f292e 100644 --- a/litellm-rust/crates/cache-s3/src/auth.rs +++ b/litellm-rust/crates/cache-s3/src/auth.rs @@ -3,20 +3,17 @@ use aws_credential_types::{ provider::{ProvideCredentials, error::CredentialsError, future}, }; use litellm_auth_aws::{AwsAuthConfig, AwsAuthService}; +use litellm_layered::SharedLookup; #[derive(Clone)] pub struct S3Credentials { auth: AwsAuthService, config: AwsAuthConfig, - env: fn(&str) -> Option, + env: SharedLookup, } impl S3Credentials { - pub fn new(config: AwsAuthConfig) -> Self { - Self::with_env(config, |name| std::env::var(name).ok()) - } - - pub fn with_env(config: AwsAuthConfig, env: fn(&str) -> Option) -> Self { + pub fn new(config: AwsAuthConfig, env: SharedLookup) -> Self { Self { auth: AwsAuthService::default(), config, @@ -44,7 +41,7 @@ impl ProvideCredentials for S3Credentials { )); } self.auth - .resolve_credentials(self.config.clone(), &self.env) + .resolve_credentials(self.config.clone(), &|name| self.env.get(name)) .await .map_err(|_| CredentialsError::provider_error("S3 cache authentication failed")) }) diff --git a/litellm-rust/crates/cache-s3/src/cache.rs b/litellm-rust/crates/cache-s3/src/cache.rs index ced948e80c6..fb5e27659cd 100644 --- a/litellm-rust/crates/cache-s3/src/cache.rs +++ b/litellm-rust/crates/cache-s3/src/cache.rs @@ -15,6 +15,7 @@ use litellm_auth_aws::AwsAuthConfig; use litellm_cache::{ BaseCache, BatchCache, CacheCodec, DisconnectCache, Error, ExactCacheContext, FlushCache, }; +use litellm_layered::SharedLookup; use tokio::runtime::Handle; use crate::{auth::S3Credentials, transport::ReqwestHttpClient}; @@ -47,13 +48,14 @@ impl S3Cache { http: litellm_http::Client, codec: C, runtime: Handle, + env: SharedLookup, ) -> Self { let endpoint_url: Option = config.endpoint.map(|endpoint| endpoint.url); let base = aws_sdk_s3::Config::builder() .behavior_version(BehaviorVersion::latest()) .region(Region::new(config.region.clone())) .http_client(ReqwestHttpClient(http)) - .credentials_provider(S3Credentials::new(config.auth)) + .credentials_provider(S3Credentials::new(config.auth, env)) .request_checksum_calculation(RequestChecksumCalculation::WhenRequired) .response_checksum_validation(ResponseChecksumValidation::WhenRequired); let builder = match &endpoint_url { diff --git a/litellm-rust/crates/cache-s3/tests/auth.rs b/litellm-rust/crates/cache-s3/tests/auth.rs index 59c5aa2e091..2794cceb8ef 100644 --- a/litellm-rust/crates/cache-s3/tests/auth.rs +++ b/litellm-rust/crates/cache-s3/tests/auth.rs @@ -42,7 +42,7 @@ async fn credentials_resolve( #[case] env: fn(&str) -> Option, #[case] expected: (&str, &str, Option<&str>), ) { - let credentials = S3Credentials::with_env(config, env) + let credentials = S3Credentials::new(config, std::sync::Arc::new(env)) .provide_credentials() .await .unwrap(); diff --git a/litellm-rust/crates/cache-s3/tests/support/mod.rs b/litellm-rust/crates/cache-s3/tests/support/mod.rs index b628c1df431..64d65158efc 100644 --- a/litellm-rust/crates/cache-s3/tests/support/mod.rs +++ b/litellm-rust/crates/cache-s3/tests/support/mod.rs @@ -37,6 +37,7 @@ pub fn cache_with(config: S3CacheConfig, runtime: Handle) -> JsonS3Cache { litellm_http::Client::plain_for_test(), JsonCodec::new(), runtime, + std::sync::Arc::new(litellm_layered::Env::default()), ) } diff --git a/litellm-rust/crates/core-utils/src/settings.rs b/litellm-rust/crates/core-utils/src/settings.rs index 67ae4e04bea..eea28d0666f 100644 --- a/litellm-rust/crates/core-utils/src/settings.rs +++ b/litellm-rust/crates/core-utils/src/settings.rs @@ -1,26 +1,5 @@ -use std::str::FromStr; - -use crate::coercion::parse_str_bool; - pub trait Lookup { fn get(&self, name: &str) -> Option; - - fn truthy(&self, name: &str) -> Option { - self.get(name).filter(|value| !value.is_empty()) - } - - fn enabled(&self, name: &str) -> Option { - self.get(name) - .is_some_and(|value| parse_str_bool(&value) == Some(true)) - .then_some(true) - } - - fn parsed(&self, name: &str) -> Option - where - Self: Sized, - { - self.get(name).and_then(|value| value.trim().parse().ok()) - } } impl Option> Lookup for F { @@ -29,14 +8,6 @@ impl Option> Lookup for F { } } -pub struct ProcessEnvironment; - -impl Lookup for ProcessEnvironment { - fn get(&self, name: &str) -> Option { - std::env::var(name).ok() - } -} - pub fn resolve_non_empty( value: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, @@ -72,35 +43,4 @@ mod tests { assert_eq!(env.get("EMPTY"), Some(String::new())); assert_eq!(env.get("ABSENT"), None); } - - #[test] - fn truthy_drops_an_empty_value_like_a_python_or_chain() { - let env = env_of(&[("EMPTY", ""), ("SET", "value")]); - assert_eq!(env.truthy("EMPTY"), None); - assert_eq!(env.truthy("SET").as_deref(), Some("value")); - } - - #[test] - fn enabled_only_switches_on_for_true_and_never_forces_off() { - let env = env_of(&[ - ("LOWER", "true"), - ("PADDED", " True "), - ("OFF", "false"), - ("ONE", "1"), - ]); - assert_eq!(env.enabled("LOWER"), Some(true)); - assert_eq!(env.enabled("PADDED"), Some(true)); - assert_eq!(env.enabled("OFF"), None); - assert_eq!(env.enabled("ONE"), None); - assert_eq!(env.enabled("ABSENT"), None); - } - - #[test] - fn parsed_trims_and_skips_values_that_do_not_parse() { - let env = env_of(&[("PADDED", " 45 "), ("WORD", "soon"), ("FRACTION", "0.5")]); - assert_eq!(env.parsed::("PADDED"), Some(45)); - assert_eq!(env.parsed::("WORD"), None); - assert_eq!(env.parsed::("FRACTION"), Some(0.5)); - assert_eq!(env.parsed::("ABSENT"), None); - } } diff --git a/litellm-rust/crates/db-testing/src/bin/sqlx-prepare.rs b/litellm-rust/crates/db-testing/src/bin/sqlx-prepare.rs index f61ee1e02c7..24d7398ea34 100644 --- a/litellm-rust/crates/db-testing/src/bin/sqlx-prepare.rs +++ b/litellm-rust/crates/db-testing/src/bin/sqlx-prepare.rs @@ -1,3 +1,4 @@ +#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment use std::{ env, process::{Command, ExitCode}, diff --git a/litellm-rust/crates/gateway-inference/Cargo.toml b/litellm-rust/crates/gateway-inference/Cargo.toml index 312d44b9a4b..29c33ac7b0d 100644 --- a/litellm-rust/crates/gateway-inference/Cargo.toml +++ b/litellm-rust/crates/gateway-inference/Cargo.toml @@ -22,6 +22,7 @@ litellm-host-http.workspace = true litellm-host.workspace = true litellm-http.workspace = true litellm-llms.workspace = true +litellm-layered.workspace = true litellm-router.workspace = true litellm-secrets.workspace = true litellm-llms-types.workspace = true diff --git a/litellm-rust/crates/gateway-inference/src/lib.rs b/litellm-rust/crates/gateway-inference/src/lib.rs index 4663151f1de..532de9f1f86 100644 --- a/litellm-rust/crates/gateway-inference/src/lib.rs +++ b/litellm-rust/crates/gateway-inference/src/lib.rs @@ -22,6 +22,7 @@ use litellm_inference_messages::MessagesRoute; use litellm_inference_ocr::OcrRoute; use litellm_inference_responses::ResponsesRoute; use litellm_inference_transcription::AudioTranscriptionRoute; +use litellm_layered::SharedLookup; use litellm_llms::base_llm::ocr::{handler::OcrClient, settings::OcrSettings}; use litellm_secrets::source::SecretSource; @@ -53,6 +54,8 @@ impl Gateway { pub fn new( resources: CoreResources, http: HttpClientConfig, + ocr: OcrSettings, + env: SharedLookup, secrets: Arc, models: ModelRouter, ) -> Result { @@ -70,14 +73,14 @@ impl Gateway { auth.clone(), secrets.clone(), ), - messages: MessagesRoute::new(provider.clone(), auth.clone(), secrets.clone()), + messages: MessagesRoute::new(provider.clone(), auth.clone(), secrets.clone(), env), responses: ResponsesRoute::new(provider, auth.clone(), secrets.clone()), ocr: OcrRoute::new(OcrClient::new( &resources.pool, &http, UrlPolicy::default(), auth, - OcrSettings::default(), + ocr, secrets.clone(), )?), models, diff --git a/litellm-rust/crates/gateway-inference/tests/support/mod.rs b/litellm-rust/crates/gateway-inference/tests/support/mod.rs index 0e37bb0aff9..45b36389410 100644 --- a/litellm-rust/crates/gateway-inference/tests/support/mod.rs +++ b/litellm-rust/crates/gateway-inference/tests/support/mod.rs @@ -13,6 +13,7 @@ use futures_util::future::BoxFuture; use litellm_gateway_inference::{Deployment, Gateway, router}; use litellm_http::{HttpClientPool, HttpSettings, Resolution, media::PublicDnsResolver}; use litellm_inference::resources::CoreResources; +use litellm_llms::base_llm::ocr::settings::OcrSettings; use litellm_secrets::{SecretValue, source::SecretSource}; use serde_json::Value; use tower::ServiceExt; @@ -83,6 +84,8 @@ fn configured_app( let gateway = Gateway::new( resources, http, + OcrSettings::default(), + Arc::new(litellm_layered::Env::default()), secrets, [( "public/model".into(), diff --git a/litellm-rust/crates/gateway/Cargo.toml b/litellm-rust/crates/gateway/Cargo.toml index 32ca2893748..9f49988b2bf 100644 --- a/litellm-rust/crates/gateway/Cargo.toml +++ b/litellm-rust/crates/gateway/Cargo.toml @@ -7,7 +7,6 @@ repository.workspace = true [dependencies] axum.workspace = true -envy = "0.4.2" http-body-util = "0.1" litellm-inference.workspace = true litellm-gateway-inference.workspace = true @@ -19,6 +18,7 @@ litellm-gateway-auth.workspace = true litellm-gateway-ui.workspace = true litellm-auth-types.workspace = true litellm-config.workspace = true +litellm-core-utils.workspace = true litellm-http = { workspace = true, features = ["mcp"] } litellm-layered.workspace = true litellm-llms.workspace = true @@ -38,3 +38,4 @@ serde_yaml_ng = "0.10.0" tempfile.workspace = true tokio = { workspace = true, features = ["sync"] } tower = { version = "0.5", features = ["util"] } +wiremock.workspace = true diff --git a/litellm-rust/crates/gateway/src/lib.rs b/litellm-rust/crates/gateway/src/lib.rs index 050c53ac574..a9617cf2630 100644 --- a/litellm-rust/crates/gateway/src/lib.rs +++ b/litellm-rust/crates/gateway/src/lib.rs @@ -20,30 +20,35 @@ use litellm_gateway_auth::Auth; use litellm_gateway_inference::{Gateway, ModelRouter}; use litellm_http::{ClientVariant, HttpClientPool, Resolution, media::PublicDnsResolver}; use litellm_inference::resources::CoreResources; +use litellm_layered::Env; +use litellm_llms::base_llm::ocr::settings::{OcrKeysPartial, OcrSettings}; use litellm_secrets::source::EnvironmentSecrets; use litellm_tracing::ByteChunk; use uuid::Uuid; -pub fn build_inference(config: &Config) -> Result, Error> { +pub fn build_inference(config: &Config, process: &Env) -> Result, Error> { let pool = Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver))); let environment = secrets::environment_values(&config.environment_variables)?; - let lookup = |name: &str| { + let env = process.clone().overlaid( environment - .get(name) - .map(|value| value.expose().to_owned()) - .or_else(|| std::env::var(name).ok()) - }; - let http_settings = http::http_settings(&lookup, &config.litellm_settings)?; + .iter() + .map(|(name, value)| (name.clone(), value.expose().to_owned())), + ); + let http_settings = http::http_settings(&env, &config.litellm_settings)?; + let ocr = OcrSettings::from(OcrKeysPartial::from_env(&env)?); let http = Resolution::from(&http_settings).config; let client = pool.client(&http, ClientVariant::Provider)?; + let env = Arc::new(env); let secrets = Arc::new(secrets::ConfigSecrets::new( environment, - Arc::new(EnvironmentSecrets::python_compatible(client)), + Arc::new(EnvironmentSecrets::python_compatible(client, env.clone())), )); let resources = CoreResources::new(pool); Ok(Arc::new(Gateway::new( resources, http, + ocr, + env, secrets, ModelRouter::from_model_list(&config.model_list), )?)) diff --git a/litellm-rust/crates/gateway/src/main.rs b/litellm-rust/crates/gateway/src/main.rs index f8150e6e860..ad3bee7036b 100644 --- a/litellm-rust/crates/gateway/src/main.rs +++ b/litellm-rust/crates/gateway/src/main.rs @@ -10,7 +10,8 @@ use serde_json::json; mod settings; -use settings::{Settings, UiSettings}; +use litellm_layered::Env; +use settings::{BootKeysPartial, Settings, UiKeysPartial, UiSettings}; struct StderrSink { level: Level, @@ -41,14 +42,15 @@ impl Sink for StderrSink { #[tokio::main] async fn main() -> Result<(), Box> { - let settings: Settings = settings::from_iter(std::env::vars_os())?; + let env = Env::from_process(); + let settings = Settings::from(BootKeysPartial::from_env(&env)?); let level = settings.log_level(); Logger::new(StderrSink { level }).install_global()?; let config = Config::load(settings.litellm_config)?; - let inference = litellm_gateway::build_inference(&config)?; - let ui = match std::env::var_os("LITELLM_UI_PATH") { + let inference = litellm_gateway::build_inference(&config, &env)?; + let ui = match &settings.ui_path { Some(directory) => { - let ui_settings: UiSettings = settings::from_iter(std::env::vars_os())?; + let ui_settings = UiSettings::try_from(UiKeysPartial::from_env(&env)?)?; let backend = UiBackend::new(ui_settings.ui_username, ui_settings.ui_password)?; Some( litellm_gateway_ui::router( diff --git a/litellm-rust/crates/gateway/src/settings.rs b/litellm-rust/crates/gateway/src/settings.rs index 2dd8872065f..725897829ec 100644 --- a/litellm-rust/crates/gateway/src/settings.rs +++ b/litellm-rust/crates/gateway/src/settings.rs @@ -1,25 +1,39 @@ -use std::ffi::OsString; - use litellm_auth_types::SecretValue; +use litellm_core_utils::coercion; +use litellm_layered::{Invalid, Layered}; use litellm_tracing::Level; -use serde::{Deserialize, de::DeserializeOwned}; -#[derive(Deserialize)] -#[serde(default)] +#[derive(Layered)] +#[expect(dead_code, reason = "only BootKeysPartial is built")] +pub(super) struct BootKeys { + #[layer(env = "HOST")] + host: String, + #[layer(env = "PORT", read = coercion::required::)] + port: u16, + #[layer(env = "LITELLM_CONFIG")] + litellm_config: String, + #[layer(env = "RUST_LOG")] + rust_log: String, + #[layer(env = "LITELLM_UI_PATH")] + ui_path: String, +} + pub(super) struct Settings { pub host: String, pub port: u16, pub litellm_config: String, + pub ui_path: Option, rust_log: Option, } -impl Default for Settings { - fn default() -> Self { +impl From for Settings { + fn from(keys: BootKeysPartial) -> Self { Self { - host: "0.0.0.0".into(), - port: 4000, - litellm_config: "config.yaml".into(), - rust_log: None, + host: keys.host.unwrap_or_else(|| "0.0.0.0".into()), + port: keys.port.unwrap_or(4000), + litellm_config: keys.litellm_config.unwrap_or_else(|| "config.yaml".into()), + ui_path: keys.ui_path, + rust_log: keys.rust_log, } } } @@ -33,64 +47,81 @@ impl Settings { } } -#[derive(Deserialize)] +#[derive(Layered)] +#[expect(dead_code, reason = "only UiKeysPartial is built")] +pub(super) struct UiKeys { + #[layer(env = "UI_USERNAME")] + ui_username: String, + #[layer(env = "UI_PASSWORD", read = secret, secret)] + ui_password: SecretValue, + #[layer(env = "LITELLM_UI_SECURE_COOKIES", read = coercion::required::)] + litellm_ui_secure_cookies: bool, +} + +fn secret(value: &str) -> Result, Invalid> { + Ok(Some(SecretValue::new(value))) +} + pub(super) struct UiSettings { - #[serde(default = "default_username")] pub ui_username: String, pub ui_password: SecretValue, - #[serde(default = "default_secure_cookies")] pub litellm_ui_secure_cookies: bool, } -fn default_username() -> String { - "admin".into() -} +#[derive(Debug, thiserror::Error)] +#[error("UI_PASSWORD must be set when LITELLM_UI_PATH serves the UI")] +pub(super) struct MissingUiPassword; -fn default_secure_cookies() -> bool { - true -} +impl TryFrom for UiSettings { + type Error = MissingUiPassword; -pub(super) fn from_iter( - variables: impl IntoIterator, -) -> envy::Result { - envy::from_iter( - variables - .into_iter() - .filter_map(|(key, value)| Some((key.into_string().ok()?, value.into_string().ok()?))), - ) + fn try_from(keys: UiKeysPartial) -> Result { + Ok(Self { + ui_username: keys.ui_username.unwrap_or_else(|| "admin".into()), + ui_password: keys.ui_password.ok_or(MissingUiPassword)?, + litellm_ui_secure_cookies: keys.litellm_ui_secure_cookies.unwrap_or(true), + }) + } } #[cfg(test)] mod tests { + use std::ffi::OsString; + + use litellm_layered::Env; use rstest::rstest; use super::*; - fn parse(variables: &[(&str, &str)]) -> envy::Result { - from_iter( - variables - .iter() - .map(|(key, value)| (OsString::from(key), OsString::from(value))), - ) + fn boot(variables: &[(&str, &str)]) -> Result { + BootKeysPartial::from_env(&Env::of(variables)).map(Settings::from) + } + + fn ui(variables: &[(&str, &str)]) -> Result> { + Ok(UiSettings::try_from(UiKeysPartial::from_env(&Env::of( + variables, + ))?)?) } #[rstest] fn defaults_allow_startup_without_ui_credentials() { - let settings: Settings = parse(&[]).unwrap(); + let settings = boot(&[]).unwrap(); assert_eq!(settings.host, "0.0.0.0"); assert_eq!(settings.port, 4000); assert_eq!(settings.litellm_config, "config.yaml"); + assert_eq!(settings.ui_path, None); assert_eq!(settings.log_level(), Level::INFO); } #[rstest] fn environment_overrides_startup_settings() { - let settings: Settings = parse(&[ + let settings = boot(&[ ("HOST", "127.0.0.1"), ("PORT", "8080"), ("LITELLM_CONFIG", "/tmp/custom.yaml"), ("RUST_LOG", "debug"), + ("LITELLM_UI_PATH", "/srv/ui"), ("LITELLM_UI_SECURE_COOKIES", "invalid-but-ui-disabled"), ]) .unwrap(); @@ -98,6 +129,7 @@ mod tests { assert_eq!(settings.host, "127.0.0.1"); assert_eq!(settings.port, 8080); assert_eq!(settings.litellm_config, "/tmp/custom.yaml"); + assert_eq!(settings.ui_path.as_deref(), Some("/srv/ui")); assert_eq!(settings.log_level(), Level::DEBUG); } @@ -106,23 +138,42 @@ mod tests { #[case::filter("litellm=debug")] #[case::invalid("invalid")] fn unrecognized_log_levels_fall_back_to_info(#[case] value: &str) { - let settings: Settings = parse(&[("RUST_LOG", value)]).unwrap(); - - assert_eq!(settings.log_level(), Level::INFO); + assert_eq!( + boot(&[("RUST_LOG", value)]).unwrap().log_level(), + Level::INFO + ); } #[rstest] - #[case::empty("")] #[case::negative("-1")] #[case::overflow("65536")] - #[case::invalid("http")] - fn invalid_ports_fail_startup(#[case] value: &str) { - assert!(parse::(&[("PORT", value)]).is_err()); + #[case::invalid("abc")] + #[case::empty("")] + #[case::blank(" ")] + fn invalid_ports_fail_startup_naming_the_variable(#[case] value: &str) { + let error = boot(&[("PORT", value)]).err().unwrap(); + assert_eq!(error.name, "PORT"); + assert!(error.to_string().contains("PORT")); + } + + #[rstest] + fn blank_text_settings_remain_set() { + let settings = boot(&[ + ("HOST", ""), + ("LITELLM_CONFIG", ""), + ("LITELLM_UI_PATH", ""), + ]) + .unwrap(); + + assert_eq!(settings.host, ""); + assert_eq!(settings.litellm_config, ""); + assert_eq!(settings.port, 4000); + assert_eq!(settings.ui_path.as_deref(), Some("")); } #[rstest] fn ui_defaults_keep_secure_cookies_and_preserve_password() { - let settings: UiSettings = parse(&[("UI_PASSWORD", " secret,with spaces ")]).unwrap(); + let settings = ui(&[("UI_PASSWORD", " secret,with spaces ")]).unwrap(); assert_eq!(settings.ui_username, "admin"); assert_eq!(settings.ui_password.expose(), " secret,with spaces "); @@ -133,7 +184,7 @@ mod tests { #[case::secure("true", true)] #[case::insecure("false", false)] fn ui_environment_overrides(#[case] value: &str, #[case] expected: bool) { - let settings: UiSettings = parse(&[ + let settings = ui(&[ ("UI_USERNAME", "operator"), ("UI_PASSWORD", "password"), ("LITELLM_UI_SECURE_COOKIES", value), @@ -148,7 +199,16 @@ mod tests { #[case::missing_password(&[])] #[case::invalid_cookie_flag(&[("UI_PASSWORD", "password"), ("LITELLM_UI_SECURE_COOKIES", "invalid")])] fn invalid_ui_settings_fail(#[case] variables: &[(&str, &str)]) { - assert!(parse::(variables).is_err()); + assert!(ui(variables).is_err()); + } + + #[rstest] + #[case::empty("")] + #[case::whitespace(" ")] + fn blank_ui_credentials_remain_set(#[case] value: &str) { + let settings = ui(&[("UI_PASSWORD", value), ("UI_USERNAME", value)]).unwrap(); + assert_eq!(settings.ui_password.expose(), value); + assert_eq!(settings.ui_username, value); } #[cfg(unix)] @@ -158,12 +218,12 @@ mod tests { fn non_unicode_environment_values_are_ignored(#[case] key: &str) { use std::os::unix::ffi::OsStringExt; - let settings: Settings = from_iter([ + let env = Env::from_vars([ (OsString::from(key), OsString::from_vec(vec![0xff])), (OsString::from_vec(vec![0xff]), OsString::from("value")), (OsString::from("PORT"), OsString::from("8080")), - ]) - .unwrap(); + ]); + let settings = Settings::from(BootKeysPartial::from_env(&env).unwrap()); assert_eq!(settings.host, "0.0.0.0"); assert_eq!(settings.port, 8080); diff --git a/litellm-rust/crates/gateway/tests/mcp.rs b/litellm-rust/crates/gateway/tests/mcp.rs index 593fbfeb129..d111e119535 100644 --- a/litellm-rust/crates/gateway/tests/mcp.rs +++ b/litellm-rust/crates/gateway/tests/mcp.rs @@ -126,7 +126,8 @@ mcp_servers: upstream.url )) .unwrap(); - let inference = litellm_gateway::build_inference(&config).unwrap(); + let inference = + litellm_gateway::build_inference(&config, &litellm_layered::Env::default()).unwrap(); let shutdown = CancellationToken::new(); let mcp = litellm_gateway::build_mcp( &config, @@ -243,7 +244,8 @@ async fn rejects_unsupported_or_invalid_settings_before_serving( #[case] expected: &str, ) { let config = Config::from_yaml(&format!("general_settings: {{master_key: gateway-key}}\nmcp_servers:\n docs:\n url: http://127.0.0.1:1/mcp\n {settings}\n")).unwrap(); - let inference = litellm_gateway::build_inference(&config).unwrap(); + let inference = + litellm_gateway::build_inference(&config, &litellm_layered::Env::default()).unwrap(); let result = litellm_gateway::build_mcp( &config, Arc::new(NoSecrets), @@ -261,7 +263,8 @@ async fn rejects_unsupported_or_invalid_settings_before_serving( #[tokio::test] async fn mcp_is_optional_and_requires_admission_credentials_when_enabled() { let empty = Config::from_yaml("{}").unwrap(); - let inference = litellm_gateway::build_inference(&empty).unwrap(); + let inference = + litellm_gateway::build_inference(&empty, &litellm_layered::Env::default()).unwrap(); assert!( litellm_gateway::build_mcp( &empty, @@ -311,7 +314,9 @@ fn applies_configured_http_environment_before_building_clients() { missing.display(), )) .unwrap(); - let error = litellm_gateway::build_inference(&config).err().unwrap(); + let error = litellm_gateway::build_inference(&config, &litellm_layered::Env::default()) + .err() + .unwrap(); assert!(matches!( error, litellm_gateway::Error::Http(litellm_http::Error::Read { path, .. }) @@ -329,7 +334,8 @@ async fn refuses_unimplemented_global_mcp_policy(#[case] settings: &str) { "{settings}\nmcp_servers: {{docs: {{url: 'http://127.0.0.1:1/mcp'}}}}\n" )) .unwrap(); - let inference = litellm_gateway::build_inference(&config).unwrap(); + let inference = + litellm_gateway::build_inference(&config, &litellm_layered::Env::default()).unwrap(); let error = litellm_gateway::build_mcp( &config, Arc::new(NoSecrets), diff --git a/litellm-rust/crates/gateway/tests/server.rs b/litellm-rust/crates/gateway/tests/server.rs index 71edd3c3650..f153e2cae67 100644 --- a/litellm-rust/crates/gateway/tests/server.rs +++ b/litellm-rust/crates/gateway/tests/server.rs @@ -13,6 +13,7 @@ use rstest::{fixture, rstest}; use serde_json::{Value, json}; use tokio::{net::TcpListener, sync::oneshot, time::timeout}; use tower::ServiceExt; +use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method}; struct LogSink(mpsc::Sender); @@ -30,7 +31,11 @@ impl Sink for LogSink { #[fixture] fn inference() -> Arc { - litellm_gateway::build_inference(&Config::from_yaml("model_list: []").unwrap()).unwrap() + litellm_gateway::build_inference( + &Config::from_yaml("model_list: []").unwrap(), + &litellm_layered::Env::default(), + ) + .unwrap() } #[rstest] @@ -113,6 +118,174 @@ async fn authenticates_before_serving_mounted_inference_routes( .unwrap(); } +#[rstest] +#[tokio::test] +async fn ocr_gives_up_at_the_request_timeout_from_config_environment_variables() { + let upstream = MockServer::start().await; + Mock::given(method("POST")) + .respond_with( + ResponseTemplate::new(200) + .set_delay(Duration::from_secs(3)) + .set_body_json(json!({"pages": []})), + ) + .mount(&upstream) + .await; + let config = Config::from_yaml(&format!( + "model_list: + - model_name: public/ocr + litellm_params: + model: mistral/test-ocr + api_base: {} + api_key: test-key +general_settings: + master_key: gateway-key +environment_variables: + REQUEST_TIMEOUT: '0.2' +", + upstream.uri() + )) + .unwrap(); + let inference = + litellm_gateway::build_inference(&config, &litellm_layered::Env::default()).unwrap(); + let request = Request::post("/ocr") + .header("authorization", "Bearer gateway-key") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "model": "public/ocr", + "document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"}, + }) + .to_string(), + )) + .unwrap(); + + let response = timeout( + Duration::from_secs(2), + litellm_gateway::router(inference, &config, None, None).oneshot(request), + ) + .await + .expect("OCR must give up at REQUEST_TIMEOUT instead of the default") + .unwrap(); + + assert_eq!(response.status().as_u16(), 408); +} + +#[rstest] +#[tokio::test] +async fn messages_thinking_budgets_read_config_environment_variables() { + let upstream = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}, + }))) + .mount(&upstream) + .await; + let config = Config::from_yaml(&format!( + "model_list: + - model_name: public/claude + litellm_params: + model: anthropic/claude-sonnet-4-5 + api_key: sk-ant-test + api_base: {} +general_settings: + master_key: gateway-key +environment_variables: + DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET: '2000' +", + upstream.uri() + )) + .unwrap(); + let inference = + litellm_gateway::build_inference(&config, &litellm_layered::Env::default()).unwrap(); + let request = Request::post("/v1/messages") + .header("authorization", "Bearer gateway-key") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "model": "public/claude", + "max_tokens": 4096, + "reasoning_effort": "low", + "messages": [{"role": "user", "content": "hi"}], + }) + .to_string(), + )) + .unwrap(); + + let response = timeout( + Duration::from_secs(10), + litellm_gateway::router(inference, &config, None, None).oneshot(request), + ) + .await + .unwrap() + .unwrap(); + + assert_eq!(response.status().as_u16(), 200); + let sent: Value = upstream.received_requests().await.unwrap()[0] + .body_json() + .unwrap(); + assert_eq!(sent["thinking"]["budget_tokens"], 2000); +} + +#[rstest] +#[tokio::test] +async fn messages_auth_reads_credentials_from_config_environment_variables() { + let directory = tempfile::tempdir().unwrap(); + let credentials = directory.path().join("service-account.json"); + std::fs::write(&credentials, "not a service account").unwrap(); + let config = Config::from_yaml(&format!( + "model_list: + - model_name: public/vertex + litellm_params: + model: vertex_ai/claude-sonnet-4-5 + vertex_project: test-project + vertex_location: us-east5 +general_settings: + master_key: gateway-key +environment_variables: + GOOGLE_APPLICATION_CREDENTIALS: '{}' +", + credentials.display() + )) + .unwrap(); + let inference = + litellm_gateway::build_inference(&config, &litellm_layered::Env::default()).unwrap(); + let request = Request::post("/v1/messages") + .header("authorization", "Bearer gateway-key") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "model": "public/vertex", + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + }) + .to_string(), + )) + .unwrap(); + + let response = timeout( + Duration::from_secs(10), + litellm_gateway::router(inference, &config, None, None).oneshot(request), + ) + .await + .unwrap() + .unwrap(); + + let body = axum::body::to_bytes(response.into_body(), 64 * 1024) + .await + .unwrap(); + assert!( + String::from_utf8_lossy(&body).contains("ApplicationCredentials"), + "{}", + String::from_utf8_lossy(&body) + ); +} + #[rstest] #[tokio::test] async fn logs_request_outcome_without_credentials_or_query(inference: Arc) { diff --git a/litellm-rust/crates/inference-messages/Cargo.toml b/litellm-rust/crates/inference-messages/Cargo.toml index e310a0be9b1..74d80f31d93 100644 --- a/litellm-rust/crates/inference-messages/Cargo.toml +++ b/litellm-rust/crates/inference-messages/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-layered.workspace = true bytes.workspace = true futures-util.workspace = true litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] } diff --git a/litellm-rust/crates/inference-messages/src/handler.rs b/litellm-rust/crates/inference-messages/src/handler.rs index 55c80d9bddf..84e0381da74 100644 --- a/litellm-rust/crates/inference-messages/src/handler.rs +++ b/litellm-rust/crates/inference-messages/src/handler.rs @@ -62,6 +62,7 @@ pub(super) async fn execute( model, call, secrets.as_ref(), + route.env.as_ref(), context, ) .boxed() @@ -100,6 +101,7 @@ async fn provider_call( model: String, call: MessagesCall, secrets: &(dyn Lookup + Send + Sync), + env: &(dyn Lookup + Send + Sync), context: &CallContext<'_, impl Interceptors>, ) -> Result { let MessagesCall { @@ -143,7 +145,7 @@ async fn provider_call( let body = config.transform_anthropic_messages_request( trimmed, - &MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params), + &MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params, env), )?; let environment = ValidatedEnvironment { headers: config.request_headers( @@ -168,7 +170,10 @@ async fn provider_call( secret_fields: Vec::new(), api_key: api_key.map(SecretValue::new), }; - let authenticated = resolve_auth(auth, environment, &|key| std::env::var(key).ok()).await?; + let authenticated = resolve_auth(auth, environment, &|key| { + secrets.get(key).or_else(|| env.get(key)) + }) + .await?; let identity = ProviderIdentity { model: request_context.model.clone(), provider: request_context.custom_llm_provider.clone(), @@ -436,6 +441,7 @@ mod tests { model, call, secrets, + &|_: &str| None, &context, ) .await diff --git a/litellm-rust/crates/inference-messages/src/lib.rs b/litellm-rust/crates/inference-messages/src/lib.rs index bfefb9c63b0..e9831fdcd2b 100644 --- a/litellm-rust/crates/inference-messages/src/lib.rs +++ b/litellm-rust/crates/inference-messages/src/lib.rs @@ -9,6 +9,7 @@ use litellm_auth::AuthServices; use litellm_host::interceptors::Interceptors; use litellm_inference::context::CallContext; use litellm_inference::provider::resolve_llm_provider; +use litellm_layered::SharedLookup; use litellm_secrets::source::SecretSource; use std::sync::Arc; @@ -25,6 +26,7 @@ pub struct MessagesRoute { http: litellm_http::Client, auth: Arc, secrets: Arc, + env: SharedLookup, cache: Option, } @@ -33,11 +35,13 @@ impl MessagesRoute { http: litellm_http::Client, auth: Arc, secrets: Arc, + env: SharedLookup, ) -> Self { Self { http, auth, secrets, + env, cache: None, } } diff --git a/litellm-rust/crates/inference-messages/tests/live/support.rs b/litellm-rust/crates/inference-messages/tests/live/support.rs index 81e295e34ad..bd591e473a8 100644 --- a/litellm-rust/crates/inference-messages/tests/live/support.rs +++ b/litellm-rust/crates/inference-messages/tests/live/support.rs @@ -69,7 +69,12 @@ impl Interceptors for LiveCall { #[rstest::fixture] pub(super) fn route() -> MessagesRoute { let resources = LiveResources::default(); - MessagesRoute::new(resources.http, resources.auth, resources.secrets) + MessagesRoute::new( + resources.http, + resources.auth, + resources.secrets, + std::sync::Arc::new(litellm_layered::Env::from_process()), + ) } pub(super) async fn request( diff --git a/litellm-rust/crates/inference-messages/tests/messages/response.rs b/litellm-rust/crates/inference-messages/tests/messages/response.rs index b5d623a5582..83225f94cbf 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/response.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/response.rs @@ -277,6 +277,7 @@ async fn the_facade_sends_through_the_injected_http_pool_configuration(call: Mes provider_http(&resources, &Resolution::from(&settings).config), resources.auth, no_secrets(), + std::sync::Arc::new(litellm_layered::Env::of(&[])), ) .execute( MessagesCall { @@ -360,6 +361,7 @@ async fn route_uses_injected_dependencies_and_optional_cache( provider_http(&resources, &http_config()), resources.auth.clone(), Arc::new(RecordingSecrets::new([("ANTHROPIC_API_KEY", "route-key")])), + std::sync::Arc::new(litellm_layered::Env::of(&[])), ); let route = if caching { route.with_cache(ScopedCache::new( diff --git a/litellm-rust/crates/inference-messages/tests/support/mod.rs b/litellm-rust/crates/inference-messages/tests/support/mod.rs index 3cecc865070..34bb26241af 100644 --- a/litellm-rust/crates/inference-messages/tests/support/mod.rs +++ b/litellm-rust/crates/inference-messages/tests/support/mod.rs @@ -22,6 +22,7 @@ pub fn messages_route(secrets: Arc) -> litellm_inference_messa provider_http(&resources, &http_config()), resources.auth, secrets, + std::sync::Arc::new(litellm_layered::Env::of(&[])), ) } diff --git a/litellm-rust/crates/inference-ocr/src/provider_config.rs b/litellm-rust/crates/inference-ocr/src/provider_config.rs index 09041e9b72b..1345078dfc1 100644 --- a/litellm-rust/crates/inference-ocr/src/provider_config.rs +++ b/litellm-rust/crates/inference-ocr/src/provider_config.rs @@ -155,7 +155,7 @@ impl OcrConfigKind { LlmProviders::AzureAi => (None, Some("AZURE_AI_API_BASE")), _ => (None, None), }; - let secret = |name: &str| secrets.truthy(name); + let secret = |name: &str| secrets.get(name).filter(|value| !value.is_empty()); let dynamic_api_key = credentials.dynamic_api_key.or_else(|| { credentials.api_key.clone().or_else(|| { preferred_api_key_env diff --git a/litellm-rust/crates/inference-responses/tests/live/openrouter.rs b/litellm-rust/crates/inference-responses/tests/live/openrouter.rs index 364dd213ec1..74f54bfe750 100644 --- a/litellm-rust/crates/inference-responses/tests/live/openrouter.rs +++ b/litellm-rust/crates/inference-responses/tests/live/openrouter.rs @@ -1,3 +1,4 @@ +#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment use std::{sync::Arc, time::Duration}; use futures_util::TryStreamExt; diff --git a/litellm-rust/crates/inference-testing/src/live.rs b/litellm-rust/crates/inference-testing/src/live.rs index 986ece5d383..ebd280188e1 100644 --- a/litellm-rust/crates/inference-testing/src/live.rs +++ b/litellm-rust/crates/inference-testing/src/live.rs @@ -32,11 +32,15 @@ impl Default for LiveResources { Self { http, auth: resources.auth, - secrets: Arc::new(EnvironmentSecrets::python_compatible(secrets_http)), + secrets: Arc::new(EnvironmentSecrets::python_compatible( + secrets_http, + Arc::new(Env::from_process()), + )), } } } +#[allow(clippy::disallowed_methods)] // live tests read their own model names pub fn model(route: &str, provider: &str) -> String { let name = format!("LITELLM_LIVE_{route}_{provider}_MODEL").to_ascii_uppercase(); std::env::var(&name) diff --git a/litellm-rust/crates/layered/AGENTS.md b/litellm-rust/crates/layered/AGENTS.md index 8137dfaebaf..e975ac8107b 100644 --- a/litellm-rust/crates/layered/AGENTS.md +++ b/litellm-rust/crates/layered/AGENTS.md @@ -17,6 +17,8 @@ Each runtime writes its own `merge([..])` list per group, and a test per runtime - `from_env` and `from_litellm_params` are generated, and every runtime that has that source calls them - A source holding typed values (the config file, a live Python global) gets a hand-written constructor in the crate that owns the source, never in `layered` or in the group's crate - `layered` never depends on `litellm-config` or pyo3, and no library crate reads the process environment itself. It takes a `Lookup`: `Env` in the Rust gateway, `ProcessEnvironment` in `python-bridge` +- `ProcessEnvironment` is private to `python-bridge`, so a library constructor takes the `Lookup` as a required argument and never falls back to the process through `new()`, `Default` or an optional builder step +- An SDK that reads the process environment itself, like `aws-config`, gets every value that has an environment form from the `Lookup` first. Only sources with no environment form (profiles, ECS, IMDS) are left to the SDK ## Hand-written constructors list every field diff --git a/litellm-rust/crates/layered/src/env.rs b/litellm-rust/crates/layered/src/env.rs index f88f4129930..96c633ea416 100644 --- a/litellm-rust/crates/layered/src/env.rs +++ b/litellm-rust/crates/layered/src/env.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, fmt}; +use std::{collections::BTreeMap, ffi::OsString, fmt}; use crate::Lookup; @@ -8,8 +8,12 @@ pub struct Env(BTreeMap); impl Env { #[allow(clippy::disallowed_methods)] pub fn from_process() -> Self { + Self::from_vars(std::env::vars_os()) + } + + pub fn from_vars(vars: impl IntoIterator) -> Self { Self( - std::env::vars_os() + vars.into_iter() .filter_map(|(name, value)| { Some((name.into_string().ok()?, value.into_string().ok()?)) }) diff --git a/litellm-rust/crates/layered/src/lib.rs b/litellm-rust/crates/layered/src/lib.rs index 2d7afea6e25..b6208b8ca94 100644 --- a/litellm-rust/crates/layered/src/lib.rs +++ b/litellm-rust/crates/layered/src/lib.rs @@ -8,6 +8,8 @@ pub use litellm_core_utils::{Invalid, coercion::FromEnv, settings::Lookup}; pub use litellm_layered_derive::Layered; pub use sources::Sources; +pub type SharedLookup = std::sync::Arc; + pub trait Partial: Default { fn or(self, lower: Self) -> Self; } diff --git a/litellm-rust/crates/llms/Cargo.toml b/litellm-rust/crates/llms/Cargo.toml index 2d3ea85d1fc..27b9a9dfb56 100644 --- a/litellm-rust/crates/llms/Cargo.toml +++ b/litellm-rust/crates/llms/Cargo.toml @@ -11,6 +11,7 @@ test-support = ["litellm-http/test-support"] [dependencies] litellm-llms-types.workspace = true litellm-core-utils.workspace = true +litellm-layered.workspace = true litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] } litellm-auth-aws.workspace = true litellm-auth-azure.workspace = true diff --git a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs index 6657cbf262f..07215655ac7 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs @@ -371,10 +371,7 @@ pub fn map_openai_context_management_to_anthropic( #[cfg(test)] mod tests { - use crate::base_llm::messages::context::{ - MessagesModelCapabilities, ThinkingBudgets, ThinkingContext, - }; - use std::process::Command; + use crate::base_llm::messages::context::MessagesModelCapabilities; use rstest::{fixture, rstest}; @@ -388,7 +385,6 @@ mod tests { const OAUTH_BETA: &str = "oauth-2025-04-20"; const BROWSER_ACCESS: (&str, &str) = ("anthropic-dangerous-direct-browser-access", "true"); const LOW_BUDGET_ENV: &str = "DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET"; - const PROCESS_ENV_PROBE: &str = "LITELLM_MESSAGES_TRANSFORM_CONTEXT_PROBE"; fn merged(base: Value, fields: Value) -> Value { Value::Object( @@ -443,7 +439,7 @@ mod tests { ANTHROPIC_MESSAGES_CONFIG .transform_anthropic_messages_request( request(fields), - &MessagesTransformContext::with_lookup(capabilities, drop_params, &no_env), + &MessagesTransformContext::new(capabilities, drop_params, &no_env), ) .map(|transformed| serde_json::to_value(transformed).unwrap()) } @@ -768,7 +764,7 @@ mod tests { #[test] fn thinking_is_translated_with_the_context_budgets() { - let context = MessagesTransformContext::with_lookup( + let context = MessagesTransformContext::new( MessagesModelCapabilities { supports_reasoning: true, ..Default::default() @@ -791,54 +787,6 @@ mod tests { ); } - #[test] - fn new_reads_thinking_budgets_from_the_process_environment() { - if std::env::var_os(PROCESS_ENV_PROBE).is_some() { - assert_eq!( - MessagesTransformContext::new(sampling_removed(), true), - MessagesTransformContext { - thinking: ThinkingContext { - capabilities: sampling_removed(), - budgets: ThinkingBudgets { - low: 2000, - ..ThinkingBudgets::default() - }, - }, - drop_params: true, - } - ); - return; - } - let (_, test_path) = concat!( - module_path!(), - "::new_reads_thinking_budgets_from_the_process_environment" - ) - .split_once("::") - .unwrap(); - let other_tiers = ["MINIMAL", "MEDIUM", "HIGH", "XHIGH", "MAX"] - .map(|tier| format!("DEFAULT_REASONING_EFFORT_{tier}_THINKING_BUDGET")); - let output = other_tiers - .iter() - .fold( - Command::new(std::env::current_exe().unwrap()), - |mut command, name| { - command.env_remove(name); - command - }, - ) - .args([test_path, "--exact"]) - .env(PROCESS_ENV_PROBE, "1") - .env(LOW_BUDGET_ENV, "2000") - .output() - .unwrap(); - let stdout = String::from_utf8_lossy(&output.stdout); - assert!( - output.status.success() && stdout.contains("1 passed"), - "{stdout}{}", - String::from_utf8_lossy(&output.stderr) - ); - } - #[rstest] #[case::public_endpoint(None, &[], "https://api.anthropic.com/v1/messages")] #[case::base_url_env( diff --git a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs index bf934fb4d4c..cdca699ffb7 100644 --- a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs @@ -379,7 +379,7 @@ mod tests { "inference_geo": "us", "litellm_metadata": {"trace": "abc"} }); - let context = MessagesTransformContext::with_lookup( + let context = MessagesTransformContext::new( MessagesModelCapabilities { supports_reasoning: true, supports_adaptive_thinking: true, @@ -406,6 +406,7 @@ mod tests { ..MessagesModelCapabilities::default() }, false, + &|_: &str| None, ) } @@ -507,7 +508,7 @@ mod tests { {"role": "assistant", "content": "hello"} ] }); - let context = MessagesTransformContext::with_lookup( + let context = MessagesTransformContext::new( MessagesModelCapabilities { supports_reasoning: true, supports_adaptive_thinking: true, diff --git a/litellm-rust/crates/llms/src/base_llm/messages/context.rs b/litellm-rust/crates/llms/src/base_llm/messages/context.rs index e59a63813b0..17a0db26554 100644 --- a/litellm-rust/crates/llms/src/base_llm/messages/context.rs +++ b/litellm-rust/crates/llms/src/base_llm/messages/context.rs @@ -1,4 +1,4 @@ -use litellm_core_utils::settings::{Lookup, ProcessEnvironment}; +use litellm_core_utils::settings::Lookup; use serde::{Deserialize, Serialize}; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -89,10 +89,11 @@ impl Default for ThinkingBudgets { } impl ThinkingBudgets { - pub fn from_lookup(env: &impl Lookup) -> Self { + pub fn from_lookup(env: &(impl Lookup + ?Sized)) -> Self { let defaults = Self::default(); let tier = |name: &str, default: u64| { - env.parsed::(&format!("DEFAULT_REASONING_EFFORT_{name}_THINKING_BUDGET")) + env.get(&format!("DEFAULT_REASONING_EFFORT_{name}_THINKING_BUDGET")) + .and_then(|value| value.trim().parse().ok()) .unwrap_or(default) }; Self { @@ -117,14 +118,10 @@ pub struct MessagesTransformContext { pub drop_params: bool, } impl MessagesTransformContext { - pub fn new(capabilities: MessagesModelCapabilities, drop_params: bool) -> Self { - Self::with_lookup(capabilities, drop_params, &ProcessEnvironment) - } - - pub fn with_lookup( + pub fn new( capabilities: MessagesModelCapabilities, drop_params: bool, - env: &impl Lookup, + env: &(impl Lookup + ?Sized), ) -> Self { Self { thinking: ThinkingContext { diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/handler.rs b/litellm-rust/crates/llms/src/base_llm/ocr/handler.rs index 2cf955ff0c6..7e2ef154144 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/handler.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/handler.rs @@ -91,6 +91,7 @@ impl OcrClient { secrets: Arc::new( litellm_secrets::source::EnvironmentSecrets::python_compatible( provider_http.clone(), + Arc::new(|_: &str| None), ), ), provider_http, diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/settings.rs b/litellm-rust/crates/llms/src/base_llm/ocr/settings.rs index af4037ddc0b..685ddecac87 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/settings.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/settings.rs @@ -1,6 +1,7 @@ use std::time::Duration; -use litellm_core_utils::settings::Lookup; +use litellm_core_utils::coercion; +use litellm_layered::{Invalid, Layered}; #[derive(Clone, Debug, PartialEq)] pub struct OcrSettings { @@ -25,70 +26,88 @@ impl Default for OcrSettings { } } -impl OcrSettings { - pub fn from_environment(env: &impl Lookup) -> Self { +#[derive(Layered)] +pub struct OcrKeys { + #[layer(env = "REQUEST_TIMEOUT", read = timeout_seconds)] + pub request_timeout: Duration, + #[layer(env = "MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", read = download_megabytes)] + pub max_download_bytes: u64, + #[layer(env = "AZURE_OPERATION_POLLING_TIMEOUT", read = polling_seconds)] + pub poll_timeout: Duration, + #[layer(env = "AZURE_DOCUMENT_INTELLIGENCE_API_VERSION")] + pub document_intelligence_api_version: String, + #[layer(env = "AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI")] + pub document_intelligence_dpi: i64, + pub enable_azure_ad_token_refresh: bool, +} + +impl From for OcrSettings { + fn from(merged: OcrKeysPartial) -> Self { let defaults = Self::default(); Self { - request_timeout: env - .parsed::("REQUEST_TIMEOUT") - .and_then(|seconds| Duration::try_from_secs_f64(seconds).ok()) - .unwrap_or(defaults.request_timeout), - max_download_bytes: env - .parsed::("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB") - .filter(|size| size.is_finite()) - .map_or(defaults.max_download_bytes, megabytes), - poll_timeout: env - .parsed::("AZURE_OPERATION_POLLING_TIMEOUT") - .map_or(defaults.poll_timeout, |seconds| { - Duration::from_secs(seconds.max(0).unsigned_abs()) - }), - document_intelligence_api_version: env - .get("AZURE_DOCUMENT_INTELLIGENCE_API_VERSION") + request_timeout: merged.request_timeout.unwrap_or(defaults.request_timeout), + max_download_bytes: merged + .max_download_bytes + .unwrap_or(defaults.max_download_bytes), + poll_timeout: merged.poll_timeout.unwrap_or(defaults.poll_timeout), + document_intelligence_api_version: merged + .document_intelligence_api_version .unwrap_or(defaults.document_intelligence_api_version), - document_intelligence_dpi: env - .parsed("AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI") + document_intelligence_dpi: merged + .document_intelligence_dpi .unwrap_or(defaults.document_intelligence_dpi), - ..defaults + enable_azure_ad_token_refresh: merged + .enable_azure_ad_token_refresh + .unwrap_or(defaults.enable_azure_ad_token_refresh), } } } +fn timeout_seconds(value: &str) -> Result, Invalid> { + Ok(coercion::parsed::(value)? + .and_then(|seconds| Duration::try_from_secs_f64(seconds).ok())) +} + +fn download_megabytes(value: &str) -> Result, Invalid> { + Ok(coercion::parsed::(value)? + .filter(|size| size.is_finite()) + .map(megabytes)) +} + +fn polling_seconds(value: &str) -> Result, Invalid> { + Ok(coercion::parsed::(value)? + .map(|seconds| Duration::from_secs(seconds.max(0).unsigned_abs()))) +} + fn megabytes(size: f64) -> u64 { (size * 1024.0 * 1024.0) as u64 } #[cfg(test)] mod tests { + use litellm_layered::Env; use rstest::rstest; use super::*; - fn env_of(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option { - move |name| { - values - .iter() - .find(|(key, _)| *key == name) - .map(|(_, value)| value.to_string()) - } + fn from_env(values: &[(&str, &str)]) -> OcrSettings { + OcrSettings::from(OcrKeysPartial::from_env(&Env::of(values)).unwrap()) } #[test] fn an_empty_environment_keeps_the_python_defaults() { - assert_eq!( - OcrSettings::from_environment(&env_of(&[])), - OcrSettings::default() - ); + assert_eq!(from_env(&[]), OcrSettings::default()); } #[test] fn every_setting_follows_its_environment_variable() { - let settings = OcrSettings::from_environment(&env_of(&[ + let settings = from_env(&[ ("REQUEST_TIMEOUT", "30.5"), ("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", "0.5"), ("AZURE_OPERATION_POLLING_TIMEOUT", " 600 "), ("AZURE_DOCUMENT_INTELLIGENCE_API_VERSION", "2025-01-01"), ("AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI", "72"), - ])); + ]); assert_eq!( settings, OcrSettings { @@ -111,30 +130,25 @@ mod tests { #[case] value: &'static str, #[case] bytes: u64, ) { - let env = - move |name: &str| (name == "MAX_IMAGE_URL_DOWNLOAD_SIZE_MB").then(|| value.to_string()); assert_eq!( - OcrSettings::from_environment(&env).max_download_bytes, + from_env(&[("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", value)]).max_download_bytes, bytes ); } #[test] fn a_negative_polling_timeout_expires_immediately() { - let env = - |name: &str| (name == "AZURE_OPERATION_POLLING_TIMEOUT").then(|| "-5".to_string()); assert_eq!( - OcrSettings::from_environment(&env).poll_timeout, + from_env(&[("AZURE_OPERATION_POLLING_TIMEOUT", "-5")]).poll_timeout, Duration::ZERO ); } #[test] - fn an_empty_api_version_is_sent_as_is_like_python_str_of_getenv() { - let env = - |name: &str| (name == "AZURE_DOCUMENT_INTELLIGENCE_API_VERSION").then(String::new); + fn an_empty_api_version_shadows_the_default() { assert_eq!( - OcrSettings::from_environment(&env).document_intelligence_api_version, + from_env(&[("AZURE_DOCUMENT_INTELLIGENCE_API_VERSION", "")]) + .document_intelligence_api_version, "" ); } diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs b/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs index 2da0e1abbfc..eafdafe1e0a 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs @@ -2,7 +2,7 @@ use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrDocument, OcrRespo use std::{collections::BTreeMap, future::Future, sync::Arc, time::Duration}; use litellm_auth::{InputSource, SecretValue, Sourced, TokenProviderHandle}; -use litellm_core_utils::{call_arguments::CallArguments, settings::ProcessEnvironment}; +use litellm_core_utils::call_arguments::CallArguments; use litellm_http::outbound::{OutboundRequest, RequestSigner}; use litellm_secrets::source::Secrets; use serde::{ @@ -147,7 +147,7 @@ impl Default for OcrConnection { ResolvedOcrCredentials::default(), OcrTransportConfig::default(), OcrSettings::default(), - Arc::new(ProcessEnvironment), + Arc::new(|_: &str| None), ) } } @@ -455,7 +455,7 @@ mod tests { ..OcrTransportConfig::default() }, settings.clone(), - Arc::new(ProcessEnvironment), + Arc::new(|_: &str| None), ) .timeout }; diff --git a/litellm-rust/crates/python-bridge/build.rs b/litellm-rust/crates/python-bridge/build.rs index 0f7293007b2..f3b6b39f14c 100644 --- a/litellm-rust/crates/python-bridge/build.rs +++ b/litellm-rust/crates/python-bridge/build.rs @@ -1,3 +1,4 @@ +#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment fn main() { if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos") { println!("cargo:rustc-cdylib-link-arg=-undefined"); diff --git a/litellm-rust/crates/python-bridge/src/cache/native/v2.rs b/litellm-rust/crates/python-bridge/src/cache/native/v2.rs index eb09b8131c3..5febe1792bf 100644 --- a/litellm-rust/crates/python-bridge/src/cache/native/v2.rs +++ b/litellm-rust/crates/python-bridge/src/cache/native/v2.rs @@ -100,6 +100,7 @@ impl NativeCacheHandle { &RedisTopology::Standalone, Some(ttl), ResponseCacheCodec, + &crate::environment::ProcessEnvironment, ) }) .map_err(cache_error)? diff --git a/litellm-rust/crates/python-bridge/src/environment.rs b/litellm-rust/crates/python-bridge/src/environment.rs new file mode 100644 index 00000000000..455f9e3edd0 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/environment.rs @@ -0,0 +1,10 @@ +use litellm_core_utils::settings::Lookup; + +pub(crate) struct ProcessEnvironment; + +impl Lookup for ProcessEnvironment { + #[allow(clippy::disallowed_methods)] // the Python host's settings live in the process environment + fn get(&self, name: &str) -> Option { + std::env::var(name).ok() + } +} diff --git a/litellm-rust/crates/python-bridge/src/http.rs b/litellm-rust/crates/python-bridge/src/http.rs index de1099a55da..b56b273c77b 100644 --- a/litellm-rust/crates/python-bridge/src/http.rs +++ b/litellm-rust/crates/python-bridge/src/http.rs @@ -4,7 +4,7 @@ use std::{ sync::{Arc, LazyLock, Mutex, PoisonError}, }; -use litellm_core_utils::settings::ProcessEnvironment; +use crate::environment::ProcessEnvironment; use litellm_http::{ Client, ClientVariant, HttpClientConfig, HttpClientPool, HttpKeysPartial, HttpSettings, Resolution, SslVerify, TlsSource, Unsupported, diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 12827b76ff3..268415df132 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -3,6 +3,7 @@ mod callable; mod coercion; mod credentials; mod diagnostics; +mod environment; mod errors; mod execution; mod http; diff --git a/litellm-rust/crates/python-bridge/src/logger/tests.rs b/litellm-rust/crates/python-bridge/src/logger/tests.rs index 1fca3be720e..dd54cd07d03 100644 --- a/litellm-rust/crates/python-bridge/src/logger/tests.rs +++ b/litellm-rust/crates/python-bridge/src/logger/tests.rs @@ -1,3 +1,4 @@ +#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment use std::{process::Command, task::Poll}; use litellm_host::{ diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs index c788760b268..d9cfdcb3230 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs @@ -18,6 +18,7 @@ fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyR .map_err(crate::http::client_error)?, crate::http::resources().auth.clone(), crate::secrets::source(py)?, + std::sync::Arc::new(crate::environment::ProcessEnvironment), ); Ok(litellm_host::call::hosted_call( request, diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs index ec7f2ce79d9..b80cc64810d 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs @@ -3,13 +3,14 @@ mod errors; mod host; mod project; +use crate::environment::ProcessEnvironment; use host::OcrPythonHost; use litellm_callbacks_legacy_python::LoggingOperation; -use litellm_core_utils::settings::ProcessEnvironment; use litellm_host_python::to_py; use litellm_inference_ocr::provider_config; -use litellm_llms::base_llm::ocr::settings::OcrSettings; -use pyo3::prelude::*; +use litellm_layered::merge; +use litellm_llms::base_llm::ocr::settings::{OcrKeysPartial, OcrSettings}; +use pyo3::{exceptions::PyValueError, prelude::*}; use super::NativeCall; @@ -55,9 +56,21 @@ fn ocr_settings(py: Python<'_>) -> PyResult { } fn project_provider_defaults(snapshot: &Snapshot<'_>) -> PyResult { - Ok(OcrSettings { - enable_azure_ad_token_refresh: snapshot.read(&ENABLE_AZURE_AD_TOKEN_REFRESH)?, - ..OcrSettings::from_environment(&ProcessEnvironment) + Ok(OcrSettings::from(merge([ + OcrKeysPartial::from_env(&ProcessEnvironment) + .map_err(|error| PyValueError::new_err(error.to_string()))?, + globals(snapshot)?, + ]))) +} + +fn globals(snapshot: &Snapshot<'_>) -> PyResult { + Ok(OcrKeysPartial { + request_timeout: None, + max_download_bytes: None, + poll_timeout: None, + document_intelligence_api_version: None, + document_intelligence_dpi: None, + enable_azure_ad_token_refresh: Some(snapshot.read(&ENABLE_AZURE_AD_TOKEN_REFRESH)?), }) } diff --git a/litellm-rust/crates/python-bridge/src/secrets/mod.rs b/litellm-rust/crates/python-bridge/src/secrets/mod.rs index cddd47d08db..38b723f7cff 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/mod.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/mod.rs @@ -29,6 +29,7 @@ pub(crate) fn source(py: Python<'_>) -> PyResult> { let Some(snapshot) = PythonSettings::SecretManager.read_or_unset(py)? else { return Ok(Arc::new(EnvironmentSecrets::python_compatible( crate::http::host_client(py, litellm_http::ClientVariant::Provider)?, + Arc::new(crate::environment::ProcessEnvironment), ))); }; if snapshot.read(&NATIVE)? { diff --git a/litellm-rust/crates/python-bridge/src/secrets/resolved.rs b/litellm-rust/crates/python-bridge/src/secrets/resolved.rs index 5a606ab1039..b520dd19965 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/resolved.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/resolved.rs @@ -1,7 +1,7 @@ use std::sync::Arc; +use crate::environment::ProcessEnvironment; use futures_util::future::BoxFuture; -use litellm_core_utils::settings::ProcessEnvironment; use litellm_host_python::PythonContext; use litellm_http::Client; use litellm_secrets::source::SecretSource; diff --git a/litellm-rust/crates/python-bridge/src/secrets/runtime.rs b/litellm-rust/crates/python-bridge/src/secrets/runtime.rs index 72ba91ded16..04f365730de 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/runtime.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/runtime.rs @@ -1,6 +1,8 @@ use std::{collections::BTreeMap, sync::Arc}; -use litellm_core_utils::settings::{Lookup, ProcessEnvironment}; +use litellm_core_utils::settings::Lookup; + +use crate::environment::ProcessEnvironment; use litellm_host_python::{from_py, json_object_field, to_py}; use litellm_secrets::{ KeyManagementSettings, KeyManagementSystem, Secret, SecretManager, load_native_manager, diff --git a/litellm-rust/crates/python-compat/tests/fixtures.rs b/litellm-rust/crates/python-compat/tests/fixtures.rs index 9c202fdc5f1..5c0660d9d66 100644 --- a/litellm-rust/crates/python-compat/tests/fixtures.rs +++ b/litellm-rust/crates/python-compat/tests/fixtures.rs @@ -1,3 +1,4 @@ +#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment //! Replays `generated/values.json`, which CPython wrote with `scripts/generate_fixtures.py`. use std::{collections::BTreeMap, fs::File, io::Write}; diff --git a/litellm-rust/crates/secrets-aws/tests/secret_manager/configuration.rs b/litellm-rust/crates/secrets-aws/tests/secret_manager/configuration.rs index 40b851fd678..4d437cc4ba0 100644 --- a/litellm-rust/crates/secrets-aws/tests/secret_manager/configuration.rs +++ b/litellm-rust/crates/secrets-aws/tests/secret_manager/configuration.rs @@ -1,3 +1,4 @@ +#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment use super::*; #[rstest] diff --git a/litellm-rust/crates/secrets-azure/tests/live.rs b/litellm-rust/crates/secrets-azure/tests/live.rs index 429cd3013f7..784b2e8a268 100644 --- a/litellm-rust/crates/secrets-azure/tests/live.rs +++ b/litellm-rust/crates/secrets-azure/tests/live.rs @@ -1,6 +1,6 @@ +#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment use std::sync::Arc; -use litellm_core_utils::settings::ProcessEnvironment; use litellm_secrets_azure::AzureKeyVault; use litellm_secrets_types::Secret; use rstest::rstest; @@ -9,7 +9,7 @@ use rstest::rstest; #[tokio::test] #[ignore] async fn reads_a_real_secret() { - let environment = Arc::new(ProcessEnvironment); + let environment = Arc::new(|name: &str| std::env::var(name).ok()); let manager = AzureKeyVault::new(litellm_http::Client::plain_for_test(), environment).unwrap(); let name = std::env::var("AZURE_KEY_VAULT_LIVE_SECRET_NAME").unwrap(); let secret = manager.get_secret(&name).await.unwrap().unwrap(); diff --git a/litellm-rust/crates/secrets-cyberark/tests/secret_manager/writes.rs b/litellm-rust/crates/secrets-cyberark/tests/secret_manager/writes.rs index 764a591bc71..bba17668a5d 100644 --- a/litellm-rust/crates/secrets-cyberark/tests/secret_manager/writes.rs +++ b/litellm-rust/crates/secrets-cyberark/tests/secret_manager/writes.rs @@ -1,3 +1,4 @@ +#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment use super::*; #[rstest] diff --git a/litellm-rust/crates/secrets-hashicorp/tests/secret_manager/writes.rs b/litellm-rust/crates/secrets-hashicorp/tests/secret_manager/writes.rs index e2468e90c20..420d52b1dcd 100644 --- a/litellm-rust/crates/secrets-hashicorp/tests/secret_manager/writes.rs +++ b/litellm-rust/crates/secrets-hashicorp/tests/secret_manager/writes.rs @@ -1,3 +1,4 @@ +#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment use super::*; #[rstest] @@ -282,7 +283,7 @@ async fn rotation_applies_timeout_to_each_request(token_values: Vec<(&str, &str) #[ignore] async fn live_vault_round_trip() { let environment: Arc = - Arc::new(litellm_core_utils::settings::ProcessEnvironment); + Arc::new(|name: &str| std::env::var(name).ok()); let manager: HashicorpVault = HashicorpVault::new(environment, true).unwrap(); let name: String = std::env::var("LITELLM_VAULT_LIVE_SECRET_NAME").unwrap(); let value: SecretValue = SecretValue::new("native-live-value"); diff --git a/litellm-rust/crates/secrets/src/source.rs b/litellm-rust/crates/secrets/src/source.rs index 3c86fd25e9a..efec5b89034 100644 --- a/litellm-rust/crates/secrets/src/source.rs +++ b/litellm-rust/crates/secrets/src/source.rs @@ -40,10 +40,10 @@ impl SecretSource for SecretResolver { pub struct EnvironmentSecrets(SecretResolver); impl EnvironmentSecrets { - pub fn python_compatible(client: litellm_http::Client) -> Self { + pub fn python_compatible(client: litellm_http::Client, env: Secrets) -> Self { Self(SecretResolver::new_python_compatible( Arc::new(crate::SecretManagerState::default()), - Arc::new(litellm_core_utils::settings::ProcessEnvironment), + env, crate::OidcResolver::new(client), )) } diff --git a/litellm-rust/crates/secrets/tests/source.rs b/litellm-rust/crates/secrets/tests/source.rs index 17210940a67..9176cd2cf6d 100644 --- a/litellm-rust/crates/secrets/tests/source.rs +++ b/litellm-rust/crates/secrets/tests/source.rs @@ -11,16 +11,17 @@ mod tests { #[tokio::test] async fn python_environment_values_are_absent_like_get_secret_str( #[case] name: &'static str, - #[case] value: &str, + #[case] value: &'static str, #[case] expected: Option<&str>, ) { - unsafe { std::env::set_var(name, value) }; - let secret = EnvironmentSecrets::python_compatible(litellm_http::Client::plain_for_test()) - .resolve(&[name]) - .await - .unwrap() - .get(name); - unsafe { std::env::remove_var(name) }; + let secret = EnvironmentSecrets::python_compatible( + litellm_http::Client::plain_for_test(), + std::sync::Arc::new(move |key: &str| (key == name).then(|| value.to_owned())), + ) + .resolve(&[name]) + .await + .unwrap() + .get(name); assert_eq!(secret.as_deref(), expected); } } diff --git a/litellm-rust/crates/spend-clickhouse/src/insert.rs b/litellm-rust/crates/spend-clickhouse/src/insert.rs index 495f0d8ecb2..583e8bf202a 100644 --- a/litellm-rust/crates/spend-clickhouse/src/insert.rs +++ b/litellm-rust/crates/spend-clickhouse/src/insert.rs @@ -15,6 +15,7 @@ use time::{OffsetDateTime, format_description::well_known::Rfc3339}; use crate::Error; use litellm_storage_clickhouse::Connection; +#[allow(clippy::disallowed_methods)] // the ClickHouse sink is not built from an injected Lookup yet fn max_insert_bytes() -> Result { let name = "CLICKHOUSE_TRACE_MAX_INSERT_BYTES"; match std::env::var(name) { diff --git a/litellm-rust/crates/spend-clickhouse/tests/storage.rs b/litellm-rust/crates/spend-clickhouse/tests/storage.rs index 2b3eb75c658..9a8e9ef5293 100644 --- a/litellm-rust/crates/spend-clickhouse/tests/storage.rs +++ b/litellm-rust/crates/spend-clickhouse/tests/storage.rs @@ -1,3 +1,4 @@ +#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment use std::{collections::BTreeMap, time::Duration}; use litellm_http::Client; diff --git a/litellm-rust/crates/storage-clickhouse/src/insert.rs b/litellm-rust/crates/storage-clickhouse/src/insert.rs index 1cdad1897c1..e721228f037 100644 --- a/litellm-rust/crates/storage-clickhouse/src/insert.rs +++ b/litellm-rust/crates/storage-clickhouse/src/insert.rs @@ -5,6 +5,7 @@ use litellm_http::Client; use crate::{Connection, Error, valid_identifier}; +#[allow(clippy::disallowed_methods)] // the ClickHouse sink is not built from an injected Lookup yet fn insert_timeout() -> Result { let name = "CLICKHOUSE_INSERT_TIMEOUT_SECONDS"; match std::env::var(name) { diff --git a/litellm-rust/crates/storage-clickhouse/tests/transport.rs b/litellm-rust/crates/storage-clickhouse/tests/transport.rs index f70fe52e1de..32fcbb1d1e0 100644 --- a/litellm-rust/crates/storage-clickhouse/tests/transport.rs +++ b/litellm-rust/crates/storage-clickhouse/tests/transport.rs @@ -1,3 +1,4 @@ +#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment use std::collections::BTreeMap; use litellm_http::Client; diff --git a/litellm-rust/crates/testkit/src/install/fetch.rs b/litellm-rust/crates/testkit/src/install/fetch.rs index 73008f7a0da..fc565a1dce8 100644 --- a/litellm-rust/crates/testkit/src/install/fetch.rs +++ b/litellm-rust/crates/testkit/src/install/fetch.rs @@ -19,6 +19,7 @@ impl HttpFetch { } } + #[allow(clippy::disallowed_methods)] // test tooling reads its own token pub fn from_env() -> Self { Self::new(std::env::var("GITHUB_TOKEN").ok()) } diff --git a/litellm-rust/crates/testkit/tests/live.rs b/litellm-rust/crates/testkit/tests/live.rs index b805596879a..7d1d2556df8 100644 --- a/litellm-rust/crates/testkit/tests/live.rs +++ b/litellm-rust/crates/testkit/tests/live.rs @@ -1,3 +1,4 @@ +#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment //! Drives the real agents through a real gateway. Run with `cargo test -p litellm-testkit --test live -- --ignored` //! after exporting `TESTKIT_GATEWAY_URL`, `TESTKIT_GATEWAY_KEY`, one `TESTKIT_MODEL_` per wire //! (`MESSAGES`, `RESPONSES`, `CHAT_COMPLETIONS`) and one `TESTKIT__VERSION` per agent