diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index c868d3e4b8a..00cfd8c6f28 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3743,12 +3743,14 @@ dependencies = [ "litellm-gateway-ui", "litellm-http", "litellm-inference", + "litellm-layered", "litellm-llms", "litellm-secrets", "litellm-tracing", "rstest", "serde", "serde_json", + "serde_yaml_ng", "tempfile", "thiserror 2.0.19", "tokio", @@ -3944,6 +3946,7 @@ dependencies = [ "http 1.4.2", "hyper-util", "litellm-core-utils", + "litellm-layered", "rcgen", "reqwest 0.12.28", "reqwest 0.13.5", @@ -4117,6 +4120,7 @@ dependencies = [ "litellm-auth", "litellm-http", "litellm-inference", + "litellm-layered", "litellm-secrets", "tokio", ] @@ -4141,6 +4145,27 @@ dependencies = [ "wiremock", ] +[[package]] +name = "litellm-layered" +version = "0.1.0" +dependencies = [ + "litellm-core-utils", + "litellm-layered-derive", + "rstest", + "serde", + "serde_json", + "thiserror 2.0.19", +] + +[[package]] +name = "litellm-layered-derive" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "litellm-llms" version = "0.1.0" @@ -4231,6 +4256,7 @@ dependencies = [ "litellm-inference-ocr", "litellm-inference-responses", "litellm-inference-transcription", + "litellm-layered", "litellm-llms", "litellm-llms-types", "litellm-router-types", diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 9e5e0a96b73..402e6e80b7d 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -50,6 +50,8 @@ litellm-http = { path = "crates/http" } litellm-llms = { path = "crates/llms" } litellm-llms-types = { path = "crates/llms-types" } litellm-core-utils = { path = "crates/core-utils" } +litellm-layered = { path = "crates/layered" } +litellm-layered-derive = { path = "crates/layered-derive" } litellm-db = { path = "crates/db" } litellm-db-testing = { path = "crates/db-testing" } litellm-cache = { path = "crates/cache" } diff --git a/litellm-rust/crates/core-utils/src/settings.rs b/litellm-rust/crates/core-utils/src/settings.rs index 6b472295508..b18a99be12b 100644 --- a/litellm-rust/crates/core-utils/src/settings.rs +++ b/litellm-rust/crates/core-utils/src/settings.rs @@ -53,17 +53,6 @@ pub fn resolve_non_empty( }) } -pub trait Layer: Default { - fn or(self, lower: Self) -> Self; -} - -pub fn merge(highest_precedence_first: impl IntoIterator) -> L { - highest_precedence_first - .into_iter() - .reduce(L::or) - .unwrap_or_default() -} - #[cfg(test)] mod tests { use super::*; @@ -114,49 +103,4 @@ mod tests { assert_eq!(env.parsed::("FRACTION"), Some(0.5)); assert_eq!(env.parsed::("ABSENT"), None); } - - #[derive(Debug, Default, PartialEq)] - struct Pair { - first: Option, - second: Option, - } - - impl Layer for Pair { - fn or(self, lower: Self) -> Self { - Self { - first: self.first.or(lower.first), - second: self.second.or(lower.second), - } - } - } - - #[test] - fn merge_takes_each_field_from_the_highest_layer_that_sets_it() { - let merged = merge([ - Pair { - first: Some(1), - second: None, - }, - Pair { - first: Some(2), - second: Some(2), - }, - Pair { - first: Some(3), - second: Some(3), - }, - ]); - assert_eq!( - merged, - Pair { - first: Some(1), - second: Some(2), - } - ); - } - - #[test] - fn merging_no_layers_yields_the_empty_layer() { - assert_eq!(merge(Vec::::new()), Pair::default()); - } } diff --git a/litellm-rust/crates/gateway/Cargo.toml b/litellm-rust/crates/gateway/Cargo.toml index 387bcd43231..32ca2893748 100644 --- a/litellm-rust/crates/gateway/Cargo.toml +++ b/litellm-rust/crates/gateway/Cargo.toml @@ -20,6 +20,7 @@ litellm-gateway-ui.workspace = true litellm-auth-types.workspace = true litellm-config.workspace = true litellm-http = { workspace = true, features = ["mcp"] } +litellm-layered.workspace = true litellm-llms.workspace = true litellm-secrets.workspace = true litellm-tracing.workspace = true @@ -33,6 +34,7 @@ tower-sessions-moka-store = "0.15.0" [dev-dependencies] base64.workspace = true rstest.workspace = true +serde_yaml_ng = "0.10.0" tempfile.workspace = true tokio = { workspace = true, features = ["sync"] } tower = { version = "0.5", features = ["util"] } diff --git a/litellm-rust/crates/gateway/src/error.rs b/litellm-rust/crates/gateway/src/error.rs index 75dc04d179d..e76ec82495a 100644 --- a/litellm-rust/crates/gateway/src/error.rs +++ b/litellm-rust/crates/gateway/src/error.rs @@ -2,6 +2,8 @@ pub enum Error { #[error(transparent)] Http(#[from] litellm_http::Error), + #[error(transparent)] + Settings(#[from] litellm_layered::Error), #[error("environment_variables values must be strings, numbers or booleans")] Environment, #[error("MCP host does not yet support configured setting {0}")] diff --git a/litellm-rust/crates/gateway/src/http.rs b/litellm-rust/crates/gateway/src/http.rs new file mode 100644 index 00000000000..4d20121ad30 --- /dev/null +++ b/litellm-rust/crates/gateway/src/http.rs @@ -0,0 +1,88 @@ +use litellm_config::{LiteLlmSettings, Spelled}; +use litellm_http::{HttpKeysPartial, HttpSettings, SslVerify}; +use litellm_layered::{Lookup, merge}; + +pub(crate) fn http_settings( + env: &(impl Lookup + ?Sized), + settings: &LiteLlmSettings, +) -> Result { + Ok(HttpSettings::from(merge([ + HttpKeysPartial::from_env(env)?, + written(settings), + ]))) +} + +fn written(settings: &LiteLlmSettings) -> HttpKeysPartial { + HttpKeysPartial { + ssl_verify: settings.ssl_verify.as_ref().map(|value| match value { + Spelled::Value(true) => SslVerify::Enabled, + Spelled::Value(false) => SslVerify::Disabled, + Spelled::Text(value) => SslVerify::parse(value), + }), + ssl_cert_file: None, + ssl_certificate: settings.ssl_certificate.as_ref().map(Into::into), + ssl_security_level: settings.ssl_security_level.clone(), + ssl_ecdh_curve: settings.ssl_ecdh_curve.clone(), + force_ipv4: settings.force_ipv4, + http2: settings.http2, + aiohttp_trust_env: settings.aiohttp_trust_env, + disable_aiohttp_trust_env: settings.disable_aiohttp_trust_env, + disable_aiohttp_transport: settings.disable_aiohttp_transport, + user_agent: None, + tcp_keepalive: None, + pool_idle_timeout: None, + proxies: None, + } +} + +#[cfg(test)] +mod tests { + use litellm_layered::Env; + use rstest::rstest; + + use super::*; + + fn file(yaml: &str) -> LiteLlmSettings { + serde_yaml_ng::from_str(yaml).unwrap() + } + + #[rstest] + #[case::env_beats_file(&[("SSL_VERIFY", "false")], "ssl_verify: true", Some(SslVerify::Disabled))] + #[case::file_fills_unset_env(&[], "ssl_verify: false", Some(SslVerify::Disabled))] + #[case::neither(&[], "{}", None)] + fn env_is_stacked_above_the_config_file( + #[case] env: &[(&str, &str)], + #[case] yaml: &str, + #[case] expected: Option, + ) { + let settings = http_settings(&Env::of(env), &file(yaml)).unwrap(); + assert_eq!(settings.ssl_verify, expected); + } + + #[rstest] + #[case::bool_false("false", "false", SslVerify::Disabled)] + #[case::text_false("'False'", "False", SslVerify::Disabled)] + #[case::text_true("'true'", "true", SslVerify::Enabled)] + #[case::bundle("/etc/ca.pem", "/etc/ca.pem", SslVerify::CaBundle("/etc/ca.pem".into()))] + fn ssl_verify_spellings_read_the_same_from_the_file_and_the_environment( + #[case] yaml: &str, + #[case] env: &str, + #[case] expected: SslVerify, + ) { + let from_file = + http_settings(&Env::default(), &file(&format!("ssl_verify: {yaml}"))).unwrap(); + let from_env = http_settings(&Env::of(&[("SSL_VERIFY", env)]), &file("{}")).unwrap(); + assert_eq!(from_file.ssl_verify, Some(expected.clone())); + assert_eq!(from_env.ssl_verify, Some(expected)); + } + + #[rstest] + fn flags_are_only_turned_on_by_the_environment() { + let settings = http_settings( + &Env::of(&[("LITELLM_HTTP2", "false")]), + &file("http2: true"), + ) + .unwrap(); + assert!(settings.http2); + } +} diff --git a/litellm-rust/crates/gateway/src/lib.rs b/litellm-rust/crates/gateway/src/lib.rs index 0b3aba81181..050c53ac574 100644 --- a/litellm-rust/crates/gateway/src/lib.rs +++ b/litellm-rust/crates/gateway/src/lib.rs @@ -1,5 +1,6 @@ mod auth; mod error; +mod http; mod secrets; pub use error::Error; @@ -17,10 +18,7 @@ use http_body_util::BodyExt; use litellm_config::Config; use litellm_gateway_auth::Auth; use litellm_gateway_inference::{Gateway, ModelRouter}; -use litellm_http::{ - ClientVariant, HttpClientPool, HttpSettings, HttpSettingsLayer, Resolution, SslVerify, - media::PublicDnsResolver, -}; +use litellm_http::{ClientVariant, HttpClientPool, Resolution, media::PublicDnsResolver}; use litellm_inference::resources::CoreResources; use litellm_secrets::source::EnvironmentSecrets; use litellm_tracing::ByteChunk; @@ -35,26 +33,7 @@ pub fn build_inference(config: &Config) -> Result, Error> { .map(|value| value.expose().to_owned()) .or_else(|| std::env::var(name).ok()) }; - let settings = &config.litellm_settings; - let http_settings = HttpSettings::from_layers([ - HttpSettingsLayer::from_environment(&lookup), - HttpSettingsLayer { - ssl_verify: settings.ssl_verify.as_ref().map(|value| match value { - litellm_config::Spelled::Value(true) => SslVerify::Enabled, - litellm_config::Spelled::Value(false) => SslVerify::Disabled, - litellm_config::Spelled::Text(value) => SslVerify::parse(value), - }), - ssl_certificate: settings.ssl_certificate.as_ref().map(Into::into), - ssl_security_level: settings.ssl_security_level.clone(), - ssl_ecdh_curve: settings.ssl_ecdh_curve.clone(), - force_ipv4: settings.force_ipv4, - http2: settings.http2, - aiohttp_trust_env: settings.aiohttp_trust_env, - disable_aiohttp_trust_env: settings.disable_aiohttp_trust_env, - disable_aiohttp_transport: settings.disable_aiohttp_transport, - ..Default::default() - }, - ]); + let http_settings = http::http_settings(&lookup, &config.litellm_settings)?; let http = Resolution::from(&http_settings).config; let client = pool.client(&http, ClientVariant::Provider)?; let secrets = Arc::new(secrets::ConfigSecrets::new( diff --git a/litellm-rust/crates/http/Cargo.toml b/litellm-rust/crates/http/Cargo.toml index 7d64467f2f7..93b7721260d 100644 --- a/litellm-rust/crates/http/Cargo.toml +++ b/litellm-rust/crates/http/Cargo.toml @@ -14,6 +14,7 @@ rmcp = { version = "=3.4.1", default-features = false, features = ["client", "tr reqwest-mcp = { package = "reqwest", version = "0.13.2", default-features = false, features = ["rustls-no-provider", "json", "stream", "http2"], optional = true } http.workspace = true litellm-core-utils.workspace = true +litellm-layered.workspace = true hyper-util.workspace = true reqwest.workspace = true rustls.workspace = true diff --git a/litellm-rust/crates/http/src/lib.rs b/litellm-rust/crates/http/src/lib.rs index d53da607d39..5549082b961 100644 --- a/litellm-rust/crates/http/src/lib.rs +++ b/litellm-rust/crates/http/src/lib.rs @@ -24,5 +24,5 @@ pub use config::{ClientIdentity, HttpClientConfig, Resolution, Verify}; pub use error::{Error, TlsSource}; pub use pool::{ClientVariant, HttpClientPool}; pub use proxy::EnvironmentProxies; -pub use settings::{HttpSettings, HttpSettingsLayer, SslVerify, TcpKeepalive}; +pub use settings::{HttpKeys, HttpKeysPartial, HttpSettings, SslVerify, TcpKeepalive}; pub use tls::{KeyExchangeGroup, Tls12CipherSuite, Unsupported}; diff --git a/litellm-rust/crates/http/src/proxy.rs b/litellm-rust/crates/http/src/proxy.rs index e7e54ed6ae2..7246f715cce 100644 --- a/litellm-rust/crates/http/src/proxy.rs +++ b/litellm-rust/crates/http/src/proxy.rs @@ -1,5 +1,5 @@ use hyper_util::client::proxy::matcher::Matcher; -use litellm_core_utils::settings::Lookup; +use litellm_layered::Lookup; use veil::Redact; #[derive(Clone, Redact, Default, PartialEq, Eq, Hash)] @@ -14,14 +14,22 @@ pub struct EnvironmentProxies { } impl EnvironmentProxies { - pub fn from_environment(env: &impl Lookup) -> Self { + pub fn from_environment(env: &(impl Lookup + ?Sized)) -> Self { Self::resolve(env, cfg!(windows)) } - fn resolve(env: &impl Lookup, names_ignore_case: bool) -> Self { + pub fn from_env(env: &(impl Lookup + ?Sized)) -> Result, litellm_layered::Error> { + Ok(Some(Self::from_environment(env)).filter(|proxies| *proxies != Self::default())) + } + + fn resolve(env: &(impl Lookup + ?Sized), names_ignore_case: bool) -> Self { let lowercase_first = |upper: Option<&str>, lower: &str| { env.get(lower) - .or_else(|| upper.and_then(|name| env.truthy(name))) + .or_else(|| { + upper + .and_then(|name| env.get(name)) + .filter(|value| !value.is_empty()) + }) .unwrap_or_default() }; let is_cgi = env.get("REQUEST_METHOD").is_some(); @@ -80,19 +88,11 @@ impl EnvironmentProxies { #[cfg(test)] mod tests { + use litellm_layered::Env; use rstest::rstest; use super::*; - fn env_of(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option { - move |name| { - values - .iter() - .find(|(key, _)| *key == name) - .map(|(_, value)| value.to_string()) - } - } - fn url(value: &str) -> reqwest::Url { reqwest::Url::parse(value).unwrap() } @@ -108,7 +108,7 @@ mod tests { #[case] target: &str, #[case] expected: bool, ) { - let proxies = EnvironmentProxies::from_environment(&env_of(env)); + let proxies = EnvironmentProxies::from_environment(&Env::of(env)); assert_eq!(proxies.matcher()(&url(target)), expected); } @@ -126,8 +126,8 @@ mod tests { #[case] equivalent: &'static [(&'static str, &'static str)], ) { assert_eq!( - EnvironmentProxies::from_environment(&env_of(env)), - EnvironmentProxies::from_environment(&env_of(equivalent)) + EnvironmentProxies::from_environment(&Env::of(env)), + EnvironmentProxies::from_environment(&Env::of(equivalent)) ); } @@ -149,7 +149,7 @@ mod tests { #[test] fn a_cgi_request_still_proxies_https_through_the_configured_proxy() { - let proxies = EnvironmentProxies::from_environment(&env_of(&[ + let proxies = EnvironmentProxies::from_environment(&Env::of(&[ ("REQUEST_METHOD", "GET"), ("HTTPS_PROXY", "http://proxy:3128"), ])); @@ -158,7 +158,7 @@ mod tests { #[test] fn debug_output_hides_proxy_credentials_but_shows_which_variables_are_set() { - let proxies = EnvironmentProxies::from_environment(&env_of(&[ + let proxies = EnvironmentProxies::from_environment(&Env::of(&[ ("HTTPS_PROXY", "http://operator:hunter2@proxy.corp:3128"), ("NO_PROXY", "internal.test"), ])); @@ -170,7 +170,7 @@ mod tests { #[test] fn an_empty_environment_proxies_nothing() { - let proxies = EnvironmentProxies::from_environment(&env_of(&[])); + let proxies = EnvironmentProxies::from_environment(&Env::of(&[])); assert_eq!(proxies, EnvironmentProxies::default()); assert!(proxies.reqwest_proxies().is_empty()); } diff --git a/litellm-rust/crates/http/src/settings.rs b/litellm-rust/crates/http/src/settings.rs index 1f437df225c..2f0edfad270 100644 --- a/litellm-rust/crates/http/src/settings.rs +++ b/litellm-rust/crates/http/src/settings.rs @@ -3,10 +3,8 @@ use std::{ time::Duration, }; -use litellm_core_utils::{ - serde_compat::parse_str_bool, - settings::{Layer, Lookup, merge}, -}; +use litellm_core_utils::serde_compat::parse_str_bool; +use litellm_layered::{Invalid, Layered, Lookup, read, read_env}; use crate::proxy::EnvironmentProxies; @@ -25,6 +23,10 @@ impl SslVerify { _ => Self::CaBundle(PathBuf::from(value)), } } + + pub fn parse_env(value: &str) -> Result, Invalid> { + Ok(Some(Self::parse(value))) + } } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] @@ -34,78 +36,62 @@ pub struct TcpKeepalive { pub retries: u32, } -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct HttpSettingsLayer { - pub ssl_verify: Option, - pub ssl_cert_file: Option, - pub ssl_certificate: Option, - pub ssl_security_level: Option, - pub ssl_ecdh_curve: Option, - pub force_ipv4: Option, - pub http2: Option, - pub aiohttp_trust_env: Option, - pub disable_aiohttp_trust_env: Option, - pub disable_aiohttp_transport: Option, - pub user_agent: Option, - pub tcp_keepalive: Option, - pub pool_idle_timeout: Option, - pub proxies: Option, -} - -impl HttpSettingsLayer { - pub fn from_environment(env: &impl Lookup) -> Self { - let seconds = |name: &str, default: u32| { - Duration::from_secs(u64::from(env.parsed::(name).unwrap_or(default))) +impl TcpKeepalive { + pub fn from_env(env: &(impl Lookup + ?Sized)) -> Result, litellm_layered::Error> { + if read_env(env, &["AIOHTTP_SO_KEEPALIVE"], false, read::enabled)?.is_none() { + return Ok(None); + } + let seconds = |names: &'static [&'static str], default: u64| { + Ok::<_, litellm_layered::Error>( + read_env(env, names, false, read::seconds)?.unwrap_or(Duration::from_secs(default)), + ) }; - Self { - ssl_verify: env.get("SSL_VERIFY").map(|value| SslVerify::parse(&value)), - ssl_cert_file: env.truthy("SSL_CERT_FILE").map(PathBuf::from), - ssl_certificate: env.get("SSL_CERTIFICATE").map(PathBuf::from), - ssl_security_level: env.get("SSL_SECURITY_LEVEL"), - ssl_ecdh_curve: env.get("SSL_ECDH_CURVE"), - force_ipv4: None, - http2: env.enabled("LITELLM_HTTP2"), - aiohttp_trust_env: env.enabled("AIOHTTP_TRUST_ENV"), - disable_aiohttp_trust_env: env.enabled("DISABLE_AIOHTTP_TRUST_ENV"), - disable_aiohttp_transport: env.enabled("DISABLE_AIOHTTP_TRANSPORT"), - user_agent: env.get("LITELLM_USER_AGENT"), - tcp_keepalive: env.enabled("AIOHTTP_SO_KEEPALIVE").map(|_| TcpKeepalive { - idle: seconds("AIOHTTP_TCP_KEEPIDLE", 60), - interval: seconds("AIOHTTP_TCP_KEEPINTVL", 30), - retries: env.parsed("AIOHTTP_TCP_KEEPCNT").unwrap_or(5), - }), - pool_idle_timeout: env - .parsed::("AIOHTTP_KEEPALIVE_TIMEOUT") - .map(|timeout| Duration::from_secs(u64::from(timeout))), - proxies: Some(EnvironmentProxies::from_environment(env)) - .filter(|proxies| *proxies != EnvironmentProxies::default()), - } + Ok(Some(Self { + idle: seconds(&["AIOHTTP_TCP_KEEPIDLE"], 60)?, + interval: seconds(&["AIOHTTP_TCP_KEEPINTVL"], 30)?, + retries: read_env(env, &["AIOHTTP_TCP_KEEPCNT"], false, read::parsed)?.unwrap_or(5), + })) } } -impl Layer for HttpSettingsLayer { - fn or(self, lower: Self) -> Self { - Self { - ssl_verify: self.ssl_verify.or(lower.ssl_verify), - ssl_cert_file: self.ssl_cert_file.or(lower.ssl_cert_file), - ssl_certificate: self.ssl_certificate.or(lower.ssl_certificate), - ssl_security_level: self.ssl_security_level.or(lower.ssl_security_level), - ssl_ecdh_curve: self.ssl_ecdh_curve.or(lower.ssl_ecdh_curve), - force_ipv4: self.force_ipv4.or(lower.force_ipv4), - http2: self.http2.or(lower.http2), - aiohttp_trust_env: self.aiohttp_trust_env.or(lower.aiohttp_trust_env), - disable_aiohttp_trust_env: self - .disable_aiohttp_trust_env - .or(lower.disable_aiohttp_trust_env), - disable_aiohttp_transport: self - .disable_aiohttp_transport - .or(lower.disable_aiohttp_transport), - user_agent: self.user_agent.or(lower.user_agent), - tcp_keepalive: self.tcp_keepalive.or(lower.tcp_keepalive), - pool_idle_timeout: self.pool_idle_timeout.or(lower.pool_idle_timeout), - proxies: self.proxies.or(lower.proxies), - } - } +#[derive(Layered)] +pub struct HttpKeys { + #[layer(env = "SSL_VERIFY", read = SslVerify::parse_env, litellm_setting = "ssl_verify")] + pub ssl_verify: SslVerify, + #[layer(env = "SSL_CERT_FILE", read = read::truthy)] + pub ssl_cert_file: PathBuf, + #[layer(env = "SSL_CERTIFICATE", read = read::text, litellm_setting = "ssl_certificate")] + pub ssl_certificate: PathBuf, + #[layer(env = "SSL_SECURITY_LEVEL", read = read::text, litellm_setting = "ssl_security_level")] + pub ssl_security_level: String, + #[layer(env = "SSL_ECDH_CURVE", read = read::text, litellm_setting = "ssl_ecdh_curve")] + pub ssl_ecdh_curve: String, + #[layer(litellm_setting = "force_ipv4")] + pub force_ipv4: bool, + #[layer(env = "LITELLM_HTTP2", read = read::enabled, litellm_setting = "http2")] + pub http2: bool, + #[layer(env = "AIOHTTP_TRUST_ENV", read = read::enabled, litellm_setting = "aiohttp_trust_env")] + pub aiohttp_trust_env: bool, + #[layer( + env = "DISABLE_AIOHTTP_TRUST_ENV", + read = read::enabled, + litellm_setting = "disable_aiohttp_trust_env" + )] + pub disable_aiohttp_trust_env: bool, + #[layer( + env = "DISABLE_AIOHTTP_TRANSPORT", + read = read::enabled, + litellm_setting = "disable_aiohttp_transport" + )] + pub disable_aiohttp_transport: bool, + #[layer(env = "LITELLM_USER_AGENT", read = read::text, keep_blank)] + pub user_agent: String, + #[layer(from_env = TcpKeepalive::from_env)] + pub tcp_keepalive: TcpKeepalive, + #[layer(env = "AIOHTTP_KEEPALIVE_TIMEOUT", read = read::seconds)] + pub pool_idle_timeout: Duration, + #[layer(from_env = EnvironmentProxies::from_env)] + pub proxies: EnvironmentProxies, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -145,11 +131,8 @@ impl Default for HttpSettings { } } -impl HttpSettings { - pub fn from_layers( - highest_precedence_first: impl IntoIterator, - ) -> Self { - let merged = merge(highest_precedence_first); +impl From for HttpSettings { + fn from(merged: HttpKeysPartial) -> Self { let defaults = Self::default(); let http2 = merged.http2.unwrap_or(defaults.http2); Self { @@ -173,7 +156,9 @@ impl HttpSettings { ..defaults } } +} +impl HttpSettings { pub fn without_missing_files(self, exists: &dyn Fn(&Path) -> bool) -> Self { Self { ssl_verify: match self.ssl_verify { @@ -190,19 +175,16 @@ impl HttpSettings { mod tests { use rstest::rstest; + use litellm_layered::{Env, merge}; + use super::*; - fn no_env(_: &str) -> Option { - None + fn stack(highest_first: impl IntoIterator) -> HttpSettings { + HttpSettings::from(merge(highest_first)) } - fn env_of(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option { - move |name| { - values - .iter() - .find(|(key, _)| *key == name) - .map(|(_, value)| value.to_string()) - } + fn from_env(values: &[(&str, &str)]) -> HttpKeysPartial { + HttpKeysPartial::from_env(&Env::of(values)).unwrap() } #[rstest] @@ -219,22 +201,22 @@ mod tests { #[test] fn higher_layers_override_lower_ones() { - let configured = HttpSettingsLayer { + let configured = HttpKeysPartial { ssl_verify: Some(SslVerify::Enabled), ssl_certificate: Some("/configured/client.pem".into()), ssl_security_level: Some("configured".into()), user_agent: Some("configured/1".into()), - ..HttpSettingsLayer::default() + ..HttpKeysPartial::default() }; - let environment = HttpSettingsLayer::from_environment(&env_of(&[ + let environment = from_env(&[ ("SSL_VERIFY", "false"), ("SSL_CERT_FILE", "/env/roots.pem"), ("SSL_CERTIFICATE", "/env/client.pem"), ("SSL_SECURITY_LEVEL", "DEFAULT@SECLEVEL=1"), ("SSL_ECDH_CURVE", "X25519"), ("LITELLM_USER_AGENT", "env/2"), - ])); - let settings = HttpSettings::from_layers([environment, configured]); + ]); + let settings = stack([environment, configured]); assert_eq!(settings.ssl_verify, Some(SslVerify::Disabled)); assert_eq!(settings.ssl_cert_file, Some("/env/roots.pem".into())); assert_eq!(settings.ssl_certificate, Some("/env/client.pem".into())); @@ -248,35 +230,32 @@ mod tests { #[test] fn an_explicit_false_in_a_higher_layer_beats_a_lower_true() { - let higher = HttpSettingsLayer { + let higher = HttpKeysPartial { http2: Some(false), force_ipv4: Some(false), - ..HttpSettingsLayer::default() + ..HttpKeysPartial::default() }; - let lower = HttpSettingsLayer { + let lower = HttpKeysPartial { http2: Some(true), force_ipv4: Some(true), - ..HttpSettingsLayer::default() + ..HttpKeysPartial::default() }; - let settings = HttpSettings::from_layers([higher, lower]); + let settings = stack([higher, lower]); assert!(!settings.http2); assert!(!settings.force_ipv4); } #[test] fn an_empty_environment_is_an_empty_layer_so_lower_layers_and_defaults_apply() { - assert_eq!( - HttpSettingsLayer::from_environment(&no_env), - HttpSettingsLayer::default() - ); - let configured = HttpSettingsLayer { + assert_eq!(from_env(&[]), HttpKeysPartial::default()); + let configured = HttpKeysPartial { ssl_verify: Some(SslVerify::CaBundle("/configured/roots.pem".into())), http2: Some(true), user_agent: Some("configured/1".into()), - ..HttpSettingsLayer::default() + ..HttpKeysPartial::default() }; assert_eq!( - HttpSettings::from_layers([HttpSettingsLayer::default(), configured]), + stack([HttpKeysPartial::default(), configured]), HttpSettings { ssl_verify: Some(SslVerify::CaBundle("/configured/roots.pem".into())), http2: true, @@ -284,35 +263,48 @@ mod tests { ..HttpSettings::default() } ); - assert_eq!(HttpSettings::from_layers([]), HttpSettings::default()); + assert_eq!(stack([]), HttpSettings::default()); } #[test] - fn empty_certificate_is_retained_for_validation_while_empty_tuning_is_absent() { - let configured = HttpSettingsLayer { + fn blank_environment_values_fall_through_to_the_configured_settings() { + let configured = HttpKeysPartial { ssl_certificate: Some("/configured/client.pem".into()), ssl_security_level: Some("configured".into()), ssl_ecdh_curve: Some("X25519".into()), - ..HttpSettingsLayer::default() + ..HttpKeysPartial::default() }; - let environment = HttpSettingsLayer::from_environment(&env_of(&[ + let environment = from_env(&[ ("SSL_CERTIFICATE", ""), - ("SSL_SECURITY_LEVEL", ""), + ("SSL_SECURITY_LEVEL", " "), ("SSL_ECDH_CURVE", ""), - ])); - let settings = HttpSettings::from_layers([environment, configured]); - assert_eq!(settings.ssl_certificate, Some(PathBuf::new())); - assert_eq!(settings.ssl_security_level, None); - assert_eq!(settings.ssl_ecdh_curve, None); + ]); + let settings = stack([environment, configured]); + assert_eq!( + settings.ssl_certificate, + Some(PathBuf::from("/configured/client.pem")) + ); + assert_eq!(settings.ssl_security_level.as_deref(), Some("configured")); + assert_eq!(settings.ssl_ecdh_curve.as_deref(), Some("X25519")); + } + + #[test] + fn an_empty_user_agent_environment_value_beats_the_configured_one() { + let configured = HttpKeysPartial { + user_agent: Some("configured/1".into()), + ..HttpKeysPartial::default() + }; + let settings = stack([from_env(&[("LITELLM_USER_AGENT", "")]), configured]); + assert_eq!(settings.user_agent.as_deref(), Some("")); } #[test] fn socket_keepalive_follows_the_aiohttp_variables_with_python_defaults() { - let tuned = HttpSettings::from_layers([HttpSettingsLayer::from_environment(&env_of(&[ + let tuned = stack([from_env(&[ ("AIOHTTP_SO_KEEPALIVE", "True"), ("AIOHTTP_TCP_KEEPIDLE", "45"), ("AIOHTTP_KEEPALIVE_TIMEOUT", "30"), - ]))]); + ])]); assert_eq!( tuned.tcp_keepalive, Some(TcpKeepalive { @@ -326,9 +318,7 @@ mod tests { #[test] fn socket_keepalive_stays_off_unless_enabled() { - let settings = HttpSettings::from_layers([HttpSettingsLayer::from_environment(&env_of( - &[("AIOHTTP_TCP_KEEPIDLE", "45")], - ))]); + let settings = stack([from_env(&[("AIOHTTP_TCP_KEEPIDLE", "45")])]); assert_eq!(settings.tcp_keepalive, None); assert_eq!(settings.pool_idle_timeout, Duration::from_secs(15)); } @@ -338,13 +328,13 @@ mod tests { disable_aiohttp_trust_env: bool, disable_aiohttp_transport: bool, http2: bool, - ) -> HttpSettingsLayer { - HttpSettingsLayer { + ) -> HttpKeysPartial { + HttpKeysPartial { aiohttp_trust_env: Some(aiohttp_trust_env), disable_aiohttp_trust_env: Some(disable_aiohttp_trust_env), disable_aiohttp_transport: Some(disable_aiohttp_transport), http2: Some(http2), - ..HttpSettingsLayer::default() + ..HttpKeysPartial::default() } } @@ -355,22 +345,21 @@ mod tests { #[case::http2_uses_httpx(proxy_flags(false, true, false, true), true)] #[case::aiohttp_disabled(proxy_flags(false, true, true, false), true)] fn environment_proxies_apply_unless_the_aiohttp_transport_opts_out( - #[case] layer: HttpSettingsLayer, + #[case] layer: HttpKeysPartial, #[case] expected: bool, ) { - assert_eq!(HttpSettings::from_layers([layer]).trust_proxy_env, expected); + assert_eq!(stack([layer]).trust_proxy_env, expected); } #[test] fn a_proxy_opt_out_in_one_source_still_yields_to_trust_env_from_another() { - let environment = - HttpSettingsLayer::from_environment(&env_of(&[("DISABLE_AIOHTTP_TRUST_ENV", "true")])); - let configured = HttpSettingsLayer { + let environment = from_env(&[("DISABLE_AIOHTTP_TRUST_ENV", "true")]); + let configured = HttpKeysPartial { aiohttp_trust_env: Some(true), - ..HttpSettingsLayer::default() + ..HttpKeysPartial::default() }; - assert!(!HttpSettings::from_layers([environment.clone()]).trust_proxy_env); - assert!(HttpSettings::from_layers([environment, configured]).trust_proxy_env); + assert!(!stack([environment.clone()]).trust_proxy_env); + assert!(stack([environment, configured]).trust_proxy_env); } #[test] @@ -411,7 +400,7 @@ mod tests { | "DISABLE_AIOHTTP_TRUST_ENV" => Some(value.to_string()), _ => None, }; - let layer = HttpSettingsLayer::from_environment(&env); + let layer = HttpKeysPartial::from_env(&env).unwrap(); assert_eq!(layer.http2, expected); assert_eq!(layer.aiohttp_trust_env, expected); assert_eq!(layer.disable_aiohttp_transport, expected); diff --git a/litellm-rust/crates/inference-testing/Cargo.toml b/litellm-rust/crates/inference-testing/Cargo.toml index 748b1df0d3b..f3935895d82 100644 --- a/litellm-rust/crates/inference-testing/Cargo.toml +++ b/litellm-rust/crates/inference-testing/Cargo.toml @@ -10,6 +10,7 @@ publish = false litellm-auth.workspace = true futures-util.workspace = true litellm-http.workspace = true +litellm-layered.workspace = true litellm-inference.workspace = true litellm-secrets.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/inference-testing/src/live.rs b/litellm-rust/crates/inference-testing/src/live.rs index b63bfd91a38..986ece5d383 100644 --- a/litellm-rust/crates/inference-testing/src/live.rs +++ b/litellm-rust/crates/inference-testing/src/live.rs @@ -1,7 +1,8 @@ use std::{future::Future, sync::Arc, time::Duration}; -use litellm_http::{ClientVariant, HttpSettings, HttpSettingsLayer, Resolution}; +use litellm_http::{ClientVariant, HttpKeysPartial, HttpSettings, Resolution}; use litellm_inference::resources::CoreResources; +use litellm_layered::Env; use litellm_secrets::source::{EnvironmentSecrets, SecretSource}; pub const DEADLINE: Duration = Duration::from_secs(120); @@ -15,10 +16,10 @@ pub struct LiveResources { impl Default for LiveResources { fn default() -> Self { let resources = CoreResources::new(Arc::new(super::http_pool())); - let settings = - HttpSettings::from_layers([HttpSettingsLayer::from_environment(&|key: &str| { - std::env::var(key).ok() - })]); + let settings = HttpSettings::from( + HttpKeysPartial::from_env(&Env::from_process()) + .expect("live HTTP environment variables must be valid"), + ); let resolution = Resolution::from(&settings); let http = resources .pool diff --git a/litellm-rust/crates/layered-derive/Cargo.toml b/litellm-rust/crates/layered-derive/Cargo.toml new file mode 100644 index 00000000000..0639e798b4d --- /dev/null +++ b/litellm-rust/crates/layered-derive/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "litellm-layered-derive" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[lib] +proc-macro = true + +[dependencies] +proc-macro2.workspace = true +quote.workspace = true +syn = { workspace = true, features = ["derive", "full", "parsing", "printing", "proc-macro"] } diff --git a/litellm-rust/crates/layered-derive/src/lib.rs b/litellm-rust/crates/layered-derive/src/lib.rs new file mode 100644 index 00000000000..39f4f5c47b1 --- /dev/null +++ b/litellm-rust/crates/layered-derive/src/lib.rs @@ -0,0 +1,281 @@ +use proc_macro::TokenStream; +use proc_macro2::{Span, TokenStream as TokenStream2}; +use quote::{format_ident, quote}; +use syn::{ + Data, DeriveInput, Expr, ExprArray, Fields, Ident, Lit, LitStr, Path, Type, Visibility, + meta::ParseNestedMeta, parse_macro_input, +}; + +/// `litellm_setting = "x"`: `litellm.x` in Python and `litellm_settings.x` in config.yaml, which the proxy copies with `setattr`. +#[proc_macro_derive(Layered, attributes(layer))] +pub fn derive_layered(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + expand(&input) + .unwrap_or_else(syn::Error::into_compile_error) + .into() +} + +#[derive(Default)] +struct FieldAttributes { + env: Vec, + read: Option, + from_env: Option, + litellm_param: Vec, + deserialize_with: Option, + litellm_setting: Option, + secret: bool, + keep_blank: bool, + setting: Option, +} + +struct LayeredField { + vis: Visibility, + ident: Ident, + ty: Type, + attributes: FieldAttributes, +} + +fn expand(input: &DeriveInput) -> syn::Result { + let Data::Struct(data) = &input.data else { + return Err(syn::Error::new_spanned(input, "Layered needs a struct")); + }; + let Fields::Named(named) = &data.fields else { + return Err(syn::Error::new_spanned(input, "Layered needs named fields")); + }; + let fields = named + .named + .iter() + .map(|field| { + Ok(LayeredField { + vis: field.vis.clone(), + ident: field.ident.clone().expect("named field"), + ty: field.ty.clone(), + attributes: field_attributes(field)?, + }) + }) + .collect::>>()?; + + let name = &input.ident; + let vis = &input.vis; + let partial = format_ident!("{name}Partial"); + let layered = quote!(::litellm_layered); + let private = quote!(#layered::__private); + + let partial_fields = fields.iter().map(|field| { + let LayeredField { vis, ident, ty, .. } = field; + quote!(#vis #ident: ::core::option::Option<#ty>) + }); + let debug_fields = fields.iter().map(|field| { + let ident = &field.ident; + let label = ident.to_string(); + if field.attributes.secret { + quote!(.field(#label, &#private::Redacted(&self.#ident))) + } else { + quote!(.field(#label, &self.#ident)) + } + }); + let or_fields = fields.iter().map(|field| { + let ident = &field.ident; + quote!(#ident: self.#ident.or(lower.#ident)) + }); + let env_fields = fields.iter().map(|field| { + let ident = &field.ident; + let attributes = &field.attributes; + let keep_blank = attributes.keep_blank; + if let Some(from_env) = &attributes.from_env { + return quote!(#ident: #from_env(env)?); + } + if attributes.env.is_empty() { + return quote!(#ident: ::core::option::Option::None); + } + let names = &attributes.env; + let read = attributes + .read + .clone() + .map_or_else(|| quote!(#layered::read::parsed), |read| quote!(#read)); + quote!(#ident: #layered::read_env(env, &[#(#names),*], #keep_blank, #read)?) + }); + let param_fields = fields.iter().map(|field| { + let LayeredField { ident, ty, .. } = field; + let attributes = &field.attributes; + let keep_blank = attributes.keep_blank; + if attributes.litellm_param.is_empty() { + return quote!(#ident: ::core::option::Option::None); + } + let names = &attributes.litellm_param; + let decode = match &attributes.deserialize_with { + Some(path) => quote!(|value| #path(value).ok().flatten()), + None => quote!(|value| <#ty as #private::serde::Deserialize>::deserialize(value).ok()), + }; + quote!(#ident: #private::from_litellm_params(params, &[#(#names),*], #keep_blank, #decode)) + }); + + let sources = fields.iter().map(|field| { + let attributes = &field.attributes; + let setting = attributes.setting.clone().unwrap_or_else(|| { + LitStr::new( + &field.ident.to_string().replace('_', " "), + Span::call_site(), + ) + }); + let params = &attributes.litellm_param; + let env = &attributes.env; + let litellm_setting = match &attributes.litellm_setting { + Some(name) => quote!(::core::option::Option::Some(#name)), + None => quote!(::core::option::Option::None), + }; + quote!(#layered::Sources { + setting: #setting, + litellm_params: &[#(#params),*], + litellm_setting: #litellm_setting, + env: &[#(#env),*], + }) + }); + let source_count = fields.len(); + let source_refs = fields.iter().enumerate().map(|(index, field)| { + let constant = Ident::new(&field.ident.to_string().to_uppercase(), field.ident.span()); + quote!(pub const #constant: &'static #layered::Sources = &Self::SOURCES[#index];) + }); + let litellm_params = fields + .iter() + .flat_map(|field| field.attributes.litellm_param.iter()); + let litellm_settings = fields.iter().filter_map(|field| { + let setting = field.attributes.litellm_setting.as_ref()?; + let label = field.ident.to_string(); + Some(quote!((#setting, #label))) + }); + let env_names = unique( + fields + .iter() + .flat_map(|field| field.attributes.env.iter().cloned()), + ); + + Ok(quote! { + #[derive(::core::clone::Clone, ::core::default::Default, ::core::cmp::PartialEq)] + #vis struct #partial { + #(#partial_fields,)* + } + + impl ::core::fmt::Debug for #partial { + fn fmt(&self, formatter: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result { + formatter.debug_struct(stringify!(#partial)) + #(#debug_fields)* + .finish() + } + } + + impl #layered::Partial for #partial { + fn or(self, lower: Self) -> Self { + Self { #(#or_fields,)* } + } + } + + impl #partial { + pub fn from_env( + env: &(impl #layered::Lookup + ?::core::marker::Sized), + ) -> ::core::result::Result { + ::core::result::Result::Ok(Self { #(#env_fields,)* }) + } + + pub fn from_litellm_params( + params: &#private::serde_json::Map<::std::string::String, #private::serde_json::Value>, + ) -> Self { + Self { #(#param_fields,)* } + } + } + + impl #name { + pub const SOURCES: [#layered::Sources; #source_count] = [#(#sources),*]; + pub const LITELLM_PARAMS: &'static [&'static str] = &[#(#litellm_params),*]; + pub const LITELLM_SETTINGS: &'static [(&'static str, &'static str)] = &[#(#litellm_settings),*]; + pub const ENV_NAMES: &'static [&'static str] = &[#(#env_names),*]; + #(#source_refs)* + } + }) +} + +fn unique(names: impl Iterator) -> Vec { + names.fold(Vec::new(), |mut seen, name| { + if !seen + .iter() + .any(|known: &LitStr| known.value() == name.value()) + { + seen.push(name); + } + seen + }) +} + +fn field_attributes(field: &syn::Field) -> syn::Result { + let mut attributes = FieldAttributes::default(); + for attribute in field + .attrs + .iter() + .filter(|attribute| attribute.path().is_ident("layer")) + { + attribute.parse_nested_meta(|meta| { + let path = &meta.path; + if path.is_ident("env") { + attributes.env = names(&meta)?; + } else if path.is_ident("litellm_param") { + attributes.litellm_param = names(&meta)?; + } else if path.is_ident("read") { + attributes.read = Some(meta.value()?.parse()?); + } else if path.is_ident("from_env") { + attributes.from_env = Some(meta.value()?.parse()?); + } else if path.is_ident("deserialize_with") { + attributes.deserialize_with = Some(meta.value()?.parse()?); + } else if path.is_ident("litellm_setting") { + attributes.litellm_setting = Some(meta.value()?.parse()?); + } else if path.is_ident("setting") { + attributes.setting = Some(meta.value()?.parse()?); + } else if path.is_ident("secret") { + attributes.secret = true; + } else if path.is_ident("keep_blank") { + attributes.keep_blank = true; + } else { + return Err(meta.error("unknown `layer` attribute")); + } + Ok(()) + })?; + } + if attributes.from_env.is_some() && (!attributes.env.is_empty() || attributes.read.is_some()) { + return Err(syn::Error::new_spanned( + field, + "`from_env` reads the environment itself, so it cannot be combined with `env` or `read`", + )); + } + if attributes.read.is_some() && attributes.env.is_empty() { + return Err(syn::Error::new_spanned(field, "`read` needs `env`")); + } + if attributes.deserialize_with.is_some() && attributes.litellm_param.is_empty() { + return Err(syn::Error::new_spanned( + field, + "`deserialize_with` needs `litellm_param`", + )); + } + Ok(attributes) +} + +fn names(meta: &ParseNestedMeta<'_>) -> syn::Result> { + match meta.value()?.parse::()? { + Expr::Lit(syn::ExprLit { + lit: Lit::Str(name), + .. + }) => Ok(vec![name]), + Expr::Array(ExprArray { elems, .. }) => elems + .into_iter() + .map(|element| match element { + Expr::Lit(syn::ExprLit { + lit: Lit::Str(name), + .. + }) => Ok(name), + other => Err(syn::Error::new_spanned(other, "expected a string literal")), + }) + .collect(), + other => Err(syn::Error::new_spanned( + other, + "expected a string literal or an array of them", + )), + } +} diff --git a/litellm-rust/crates/layered/AGENTS.md b/litellm-rust/crates/layered/AGENTS.md new file mode 100644 index 00000000000..06f86946e02 --- /dev/null +++ b/litellm-rust/crates/layered/AGENTS.md @@ -0,0 +1,27 @@ +# layered + +`#[derive(Layered)]` declares where each setting of a group comes from. The derive generates `Partial`, with `from_env` and `from_litellm_params`, and the name lists and `Sources` rows for errors and projection. `merge` stacks partials highest first, and the group's crate lowers the merged partial into its final type, where defaults live + +## Runtimes stack different sources + +| Runtime | Stack, highest first | Where config.yaml shows up | +|---|---|---| +| Python SDK | litellm params, env, `litellm.*` globals, defaults | nowhere | +| Python proxy | litellm params, env, `litellm.*` globals, defaults | already copied into env and globals by the proxy | +| Rust gateway | litellm params, env, config file, defaults | read by the gateway | + +Each runtime writes its own `merge([..])` list per group, and a test per runtime pins it. Credentials put litellm params first and the global moves per provider, so never assume one order across groups + +## Who builds which partial + +- `from_env` and `from_litellm_params` are generated, and every runtime that has that source calls them +- A source holding typed values (the config file, a live Python global) gets a hand-written constructor in the crate that owns the source, never in `layered` or in the group's crate +- `layered` never depends on `litellm-config` or pyo3, and no library crate reads the process environment itself. It takes a `Lookup`: `Env` in the Rust gateway, `ProcessEnvironment` in `python-bridge` + +## Hand-written constructors list every field + +A hand-written constructor names every field of the partial and writes `None` where its source has no such key, never `..Default::default()`. Adding a field then fails to compile in each constructor until someone decides what that source does with it + +## Blank means unset + +An empty or whitespace-only env value or string litellm param counts as unset, so the lookup moves on to the next name and the next layer. Mark a field `keep_blank` only when an empty value is a deliberate choice, like `LITELLM_USER_AGENT=""` sending no user agent. Python's `os.getenv(X, litellm.x)` lets an empty env var shadow the global for the SSL settings, and we diverge from that on purpose diff --git a/litellm-rust/crates/layered/Cargo.toml b/litellm-rust/crates/layered/Cargo.toml new file mode 100644 index 00000000000..0be9ad4c7fa --- /dev/null +++ b/litellm-rust/crates/layered/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "litellm-layered" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-core-utils.workspace = true +litellm-layered-derive.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true + +[dev-dependencies] +rstest.workspace = true diff --git a/litellm-rust/crates/layered/src/env.rs b/litellm-rust/crates/layered/src/env.rs new file mode 100644 index 00000000000..f88f4129930 --- /dev/null +++ b/litellm-rust/crates/layered/src/env.rs @@ -0,0 +1,70 @@ +use std::{collections::BTreeMap, fmt}; + +use crate::Lookup; + +#[derive(Clone, Default, PartialEq, Eq)] +pub struct Env(BTreeMap); + +impl Env { + #[allow(clippy::disallowed_methods)] + pub fn from_process() -> Self { + Self( + std::env::vars_os() + .filter_map(|(name, value)| { + Some((name.into_string().ok()?, value.into_string().ok()?)) + }) + .collect(), + ) + } + + pub fn overlaid(self, values: impl IntoIterator) -> Self { + Self(self.0.into_iter().chain(values).collect()) + } + + pub fn of(values: &[(&str, &str)]) -> Self { + Self( + values + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .collect(), + ) + } +} + +impl Lookup for Env { + fn get(&self, name: &str) -> Option { + self.0.get(name).cloned() + } +} + +impl fmt::Debug for Env { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_tuple("Env") + .field(&self.0.keys().collect::>()) + .finish() + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + fn overlaid_values_win_over_the_snapshot() { + let env = Env::of(&[("SHARED", "process"), ("PROCESS_ONLY", "process")]) + .overlaid([("SHARED".to_owned(), "config".to_owned())]); + assert_eq!(env.get("SHARED").as_deref(), Some("config")); + assert_eq!(env.get("PROCESS_ONLY").as_deref(), Some("process")); + assert_eq!(env.get("ABSENT"), None); + } + + #[rstest] + fn debug_lists_names_without_values() { + let debug = format!("{:?}", Env::of(&[("API_KEY", "sk-hidden")])); + assert!(debug.contains("API_KEY")); + assert!(!debug.contains("sk-hidden")); + } +} diff --git a/litellm-rust/crates/layered/src/error.rs b/litellm-rust/crates/layered/src/error.rs new file mode 100644 index 00000000000..e774469912e --- /dev/null +++ b/litellm-rust/crates/layered/src/error.rs @@ -0,0 +1,18 @@ +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +#[error("expected {expected}")] +pub struct Invalid { + expected: &'static str, +} + +impl Invalid { + pub fn expected(expected: &'static str) -> Self { + Self { expected } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +#[error("invalid environment variable {name}: {invalid}")] +pub struct Error { + pub name: &'static str, + pub invalid: Invalid, +} diff --git a/litellm-rust/crates/layered/src/lib.rs b/litellm-rust/crates/layered/src/lib.rs new file mode 100644 index 00000000000..f176a1d4a4d --- /dev/null +++ b/litellm-rust/crates/layered/src/lib.rs @@ -0,0 +1,79 @@ +mod env; +mod error; +pub mod read; +mod sources; + +pub use env::Env; +pub use error::{Error, Invalid}; +pub use litellm_core_utils::settings::Lookup; +pub use litellm_layered_derive::Layered; +pub use sources::Sources; + +pub trait Partial: Default { + fn or(self, lower: Self) -> Self; +} + +pub fn merge(highest_precedence_first: impl IntoIterator) -> P { + highest_precedence_first + .into_iter() + .reduce(P::or) + .unwrap_or_default() +} + +pub fn read_env( + env: &(impl Lookup + ?Sized), + names: &'static [&'static str], + keep_blank: bool, + read: impl Fn(&str) -> Result, Invalid>, +) -> Result, Error> { + for name in names { + let Some(value) = env.get(name) else { + continue; + }; + if !keep_blank && value.trim().is_empty() { + continue; + } + if let Some(value) = read(&value).map_err(|invalid| Error { name, invalid })? { + return Ok(Some(value)); + } + } + Ok(None) +} + +#[doc(hidden)] +pub mod __private { + use std::fmt; + + use serde_json::{Map, Value}; + + pub use serde; + pub use serde_json; + + pub fn from_litellm_params( + params: &Map, + names: &[&str], + keep_blank: bool, + decode: impl Fn(&Value) -> Option, + ) -> Option { + names + .iter() + .filter_map(|name| params.get(*name)) + .filter(|value| keep_blank || !is_blank(value)) + .find_map(decode) + } + + fn is_blank(value: &Value) -> bool { + value.as_str().is_some_and(|text| text.trim().is_empty()) + } + + pub struct Redacted<'a, T>(pub &'a Option); + + impl fmt::Debug for Redacted<'_, T> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0 { + Some(_) => formatter.write_str("Some(\"[REDACTED]\")"), + None => formatter.write_str("None"), + } + } + } +} diff --git a/litellm-rust/crates/layered/src/read.rs b/litellm-rust/crates/layered/src/read.rs new file mode 100644 index 00000000000..8ad7dfbc372 --- /dev/null +++ b/litellm-rust/crates/layered/src/read.rs @@ -0,0 +1,90 @@ +use std::{any::type_name, str::FromStr, time::Duration}; + +use litellm_core_utils::serde_compat::parse_str_bool; + +use crate::Invalid; + +pub fn text From<&'a str>>(value: &str) -> Result, Invalid> { + Ok(Some(T::from(value))) +} + +pub fn truthy From<&'a str>>(value: &str) -> Result, Invalid> { + Ok((!value.is_empty()).then(|| T::from(value))) +} + +pub fn enabled(value: &str) -> Result, Invalid> { + Ok((parse_str_bool(value) == Some(true)).then_some(true)) +} + +pub fn parsed(value: &str) -> Result, Invalid> { + Ok(value.trim().parse().ok()) +} + +pub fn required(value: &str) -> Result, Invalid> { + value + .trim() + .parse() + .map(Some) + .map_err(|_| Invalid::expected(type_name::())) +} + +pub fn seconds(value: &str) -> Result, Invalid> { + Ok(parsed::(value)?.map(|seconds| Duration::from_secs(u64::from(seconds)))) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::lower("true", Some(true))] + #[case::padded(" True ", Some(true))] + #[case::off("false", None)] + #[case::one("1", None)] + fn enabled_only_switches_on_and_never_forces_off( + #[case] value: &str, + #[case] expected: Option, + ) { + assert_eq!(enabled(value), Ok(expected)); + } + + #[rstest] + #[case::empty("", None)] + #[case::kept_untrimmed(" value ", Some(" value "))] + fn truthy_drops_only_empty_values(#[case] value: &str, #[case] expected: Option<&str>) { + assert_eq!(truthy::(value).unwrap().as_deref(), expected); + } + + #[rstest] + fn text_keeps_the_value_as_written() { + assert_eq!(text::(" /a b "), Ok(Some(PathBuf::from(" /a b ")))); + } + + #[rstest] + #[case::padded(" 45 ", Some(45))] + #[case::word("soon", None)] + #[case::fraction("0.5", None)] + fn parsed_trims_and_treats_junk_as_unset(#[case] value: &str, #[case] expected: Option) { + assert_eq!(parsed::(value), Ok(expected)); + } + + #[rstest] + #[case::padded(" 8080 ", Ok(Some(8080)))] + #[case::word("http", Err(Invalid::expected("u16")))] + #[case::empty("", Err(Invalid::expected("u16")))] + #[case::negative("-1", Err(Invalid::expected("u16")))] + fn required_fails_on_junk(#[case] value: &str, #[case] expected: Result, Invalid>) { + assert_eq!(required::(value), expected); + } + + #[rstest] + #[case::whole("45", Some(Duration::from_secs(45)))] + #[case::junk("later", None)] + fn seconds_reads_whole_seconds(#[case] value: &str, #[case] expected: Option) { + assert_eq!(seconds(value), Ok(expected)); + } +} diff --git a/litellm-rust/crates/layered/src/sources.rs b/litellm-rust/crates/layered/src/sources.rs new file mode 100644 index 00000000000..9667ad5e4da --- /dev/null +++ b/litellm-rust/crates/layered/src/sources.rs @@ -0,0 +1,26 @@ +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Sources { + pub setting: &'static str, + pub litellm_params: &'static [&'static str], + pub litellm_setting: Option<&'static str>, + pub env: &'static [&'static str], +} + +impl Sources { + pub fn guidance(&self) -> String { + let steps: Vec = [ + (!self.litellm_params.is_empty()) + .then(|| format!("pass {}", self.litellm_params.join(" or "))), + self.litellm_setting + .map(|setting| format!("set litellm.{setting}")), + (!self.env.is_empty()).then(|| format!("set {}", self.env.join(" or "))), + ] + .into_iter() + .flatten() + .collect(); + match steps.as_slice() { + [first, second, third] => format!("{first}, {second}, or {third}"), + steps => steps.join(" or "), + } + } +} diff --git a/litellm-rust/crates/layered/tests/env.rs b/litellm-rust/crates/layered/tests/env.rs new file mode 100644 index 00000000000..7aa4fab59f7 --- /dev/null +++ b/litellm-rust/crates/layered/tests/env.rs @@ -0,0 +1,121 @@ +use std::time::Duration; + +use litellm_layered::{Env, Error, Layered, Lookup, read, read_env}; +use rstest::rstest; +use serde_json::{Value, json}; + +#[derive(Layered)] +#[expect(dead_code, reason = "only the derived partial is built")] +struct Settings { + #[layer(env = ["FIRST_NAME", "SECOND_NAME"], read = read::text)] + name: String, + #[layer(env = "SWITCH", read = read::enabled)] + switch: bool, + #[layer(env = "COUNT", read = read::parsed::)] + count: u32, + #[layer(env = "PORT", read = read::required::)] + port: u16, + #[layer(env = "IDLE", read = read::seconds)] + idle: Duration, + #[layer(from_env = window_from_env)] + window: (u32, u32), + #[layer(litellm_param = "label", env = "LABEL", read = read::text, keep_blank)] + label: String, + untouched_by_env: bool, +} + +fn window_from_env(env: &(impl Lookup + ?Sized)) -> Result, Error> { + let start = read_env(env, &["WINDOW_START"], false, read::required::)?; + let end = read_env(env, &["WINDOW_END"], false, read::required::)?; + Ok(start.zip(end)) +} + +fn params(value: Value) -> serde_json::Map { + serde_json::from_value(value).unwrap() +} + +#[rstest] +fn each_field_reads_its_env_names_in_order() { + let partial = SettingsPartial::from_env(&Env::of(&[ + ("SECOND_NAME", "second"), + ("SWITCH", "True"), + ("COUNT", " 7 "), + ("PORT", "8080"), + ("IDLE", "45"), + ("WINDOW_START", "1"), + ("WINDOW_END", "2"), + ])) + .unwrap(); + assert_eq!( + partial, + SettingsPartial { + name: Some("second".into()), + switch: Some(true), + count: Some(7), + port: Some(8080), + idle: Some(Duration::from_secs(45)), + window: Some((1, 2)), + label: None, + untouched_by_env: None, + } + ); +} + +#[rstest] +#[case::empty("")] +#[case::whitespace(" ")] +fn blank_values_fall_through_to_the_next_name(#[case] blank: &str) { + let partial = SettingsPartial::from_env(&Env::of(&[ + ("FIRST_NAME", blank), + ("SECOND_NAME", "second"), + ])) + .unwrap(); + assert_eq!(partial.name.as_deref(), Some("second")); +} + +#[rstest] +fn keep_blank_fields_count_a_blank_value_as_set() { + assert_eq!( + SettingsPartial::from_env(&Env::of(&[("LABEL", "")])) + .unwrap() + .label + .as_deref(), + Some("") + ); + assert_eq!( + SettingsPartial::from_litellm_params(¶ms(json!({"label": " "}))) + .label + .as_deref(), + Some(" ") + ); +} + +#[rstest] +fn a_field_without_env_never_comes_from_env() { + let partial = SettingsPartial::from_env(&Env::of(&[("UNTOUCHED_BY_ENV", "true")])).unwrap(); + assert_eq!(partial, SettingsPartial::default()); +} + +#[rstest] +#[case::switch_off("SWITCH", "false")] +#[case::switch_one("SWITCH", "1")] +#[case::lax_count("COUNT", "many")] +fn lax_values_that_do_not_read_are_unset(#[case] name: &str, #[case] value: &str) { + assert_eq!( + SettingsPartial::from_env(&Env::of(&[(name, value)])).unwrap(), + SettingsPartial::default() + ); +} + +#[rstest] +#[case::port("PORT", "abc")] +#[case::port_overflow("PORT", "65536")] +#[case::from_env_field("WINDOW_START", "soon")] +fn strict_values_that_do_not_read_fail_naming_the_variable( + #[case] name: &'static str, + #[case] value: &str, +) { + let error = SettingsPartial::from_env(&Env::of(&[(name, value)])).unwrap_err(); + assert_eq!(error.name, name); + assert!(error.to_string().contains(name)); +} diff --git a/litellm-rust/crates/layered/tests/partial.rs b/litellm-rust/crates/layered/tests/partial.rs new file mode 100644 index 00000000000..88983169359 --- /dev/null +++ b/litellm-rust/crates/layered/tests/partial.rs @@ -0,0 +1,137 @@ +use litellm_layered::{Layered, Sources, merge, read}; +use rstest::rstest; +use serde::{Deserialize, Deserializer}; +use serde_json::{Value, json}; + +#[derive(Layered)] +#[expect(dead_code, reason = "only the derived partial is built")] +struct Credentials { + #[layer( + litellm_param = ["project", "legacy_project"], + litellm_setting = "credential_project", + env = ["PROJECT", "PROJECT_FALLBACK"], + read = read::text, + )] + project: String, + #[layer(litellm_param = "token", env = "TOKEN", read = read::text, secret, setting = "API token")] + token: String, + #[layer(litellm_param = "document", deserialize_with = document_text)] + document: String, + #[layer(env = "LIMIT")] + limit: u32, +} + +fn document_text<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + Ok(match Value::deserialize(deserializer)? { + Value::String(text) => Some(text), + Value::Object(object) => Some(Value::Object(object).to_string()), + _ => None, + }) +} + +fn params(value: Value) -> serde_json::Map { + serde_json::from_value(value).unwrap() +} + +#[rstest] +#[case::first_alias(json!({"project": "first", "legacy_project": "second"}), Some("first"))] +#[case::second_alias(json!({"legacy_project": "second"}), Some("second"))] +#[case::blank_falls_to_alias(json!({"project": " ", "legacy_project": "second"}), Some("second"))] +#[case::null_falls_to_alias(json!({"project": null, "legacy_project": "second"}), Some("second"))] +#[case::non_string_is_ignored(json!({"project": 5}), None)] +#[case::absent(json!({}), None)] +fn litellm_params_read_aliases_in_order(#[case] value: Value, #[case] expected: Option<&str>) { + assert_eq!( + CredentialsPartial::from_litellm_params(¶ms(value)) + .project + .as_deref(), + expected + ); +} + +#[rstest] +fn litellm_params_use_the_field_decoder() { + let partial = CredentialsPartial::from_litellm_params(¶ms(json!({ + "document": {"type": "service_account"}, + "limit": 5, + }))); + assert_eq!( + partial.document.as_deref(), + Some(r#"{"type":"service_account"}"#) + ); + assert_eq!(partial.limit, None); +} + +#[rstest] +fn merge_takes_each_field_from_the_highest_partial_that_sets_it() { + let merged = merge([ + CredentialsPartial { + project: Some("params".into()), + ..CredentialsPartial::default() + }, + CredentialsPartial { + project: Some("env".into()), + token: Some("env-token".into()), + ..CredentialsPartial::default() + }, + ]); + assert_eq!(merged.project.as_deref(), Some("params")); + assert_eq!(merged.token.as_deref(), Some("env-token")); + assert_eq!( + merge(Vec::::new()), + CredentialsPartial::default() + ); +} + +#[rstest] +fn debug_redacts_secret_fields() { + let debug = format!( + "{:?}", + CredentialsPartial { + project: Some("visible-project".into()), + token: Some("hidden-token".into()), + ..CredentialsPartial::default() + } + ); + assert!(debug.contains("visible-project")); + assert!(!debug.contains("hidden-token")); + assert!(debug.contains("REDACTED")); +} + +#[rstest] +fn name_lists_follow_the_declaration() { + assert_eq!( + Credentials::LITELLM_PARAMS, + ["project", "legacy_project", "token", "document"] + ); + assert_eq!( + Credentials::LITELLM_SETTINGS, + [("credential_project", "project")] + ); + assert_eq!( + Credentials::ENV_NAMES, + ["PROJECT", "PROJECT_FALLBACK", "TOKEN", "LIMIT"] + ); + assert_eq!( + Credentials::PROJECT, + &Sources { + setting: "project", + litellm_params: &["project", "legacy_project"], + litellm_setting: Some("credential_project"), + env: &["PROJECT", "PROJECT_FALLBACK"], + } + ); + assert_eq!(Credentials::SOURCES[1].setting, "API token"); +} + +#[rstest] +#[case::every_source( + Credentials::PROJECT, + "pass project or legacy_project, set litellm.credential_project, or set PROJECT or PROJECT_FALLBACK" +)] +#[case::params_and_env(Credentials::TOKEN, "pass token or set TOKEN")] +#[case::params_only(Credentials::DOCUMENT, "pass document")] +#[case::env_only(Credentials::LIMIT, "set LIMIT")] +fn guidance_names_exactly_the_declared_sources(#[case] sources: &Sources, #[case] expected: &str) { + assert_eq!(sources.guidance(), expected); +} diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 6fc05c43146..192a8420909 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -42,6 +42,7 @@ litellm-inference-chat.workspace = true litellm-inference-ocr.workspace = true litellm-core-utils.workspace = true litellm-http.workspace = true +litellm-layered.workspace = true litellm-llms.workspace = true litellm-secrets = { workspace = true, features = ["aws", "azure", "google", "hashicorp", "cyberark"] } litellm-secrets-types.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/http.rs b/litellm-rust/crates/python-bridge/src/http.rs index 57bf8aafc4e..de1099a55da 100644 --- a/litellm-rust/crates/python-bridge/src/http.rs +++ b/litellm-rust/crates/python-bridge/src/http.rs @@ -6,10 +6,11 @@ use std::{ use litellm_core_utils::settings::ProcessEnvironment; use litellm_http::{ - Client, ClientVariant, HttpClientConfig, HttpClientPool, HttpSettings, HttpSettingsLayer, + Client, ClientVariant, HttpClientConfig, HttpClientPool, HttpKeysPartial, HttpSettings, Resolution, SslVerify, TlsSource, Unsupported, media::{PublicDnsResolver, UrlPolicy}, }; +use litellm_layered::merge; use pyo3::{ exceptions::PyValueError, prelude::*, @@ -101,14 +102,15 @@ pub(crate) fn call_config( kwargs: &Bound<'_, PyDict>, asynchronous: bool, ) -> PyResult { - let settings = HttpSettings::from_layers([ + let settings = HttpSettings::from(merge([ for_call(call_ssl_verify(kwargs)?, asynchronous), - HttpSettingsLayer::from_environment(&ProcessEnvironment), + HttpKeysPartial::from_env(&ProcessEnvironment) + .map_err(|error| PyValueError::new_err(error.to_string()))?, match PythonSettings::Http.read_or_unset(py)? { Some(snapshot) => configured(&snapshot)?, - None => HttpSettingsLayer::default(), + None => HttpKeysPartial::default(), }, - ]) + ])) .without_missing_files(&|path: &Path| path.exists()); let resolution = Resolution::from(&settings); for unsupported in unreported(&REPORTED_UNSUPPORTED, resolution.unsupported) { @@ -191,17 +193,29 @@ fn call_ssl_verify(kwargs: &Bound<'_, PyDict>) -> PyResult> { } } -fn for_call(call_ssl_verify: Option, asynchronous: bool) -> HttpSettingsLayer { - HttpSettingsLayer { +fn for_call(call_ssl_verify: Option, asynchronous: bool) -> HttpKeysPartial { + HttpKeysPartial { ssl_verify: call_ssl_verify, + ssl_cert_file: None, + ssl_certificate: None, + ssl_security_level: None, + ssl_ecdh_curve: None, + force_ipv4: None, + http2: None, + aiohttp_trust_env: None, + disable_aiohttp_trust_env: None, disable_aiohttp_transport: (!asynchronous).then_some(true), - ..HttpSettingsLayer::default() + user_agent: None, + tcp_keepalive: None, + pool_idle_timeout: None, + proxies: None, } } -fn configured(snapshot: &Snapshot<'_>) -> PyResult { - Ok(HttpSettingsLayer { +fn configured(snapshot: &Snapshot<'_>) -> PyResult { + Ok(HttpKeysPartial { ssl_verify: snapshot.read(&SSL_VERIFY)?, + ssl_cert_file: None, ssl_certificate: snapshot.read(&SSL_CERTIFICATE)?.map(PathBuf::from), ssl_security_level: snapshot.read(&SSL_SECURITY_LEVEL)?, ssl_ecdh_curve: snapshot.read(&SSL_ECDH_CURVE)?, @@ -211,7 +225,9 @@ fn configured(snapshot: &Snapshot<'_>) -> PyResult { disable_aiohttp_trust_env: Some(snapshot.read(&DISABLE_AIOHTTP_TRUST_ENV)?), disable_aiohttp_transport: Some(snapshot.read(&DISABLE_AIOHTTP_TRANSPORT)?), user_agent: Some(snapshot.read(&USER_AGENT)?), - ..HttpSettingsLayer::default() + tcp_keepalive: None, + pool_idle_timeout: None, + proxies: None, }) } @@ -221,8 +237,14 @@ mod tests { use pyo3::exceptions::PyRuntimeError; use rstest::rstest; + use litellm_layered::Env; + use super::*; + fn stack(highest_first: impl IntoIterator) -> HttpSettings { + HttpSettings::from(merge(highest_first)) + } + fn evaluate<'py>(py: Python<'py>, source: &str) -> Bound<'py, PyAny> { py.eval(&std::ffi::CString::new(source).unwrap(), None, None) .unwrap() @@ -260,7 +282,7 @@ settings = types.SimpleNamespace(**defaults) Python::attach(|py| { let layer = configured(&python_settings(py, "")).unwrap(); assert_eq!( - HttpSettings::from_layers([layer]), + stack([layer]), HttpSettings { ssl_verify: Some(SslVerify::Enabled), user_agent: Some("litellm/test".into()), @@ -319,7 +341,7 @@ user_agent='litellm/9.9.9', .unwrap(); assert_eq!( layer, - HttpSettingsLayer { + HttpKeysPartial { ssl_verify: Some(SslVerify::CaBundle("/etc/ssl/corp.pem".into())), ssl_certificate: Some("/etc/ssl/client.pem".into()), ssl_security_level: Some("2".into()), @@ -330,7 +352,7 @@ user_agent='litellm/9.9.9', disable_aiohttp_trust_env: Some(true), disable_aiohttp_transport: Some(true), user_agent: Some("litellm/9.9.9".into()), - ..HttpSettingsLayer::default() + ..HttpKeysPartial::default() } ); }); @@ -340,10 +362,9 @@ user_agent='litellm/9.9.9', fn user_agent_environment_variable_beats_the_python_default() { Python::initialize(); Python::attach(|py| { - let settings = HttpSettings::from_layers([ - HttpSettingsLayer::from_environment(&|name: &str| { - (name == "LITELLM_USER_AGENT").then(|| "operator/1".to_string()) - }), + let settings = stack([ + HttpKeysPartial::from_env(&Env::of(&[("LITELLM_USER_AGENT", "operator/1")])) + .unwrap(), configured(&python_settings(py, "")).unwrap(), ]); assert_eq!(settings.user_agent.as_deref(), Some("operator/1")); @@ -362,7 +383,7 @@ user_agent='litellm/9.9.9', Python::initialize(); Python::attach(|py| { let layer = configured(&python_settings(py, overrides)).unwrap(); - let config = Resolution::from(&HttpSettings::from_layers([layer])).config; + let config = Resolution::from(&stack([layer])).config; assert_eq!(config.verify, expected); }); } @@ -411,10 +432,10 @@ user_agent='litellm/9.9.9', }); } - fn configured_ssl_verify(ssl_verify: SslVerify) -> HttpSettingsLayer { - HttpSettingsLayer { + fn configured_ssl_verify(ssl_verify: SslVerify) -> HttpKeysPartial { + HttpKeysPartial { ssl_verify: Some(ssl_verify), - ..HttpSettingsLayer::default() + ..HttpKeysPartial::default() } } @@ -425,8 +446,7 @@ user_agent='litellm/9.9.9', let kwargs = PyDict::new(py); kwargs.set_item("ssl_verify", false).unwrap(); let call = for_call(call_ssl_verify(&kwargs).unwrap(), true); - let settings = - HttpSettings::from_layers([call, configured_ssl_verify(SslVerify::Enabled)]); + let settings = stack([call, configured_ssl_verify(SslVerify::Enabled)]); assert_eq!(settings.ssl_verify, Some(SslVerify::Disabled)); }); } @@ -438,8 +458,7 @@ user_agent='litellm/9.9.9', let kwargs = PyDict::new(py); kwargs.set_item("ssl_verify", py.None()).unwrap(); let call = for_call(call_ssl_verify(&kwargs).unwrap(), true); - let settings = - HttpSettings::from_layers([call, configured_ssl_verify(SslVerify::Disabled)]); + let settings = stack([call, configured_ssl_verify(SslVerify::Disabled)]); assert_eq!(settings.ssl_verify, Some(SslVerify::Disabled)); }); } @@ -485,12 +504,12 @@ user_agent='litellm/9.9.9', #[case] asynchronous: bool, #[case] expected: bool, ) { - let opted_out = HttpSettingsLayer { + let opted_out = HttpKeysPartial { disable_aiohttp_trust_env: Some(true), disable_aiohttp_transport: Some(false), - ..HttpSettingsLayer::default() + ..HttpKeysPartial::default() }; - let settings = HttpSettings::from_layers([for_call(None, asynchronous), opted_out]); + let settings = stack([for_call(None, asynchronous), opted_out]); assert_eq!(settings.trust_proxy_env, expected); } #[rstest]