This commit is contained in:
yujonglee 2026-10-24 10:52:24 -07:00 • committed by GitHub
commit 12016b1dc1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
26 changed files with 1199 additions and 265 deletions

View file

@ -3743,12 +3743,14 @@ dependencies = [
"litellm-gateway-ui",
"litellm-http",
"litellm-inference",
"litellm-layered",
"litellm-llms",
"litellm-secrets",
"litellm-tracing",
"rstest",
"serde",
"serde_json",
"serde_yaml_ng",
"tempfile",
"thiserror 2.0.19",
"tokio",
@ -3944,6 +3946,7 @@ dependencies = [
"http 1.4.2",
"hyper-util",
"litellm-core-utils",
"litellm-layered",
"rcgen",
"reqwest 0.12.28",
"reqwest 0.13.5",
@ -4117,6 +4120,7 @@ dependencies = [
"litellm-auth",
"litellm-http",
"litellm-inference",
"litellm-layered",
"litellm-secrets",
"tokio",
]
@ -4141,6 +4145,27 @@ dependencies = [
"wiremock",
]
[[package]]
name = "litellm-layered"
version = "0.1.0"
dependencies = [
"litellm-core-utils",
"litellm-layered-derive",
"rstest",
"serde",
"serde_json",
"thiserror 2.0.19",
]
[[package]]
name = "litellm-layered-derive"
version = "0.1.0"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "litellm-llms"
version = "0.1.0"
@ -4231,6 +4256,7 @@ dependencies = [
"litellm-inference-ocr",
"litellm-inference-responses",
"litellm-inference-transcription",
"litellm-layered",
"litellm-llms",
"litellm-llms-types",
"litellm-router-types",

View file

@ -50,6 +50,8 @@ litellm-http = { path = "crates/http" }
litellm-llms = { path = "crates/llms" }
litellm-llms-types = { path = "crates/llms-types" }
litellm-core-utils = { path = "crates/core-utils" }
litellm-layered = { path = "crates/layered" }
litellm-layered-derive = { path = "crates/layered-derive" }
litellm-db = { path = "crates/db" }
litellm-db-testing = { path = "crates/db-testing" }
litellm-cache = { path = "crates/cache" }

View file

@ -53,17 +53,6 @@ pub fn resolve_non_empty(
})
}
pub trait Layer: Default {
fn or(self, lower: Self) -> Self;
}
pub fn merge<L: Layer>(highest_precedence_first: impl IntoIterator<Item = L>) -> L {
highest_precedence_first
.into_iter()
.reduce(L::or)
.unwrap_or_default()
}
#[cfg(test)]
mod tests {
use super::*;
@ -114,49 +103,4 @@ mod tests {
assert_eq!(env.parsed::<f64>("FRACTION"), Some(0.5));
assert_eq!(env.parsed::<u32>("ABSENT"), None);
}
#[derive(Debug, Default, PartialEq)]
struct Pair {
first: Option<u8>,
second: Option<u8>,
}
impl Layer for Pair {
fn or(self, lower: Self) -> Self {
Self {
first: self.first.or(lower.first),
second: self.second.or(lower.second),
}
}
}
#[test]
fn merge_takes_each_field_from_the_highest_layer_that_sets_it() {
let merged = merge([
Pair {
first: Some(1),
second: None,
},
Pair {
first: Some(2),
second: Some(2),
},
Pair {
first: Some(3),
second: Some(3),
},
]);
assert_eq!(
merged,
Pair {
first: Some(1),
second: Some(2),
}
);
}
#[test]
fn merging_no_layers_yields_the_empty_layer() {
assert_eq!(merge(Vec::<Pair>::new()), Pair::default());
}
}

View file

@ -20,6 +20,7 @@ litellm-gateway-ui.workspace = true
litellm-auth-types.workspace = true
litellm-config.workspace = true
litellm-http = { workspace = true, features = ["mcp"] }
litellm-layered.workspace = true
litellm-llms.workspace = true
litellm-secrets.workspace = true
litellm-tracing.workspace = true
@ -33,6 +34,7 @@ tower-sessions-moka-store = "0.15.0"
[dev-dependencies]
base64.workspace = true
rstest.workspace = true
serde_yaml_ng = "0.10.0"
tempfile.workspace = true
tokio = { workspace = true, features = ["sync"] }
tower = { version = "0.5", features = ["util"] }

View file

@ -2,6 +2,8 @@
pub enum Error {
#[error(transparent)]
Http(#[from] litellm_http::Error),
#[error(transparent)]
Settings(#[from] litellm_layered::Error),
#[error("environment_variables values must be strings, numbers or booleans")]
Environment,
#[error("MCP host does not yet support configured setting {0}")]

View file

@ -0,0 +1,88 @@
use litellm_config::{LiteLlmSettings, Spelled};
use litellm_http::{HttpKeysPartial, HttpSettings, SslVerify};
use litellm_layered::{Lookup, merge};
pub(crate) fn http_settings(
env: &(impl Lookup + ?Sized),
settings: &LiteLlmSettings,
) -> Result<HttpSettings, litellm_layered::Error> {
Ok(HttpSettings::from(merge([
HttpKeysPartial::from_env(env)?,
written(settings),
])))
}
fn written(settings: &LiteLlmSettings) -> HttpKeysPartial {
HttpKeysPartial {
ssl_verify: settings.ssl_verify.as_ref().map(|value| match value {
Spelled::Value(true) => SslVerify::Enabled,
Spelled::Value(false) => SslVerify::Disabled,
Spelled::Text(value) => SslVerify::parse(value),
}),
ssl_cert_file: None,
ssl_certificate: settings.ssl_certificate.as_ref().map(Into::into),
ssl_security_level: settings.ssl_security_level.clone(),
ssl_ecdh_curve: settings.ssl_ecdh_curve.clone(),
force_ipv4: settings.force_ipv4,
http2: settings.http2,
aiohttp_trust_env: settings.aiohttp_trust_env,
disable_aiohttp_trust_env: settings.disable_aiohttp_trust_env,
disable_aiohttp_transport: settings.disable_aiohttp_transport,
user_agent: None,
tcp_keepalive: None,
pool_idle_timeout: None,
proxies: None,
}
}
#[cfg(test)]
mod tests {
use litellm_layered::Env;
use rstest::rstest;
use super::*;
fn file(yaml: &str) -> LiteLlmSettings {
serde_yaml_ng::from_str(yaml).unwrap()
}
#[rstest]
#[case::env_beats_file(&[("SSL_VERIFY", "false")], "ssl_verify: true", Some(SslVerify::Disabled))]
#[case::file_fills_unset_env(&[], "ssl_verify: false", Some(SslVerify::Disabled))]
#[case::neither(&[], "{}", None)]
fn env_is_stacked_above_the_config_file(
#[case] env: &[(&str, &str)],
#[case] yaml: &str,
#[case] expected: Option<SslVerify>,
) {
let settings = http_settings(&Env::of(env), &file(yaml)).unwrap();
assert_eq!(settings.ssl_verify, expected);
}
#[rstest]
#[case::bool_false("false", "false", SslVerify::Disabled)]
#[case::text_false("'False'", "False", SslVerify::Disabled)]
#[case::text_true("'true'", "true", SslVerify::Enabled)]
#[case::bundle("/etc/ca.pem", "/etc/ca.pem", SslVerify::CaBundle("/etc/ca.pem".into()))]
fn ssl_verify_spellings_read_the_same_from_the_file_and_the_environment(
#[case] yaml: &str,
#[case] env: &str,
#[case] expected: SslVerify,
) {
let from_file =
http_settings(&Env::default(), &file(&format!("ssl_verify: {yaml}"))).unwrap();
let from_env = http_settings(&Env::of(&[("SSL_VERIFY", env)]), &file("{}")).unwrap();
assert_eq!(from_file.ssl_verify, Some(expected.clone()));
assert_eq!(from_env.ssl_verify, Some(expected));
}
#[rstest]
fn flags_are_only_turned_on_by_the_environment() {
let settings = http_settings(
&Env::of(&[("LITELLM_HTTP2", "false")]),
&file("http2: true"),
)
.unwrap();
assert!(settings.http2);
}
}

View file

@ -1,5 +1,6 @@
mod auth;
mod error;
mod http;
mod secrets;
pub use error::Error;
@ -17,10 +18,7 @@ use http_body_util::BodyExt;
use litellm_config::Config;
use litellm_gateway_auth::Auth;
use litellm_gateway_inference::{Gateway, ModelRouter};
use litellm_http::{
ClientVariant, HttpClientPool, HttpSettings, HttpSettingsLayer, Resolution, SslVerify,
media::PublicDnsResolver,
};
use litellm_http::{ClientVariant, HttpClientPool, Resolution, media::PublicDnsResolver};
use litellm_inference::resources::CoreResources;
use litellm_secrets::source::EnvironmentSecrets;
use litellm_tracing::ByteChunk;
@ -35,26 +33,7 @@ pub fn build_inference(config: &Config) -> Result<Arc<Gateway>, Error> {
.map(|value| value.expose().to_owned())
.or_else(|| std::env::var(name).ok())
};
let settings = &config.litellm_settings;
let http_settings = HttpSettings::from_layers([
HttpSettingsLayer::from_environment(&lookup),
HttpSettingsLayer {
ssl_verify: settings.ssl_verify.as_ref().map(|value| match value {
litellm_config::Spelled::Value(true) => SslVerify::Enabled,
litellm_config::Spelled::Value(false) => SslVerify::Disabled,
litellm_config::Spelled::Text(value) => SslVerify::parse(value),
}),
ssl_certificate: settings.ssl_certificate.as_ref().map(Into::into),
ssl_security_level: settings.ssl_security_level.clone(),
ssl_ecdh_curve: settings.ssl_ecdh_curve.clone(),
force_ipv4: settings.force_ipv4,
http2: settings.http2,
aiohttp_trust_env: settings.aiohttp_trust_env,
disable_aiohttp_trust_env: settings.disable_aiohttp_trust_env,
disable_aiohttp_transport: settings.disable_aiohttp_transport,
..Default::default()
},
]);
let http_settings = http::http_settings(&lookup, &config.litellm_settings)?;
let http = Resolution::from(&http_settings).config;
let client = pool.client(&http, ClientVariant::Provider)?;
let secrets = Arc::new(secrets::ConfigSecrets::new(

View file

@ -14,6 +14,7 @@ rmcp = { version = "=3.4.1", default-features = false, features = ["client", "tr
reqwest-mcp = { package = "reqwest", version = "0.13.2", default-features = false, features = ["rustls-no-provider", "json", "stream", "http2"], optional = true }
http.workspace = true
litellm-core-utils.workspace = true
litellm-layered.workspace = true
hyper-util.workspace = true
reqwest.workspace = true
rustls.workspace = true

View file

@ -24,5 +24,5 @@ pub use config::{ClientIdentity, HttpClientConfig, Resolution, Verify};
pub use error::{Error, TlsSource};
pub use pool::{ClientVariant, HttpClientPool};
pub use proxy::EnvironmentProxies;
pub use settings::{HttpSettings, HttpSettingsLayer, SslVerify, TcpKeepalive};
pub use settings::{HttpKeys, HttpKeysPartial, HttpSettings, SslVerify, TcpKeepalive};
pub use tls::{KeyExchangeGroup, Tls12CipherSuite, Unsupported};

View file

@ -1,5 +1,5 @@
use hyper_util::client::proxy::matcher::Matcher;
use litellm_core_utils::settings::Lookup;
use litellm_layered::Lookup;
use veil::Redact;
#[derive(Clone, Redact, Default, PartialEq, Eq, Hash)]
@ -14,14 +14,22 @@ pub struct EnvironmentProxies {
}
impl EnvironmentProxies {
pub fn from_environment(env: &impl Lookup) -> Self {
pub fn from_environment(env: &(impl Lookup + ?Sized)) -> Self {
Self::resolve(env, cfg!(windows))
}
fn resolve(env: &impl Lookup, names_ignore_case: bool) -> Self {
pub fn from_env(env: &(impl Lookup + ?Sized)) -> Result<Option<Self>, litellm_layered::Error> {
Ok(Some(Self::from_environment(env)).filter(|proxies| *proxies != Self::default()))
}
fn resolve(env: &(impl Lookup + ?Sized), names_ignore_case: bool) -> Self {
let lowercase_first = |upper: Option<&str>, lower: &str| {
env.get(lower)
.or_else(|| upper.and_then(|name| env.truthy(name)))
.or_else(|| {
upper
.and_then(|name| env.get(name))
.filter(|value| !value.is_empty())
})
.unwrap_or_default()
};
let is_cgi = env.get("REQUEST_METHOD").is_some();
@ -80,19 +88,11 @@ impl EnvironmentProxies {
#[cfg(test)]
mod tests {
use litellm_layered::Env;
use rstest::rstest;
use super::*;
fn env_of(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
move |name| {
values
.iter()
.find(|(key, _)| *key == name)
.map(|(_, value)| value.to_string())
}
}
fn url(value: &str) -> reqwest::Url {
reqwest::Url::parse(value).unwrap()
}
@ -108,7 +108,7 @@ mod tests {
#[case] target: &str,
#[case] expected: bool,
) {
let proxies = EnvironmentProxies::from_environment(&env_of(env));
let proxies = EnvironmentProxies::from_environment(&Env::of(env));
assert_eq!(proxies.matcher()(&url(target)), expected);
}
@ -126,8 +126,8 @@ mod tests {
#[case] equivalent: &'static [(&'static str, &'static str)],
) {
assert_eq!(
EnvironmentProxies::from_environment(&env_of(env)),
EnvironmentProxies::from_environment(&env_of(equivalent))
EnvironmentProxies::from_environment(&Env::of(env)),
EnvironmentProxies::from_environment(&Env::of(equivalent))
);
}
@ -149,7 +149,7 @@ mod tests {
#[test]
fn a_cgi_request_still_proxies_https_through_the_configured_proxy() {
let proxies = EnvironmentProxies::from_environment(&env_of(&[
let proxies = EnvironmentProxies::from_environment(&Env::of(&[
("REQUEST_METHOD", "GET"),
("HTTPS_PROXY", "http://proxy:3128"),
]));
@ -158,7 +158,7 @@ mod tests {
#[test]
fn debug_output_hides_proxy_credentials_but_shows_which_variables_are_set() {
let proxies = EnvironmentProxies::from_environment(&env_of(&[
let proxies = EnvironmentProxies::from_environment(&Env::of(&[
("HTTPS_PROXY", "http://operator:hunter2@proxy.corp:3128"),
("NO_PROXY", "internal.test"),
]));
@ -170,7 +170,7 @@ mod tests {
#[test]
fn an_empty_environment_proxies_nothing() {
let proxies = EnvironmentProxies::from_environment(&env_of(&[]));
let proxies = EnvironmentProxies::from_environment(&Env::of(&[]));
assert_eq!(proxies, EnvironmentProxies::default());
assert!(proxies.reqwest_proxies().is_empty());
}

View file

@ -3,10 +3,8 @@ use std::{
time::Duration,
};
use litellm_core_utils::{
serde_compat::parse_str_bool,
settings::{Layer, Lookup, merge},
};
use litellm_core_utils::serde_compat::parse_str_bool;
use litellm_layered::{Invalid, Layered, Lookup, read, read_env};
use crate::proxy::EnvironmentProxies;
@ -25,6 +23,10 @@ impl SslVerify {
_ => Self::CaBundle(PathBuf::from(value)),
}
}
pub fn parse_env(value: &str) -> Result<Option<Self>, Invalid> {
Ok(Some(Self::parse(value)))
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
@ -34,78 +36,62 @@ pub struct TcpKeepalive {
pub retries: u32,
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct HttpSettingsLayer {
pub ssl_verify: Option<SslVerify>,
pub ssl_cert_file: Option<PathBuf>,
pub ssl_certificate: Option<PathBuf>,
pub ssl_security_level: Option<String>,
pub ssl_ecdh_curve: Option<String>,
pub force_ipv4: Option<bool>,
pub http2: Option<bool>,
pub aiohttp_trust_env: Option<bool>,
pub disable_aiohttp_trust_env: Option<bool>,
pub disable_aiohttp_transport: Option<bool>,
pub user_agent: Option<String>,
pub tcp_keepalive: Option<TcpKeepalive>,
pub pool_idle_timeout: Option<Duration>,
pub proxies: Option<EnvironmentProxies>,
}
impl HttpSettingsLayer {
pub fn from_environment(env: &impl Lookup) -> Self {
let seconds = |name: &str, default: u32| {
Duration::from_secs(u64::from(env.parsed::<u32>(name).unwrap_or(default)))
impl TcpKeepalive {
pub fn from_env(env: &(impl Lookup + ?Sized)) -> Result<Option<Self>, litellm_layered::Error> {
if read_env(env, &["AIOHTTP_SO_KEEPALIVE"], false, read::enabled)?.is_none() {
return Ok(None);
}
let seconds = |names: &'static [&'static str], default: u64| {
Ok::<_, litellm_layered::Error>(
read_env(env, names, false, read::seconds)?.unwrap_or(Duration::from_secs(default)),
)
};
Self {
ssl_verify: env.get("SSL_VERIFY").map(|value| SslVerify::parse(&value)),
ssl_cert_file: env.truthy("SSL_CERT_FILE").map(PathBuf::from),
ssl_certificate: env.get("SSL_CERTIFICATE").map(PathBuf::from),
ssl_security_level: env.get("SSL_SECURITY_LEVEL"),
ssl_ecdh_curve: env.get("SSL_ECDH_CURVE"),
force_ipv4: None,
http2: env.enabled("LITELLM_HTTP2"),
aiohttp_trust_env: env.enabled("AIOHTTP_TRUST_ENV"),
disable_aiohttp_trust_env: env.enabled("DISABLE_AIOHTTP_TRUST_ENV"),
disable_aiohttp_transport: env.enabled("DISABLE_AIOHTTP_TRANSPORT"),
user_agent: env.get("LITELLM_USER_AGENT"),
tcp_keepalive: env.enabled("AIOHTTP_SO_KEEPALIVE").map(|_| TcpKeepalive {
idle: seconds("AIOHTTP_TCP_KEEPIDLE", 60),
interval: seconds("AIOHTTP_TCP_KEEPINTVL", 30),
retries: env.parsed("AIOHTTP_TCP_KEEPCNT").unwrap_or(5),
}),
pool_idle_timeout: env
.parsed::<u32>("AIOHTTP_KEEPALIVE_TIMEOUT")
.map(|timeout| Duration::from_secs(u64::from(timeout))),
proxies: Some(EnvironmentProxies::from_environment(env))
.filter(|proxies| *proxies != EnvironmentProxies::default()),
}
Ok(Some(Self {
idle: seconds(&["AIOHTTP_TCP_KEEPIDLE"], 60)?,
interval: seconds(&["AIOHTTP_TCP_KEEPINTVL"], 30)?,
retries: read_env(env, &["AIOHTTP_TCP_KEEPCNT"], false, read::parsed)?.unwrap_or(5),
}))
}
}
impl Layer for HttpSettingsLayer {
fn or(self, lower: Self) -> Self {
Self {
ssl_verify: self.ssl_verify.or(lower.ssl_verify),
ssl_cert_file: self.ssl_cert_file.or(lower.ssl_cert_file),
ssl_certificate: self.ssl_certificate.or(lower.ssl_certificate),
ssl_security_level: self.ssl_security_level.or(lower.ssl_security_level),
ssl_ecdh_curve: self.ssl_ecdh_curve.or(lower.ssl_ecdh_curve),
force_ipv4: self.force_ipv4.or(lower.force_ipv4),
http2: self.http2.or(lower.http2),
aiohttp_trust_env: self.aiohttp_trust_env.or(lower.aiohttp_trust_env),
disable_aiohttp_trust_env: self
.disable_aiohttp_trust_env
.or(lower.disable_aiohttp_trust_env),
disable_aiohttp_transport: self
.disable_aiohttp_transport
.or(lower.disable_aiohttp_transport),
user_agent: self.user_agent.or(lower.user_agent),
tcp_keepalive: self.tcp_keepalive.or(lower.tcp_keepalive),
pool_idle_timeout: self.pool_idle_timeout.or(lower.pool_idle_timeout),
proxies: self.proxies.or(lower.proxies),
}
}
#[derive(Layered)]
pub struct HttpKeys {
#[layer(env = "SSL_VERIFY", read = SslVerify::parse_env, litellm_setting = "ssl_verify")]
pub ssl_verify: SslVerify,
#[layer(env = "SSL_CERT_FILE", read = read::truthy)]
pub ssl_cert_file: PathBuf,
#[layer(env = "SSL_CERTIFICATE", read = read::text, litellm_setting = "ssl_certificate")]
pub ssl_certificate: PathBuf,
#[layer(env = "SSL_SECURITY_LEVEL", read = read::text, litellm_setting = "ssl_security_level")]
pub ssl_security_level: String,
#[layer(env = "SSL_ECDH_CURVE", read = read::text, litellm_setting = "ssl_ecdh_curve")]
pub ssl_ecdh_curve: String,
#[layer(litellm_setting = "force_ipv4")]
pub force_ipv4: bool,
#[layer(env = "LITELLM_HTTP2", read = read::enabled, litellm_setting = "http2")]
pub http2: bool,
#[layer(env = "AIOHTTP_TRUST_ENV", read = read::enabled, litellm_setting = "aiohttp_trust_env")]
pub aiohttp_trust_env: bool,
#[layer(
env = "DISABLE_AIOHTTP_TRUST_ENV",
read = read::enabled,
litellm_setting = "disable_aiohttp_trust_env"
)]
pub disable_aiohttp_trust_env: bool,
#[layer(
env = "DISABLE_AIOHTTP_TRANSPORT",
read = read::enabled,
litellm_setting = "disable_aiohttp_transport"
)]
pub disable_aiohttp_transport: bool,
#[layer(env = "LITELLM_USER_AGENT", read = read::text, keep_blank)]
pub user_agent: String,
#[layer(from_env = TcpKeepalive::from_env)]
pub tcp_keepalive: TcpKeepalive,
#[layer(env = "AIOHTTP_KEEPALIVE_TIMEOUT", read = read::seconds)]
pub pool_idle_timeout: Duration,
#[layer(from_env = EnvironmentProxies::from_env)]
pub proxies: EnvironmentProxies,
}
#[derive(Clone, Debug, PartialEq, Eq)]
@ -145,11 +131,8 @@ impl Default for HttpSettings {
}
}
impl HttpSettings {
pub fn from_layers(
highest_precedence_first: impl IntoIterator<Item = HttpSettingsLayer>,
) -> Self {
let merged = merge(highest_precedence_first);
impl From<HttpKeysPartial> for HttpSettings {
fn from(merged: HttpKeysPartial) -> Self {
let defaults = Self::default();
let http2 = merged.http2.unwrap_or(defaults.http2);
Self {
@ -173,7 +156,9 @@ impl HttpSettings {
..defaults
}
}
}
impl HttpSettings {
pub fn without_missing_files(self, exists: &dyn Fn(&Path) -> bool) -> Self {
Self {
ssl_verify: match self.ssl_verify {
@ -190,19 +175,16 @@ impl HttpSettings {
mod tests {
use rstest::rstest;
use litellm_layered::{Env, merge};
use super::*;
fn no_env(_: &str) -> Option<String> {
None
fn stack(highest_first: impl IntoIterator<Item = HttpKeysPartial>) -> HttpSettings {
HttpSettings::from(merge(highest_first))
}
fn env_of(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
move |name| {
values
.iter()
.find(|(key, _)| *key == name)
.map(|(_, value)| value.to_string())
}
fn from_env(values: &[(&str, &str)]) -> HttpKeysPartial {
HttpKeysPartial::from_env(&Env::of(values)).unwrap()
}
#[rstest]
@ -219,22 +201,22 @@ mod tests {
#[test]
fn higher_layers_override_lower_ones() {
let configured = HttpSettingsLayer {
let configured = HttpKeysPartial {
ssl_verify: Some(SslVerify::Enabled),
ssl_certificate: Some("/configured/client.pem".into()),
ssl_security_level: Some("configured".into()),
user_agent: Some("configured/1".into()),
..HttpSettingsLayer::default()
..HttpKeysPartial::default()
};
let environment = HttpSettingsLayer::from_environment(&env_of(&[
let environment = from_env(&[
("SSL_VERIFY", "false"),
("SSL_CERT_FILE", "/env/roots.pem"),
("SSL_CERTIFICATE", "/env/client.pem"),
("SSL_SECURITY_LEVEL", "DEFAULT@SECLEVEL=1"),
("SSL_ECDH_CURVE", "X25519"),
("LITELLM_USER_AGENT", "env/2"),
]));
let settings = HttpSettings::from_layers([environment, configured]);
]);
let settings = stack([environment, configured]);
assert_eq!(settings.ssl_verify, Some(SslVerify::Disabled));
assert_eq!(settings.ssl_cert_file, Some("/env/roots.pem".into()));
assert_eq!(settings.ssl_certificate, Some("/env/client.pem".into()));
@ -248,35 +230,32 @@ mod tests {
#[test]
fn an_explicit_false_in_a_higher_layer_beats_a_lower_true() {
let higher = HttpSettingsLayer {
let higher = HttpKeysPartial {
http2: Some(false),
force_ipv4: Some(false),
..HttpSettingsLayer::default()
..HttpKeysPartial::default()
};
let lower = HttpSettingsLayer {
let lower = HttpKeysPartial {
http2: Some(true),
force_ipv4: Some(true),
..HttpSettingsLayer::default()
..HttpKeysPartial::default()
};
let settings = HttpSettings::from_layers([higher, lower]);
let settings = stack([higher, lower]);
assert!(!settings.http2);
assert!(!settings.force_ipv4);
}
#[test]
fn an_empty_environment_is_an_empty_layer_so_lower_layers_and_defaults_apply() {
assert_eq!(
HttpSettingsLayer::from_environment(&no_env),
HttpSettingsLayer::default()
);
let configured = HttpSettingsLayer {
assert_eq!(from_env(&[]), HttpKeysPartial::default());
let configured = HttpKeysPartial {
ssl_verify: Some(SslVerify::CaBundle("/configured/roots.pem".into())),
http2: Some(true),
user_agent: Some("configured/1".into()),
..HttpSettingsLayer::default()
..HttpKeysPartial::default()
};
assert_eq!(
HttpSettings::from_layers([HttpSettingsLayer::default(), configured]),
stack([HttpKeysPartial::default(), configured]),
HttpSettings {
ssl_verify: Some(SslVerify::CaBundle("/configured/roots.pem".into())),
http2: true,
@ -284,35 +263,48 @@ mod tests {
..HttpSettings::default()
}
);
assert_eq!(HttpSettings::from_layers([]), HttpSettings::default());
assert_eq!(stack([]), HttpSettings::default());
}
#[test]
fn empty_certificate_is_retained_for_validation_while_empty_tuning_is_absent() {
let configured = HttpSettingsLayer {
fn blank_environment_values_fall_through_to_the_configured_settings() {
let configured = HttpKeysPartial {
ssl_certificate: Some("/configured/client.pem".into()),
ssl_security_level: Some("configured".into()),
ssl_ecdh_curve: Some("X25519".into()),
..HttpSettingsLayer::default()
..HttpKeysPartial::default()
};
let environment = HttpSettingsLayer::from_environment(&env_of(&[
let environment = from_env(&[
("SSL_CERTIFICATE", ""),
("SSL_SECURITY_LEVEL", ""),
("SSL_SECURITY_LEVEL", " "),
("SSL_ECDH_CURVE", ""),
]));
let settings = HttpSettings::from_layers([environment, configured]);
assert_eq!(settings.ssl_certificate, Some(PathBuf::new()));
assert_eq!(settings.ssl_security_level, None);
assert_eq!(settings.ssl_ecdh_curve, None);
]);
let settings = stack([environment, configured]);
assert_eq!(
settings.ssl_certificate,
Some(PathBuf::from("/configured/client.pem"))
);
assert_eq!(settings.ssl_security_level.as_deref(), Some("configured"));
assert_eq!(settings.ssl_ecdh_curve.as_deref(), Some("X25519"));
}
#[test]
fn an_empty_user_agent_environment_value_beats_the_configured_one() {
let configured = HttpKeysPartial {
user_agent: Some("configured/1".into()),
..HttpKeysPartial::default()
};
let settings = stack([from_env(&[("LITELLM_USER_AGENT", "")]), configured]);
assert_eq!(settings.user_agent.as_deref(), Some(""));
}
#[test]
fn socket_keepalive_follows_the_aiohttp_variables_with_python_defaults() {
let tuned = HttpSettings::from_layers([HttpSettingsLayer::from_environment(&env_of(&[
let tuned = stack([from_env(&[
("AIOHTTP_SO_KEEPALIVE", "True"),
("AIOHTTP_TCP_KEEPIDLE", "45"),
("AIOHTTP_KEEPALIVE_TIMEOUT", "30"),
]))]);
])]);
assert_eq!(
tuned.tcp_keepalive,
Some(TcpKeepalive {
@ -326,9 +318,7 @@ mod tests {
#[test]
fn socket_keepalive_stays_off_unless_enabled() {
let settings = HttpSettings::from_layers([HttpSettingsLayer::from_environment(&env_of(
&[("AIOHTTP_TCP_KEEPIDLE", "45")],
))]);
let settings = stack([from_env(&[("AIOHTTP_TCP_KEEPIDLE", "45")])]);
assert_eq!(settings.tcp_keepalive, None);
assert_eq!(settings.pool_idle_timeout, Duration::from_secs(15));
}
@ -338,13 +328,13 @@ mod tests {
disable_aiohttp_trust_env: bool,
disable_aiohttp_transport: bool,
http2: bool,
) -> HttpSettingsLayer {
HttpSettingsLayer {
) -> HttpKeysPartial {
HttpKeysPartial {
aiohttp_trust_env: Some(aiohttp_trust_env),
disable_aiohttp_trust_env: Some(disable_aiohttp_trust_env),
disable_aiohttp_transport: Some(disable_aiohttp_transport),
http2: Some(http2),
..HttpSettingsLayer::default()
..HttpKeysPartial::default()
}
}
@ -355,22 +345,21 @@ mod tests {
#[case::http2_uses_httpx(proxy_flags(false, true, false, true), true)]
#[case::aiohttp_disabled(proxy_flags(false, true, true, false), true)]
fn environment_proxies_apply_unless_the_aiohttp_transport_opts_out(
#[case] layer: HttpSettingsLayer,
#[case] layer: HttpKeysPartial,
#[case] expected: bool,
) {
assert_eq!(HttpSettings::from_layers([layer]).trust_proxy_env, expected);
assert_eq!(stack([layer]).trust_proxy_env, expected);
}
#[test]
fn a_proxy_opt_out_in_one_source_still_yields_to_trust_env_from_another() {
let environment =
HttpSettingsLayer::from_environment(&env_of(&[("DISABLE_AIOHTTP_TRUST_ENV", "true")]));
let configured = HttpSettingsLayer {
let environment = from_env(&[("DISABLE_AIOHTTP_TRUST_ENV", "true")]);
let configured = HttpKeysPartial {
aiohttp_trust_env: Some(true),
..HttpSettingsLayer::default()
..HttpKeysPartial::default()
};
assert!(!HttpSettings::from_layers([environment.clone()]).trust_proxy_env);
assert!(HttpSettings::from_layers([environment, configured]).trust_proxy_env);
assert!(!stack([environment.clone()]).trust_proxy_env);
assert!(stack([environment, configured]).trust_proxy_env);
}
#[test]
@ -411,7 +400,7 @@ mod tests {
| "DISABLE_AIOHTTP_TRUST_ENV" => Some(value.to_string()),
_ => None,
};
let layer = HttpSettingsLayer::from_environment(&env);
let layer = HttpKeysPartial::from_env(&env).unwrap();
assert_eq!(layer.http2, expected);
assert_eq!(layer.aiohttp_trust_env, expected);
assert_eq!(layer.disable_aiohttp_transport, expected);

View file

@ -10,6 +10,7 @@ publish = false
litellm-auth.workspace = true
futures-util.workspace = true
litellm-http.workspace = true
litellm-layered.workspace = true
litellm-inference.workspace = true
litellm-secrets.workspace = true
tokio.workspace = true

View file

@ -1,7 +1,8 @@
use std::{future::Future, sync::Arc, time::Duration};
use litellm_http::{ClientVariant, HttpSettings, HttpSettingsLayer, Resolution};
use litellm_http::{ClientVariant, HttpKeysPartial, HttpSettings, Resolution};
use litellm_inference::resources::CoreResources;
use litellm_layered::Env;
use litellm_secrets::source::{EnvironmentSecrets, SecretSource};
pub const DEADLINE: Duration = Duration::from_secs(120);
@ -15,10 +16,10 @@ pub struct LiveResources {
impl Default for LiveResources {
fn default() -> Self {
let resources = CoreResources::new(Arc::new(super::http_pool()));
let settings =
HttpSettings::from_layers([HttpSettingsLayer::from_environment(&|key: &str| {
std::env::var(key).ok()
})]);
let settings = HttpSettings::from(
HttpKeysPartial::from_env(&Env::from_process())
.expect("live HTTP environment variables must be valid"),
);
let resolution = Resolution::from(&settings);
let http = resources
.pool

View file

@ -0,0 +1,14 @@
[package]
name = "litellm-layered-derive"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[lib]
proc-macro = true
[dependencies]
proc-macro2.workspace = true
quote.workspace = true
syn = { workspace = true, features = ["derive", "full", "parsing", "printing", "proc-macro"] }

View file

@ -0,0 +1,281 @@
use proc_macro::TokenStream;
use proc_macro2::{Span, TokenStream as TokenStream2};
use quote::{format_ident, quote};
use syn::{
Data, DeriveInput, Expr, ExprArray, Fields, Ident, Lit, LitStr, Path, Type, Visibility,
meta::ParseNestedMeta, parse_macro_input,
};
/// `litellm_setting = "x"`: `litellm.x` in Python and `litellm_settings.x` in config.yaml, which the proxy copies with `setattr`.
#[proc_macro_derive(Layered, attributes(layer))]
pub fn derive_layered(input: TokenStream) -> TokenStream {
let input = parse_macro_input!(input as DeriveInput);
expand(&input)
.unwrap_or_else(syn::Error::into_compile_error)
.into()
}
#[derive(Default)]
struct FieldAttributes {
env: Vec<LitStr>,
read: Option<Expr>,
from_env: Option<Path>,
litellm_param: Vec<LitStr>,
deserialize_with: Option<Path>,
litellm_setting: Option<LitStr>,
secret: bool,
keep_blank: bool,
setting: Option<LitStr>,
}
struct LayeredField {
vis: Visibility,
ident: Ident,
ty: Type,
attributes: FieldAttributes,
}
fn expand(input: &DeriveInput) -> syn::Result<TokenStream2> {
let Data::Struct(data) = &input.data else {
return Err(syn::Error::new_spanned(input, "Layered needs a struct"));
};
let Fields::Named(named) = &data.fields else {
return Err(syn::Error::new_spanned(input, "Layered needs named fields"));
};
let fields = named
.named
.iter()
.map(|field| {
Ok(LayeredField {
vis: field.vis.clone(),
ident: field.ident.clone().expect("named field"),
ty: field.ty.clone(),
attributes: field_attributes(field)?,
})
})
.collect::<syn::Result<Vec<_>>>()?;
let name = &input.ident;
let vis = &input.vis;
let partial = format_ident!("{name}Partial");
let layered = quote!(::litellm_layered);
let private = quote!(#layered::__private);
let partial_fields = fields.iter().map(|field| {
let LayeredField { vis, ident, ty, .. } = field;
quote!(#vis #ident: ::core::option::Option<#ty>)
});
let debug_fields = fields.iter().map(|field| {
let ident = &field.ident;
let label = ident.to_string();
if field.attributes.secret {
quote!(.field(#label, &#private::Redacted(&self.#ident)))
} else {
quote!(.field(#label, &self.#ident))
}
});
let or_fields = fields.iter().map(|field| {
let ident = &field.ident;
quote!(#ident: self.#ident.or(lower.#ident))
});
let env_fields = fields.iter().map(|field| {
let ident = &field.ident;
let attributes = &field.attributes;
let keep_blank = attributes.keep_blank;
if let Some(from_env) = &attributes.from_env {
return quote!(#ident: #from_env(env)?);
}
if attributes.env.is_empty() {
return quote!(#ident: ::core::option::Option::None);
}
let names = &attributes.env;
let read = attributes
.read
.clone()
.map_or_else(|| quote!(#layered::read::parsed), |read| quote!(#read));
quote!(#ident: #layered::read_env(env, &[#(#names),*], #keep_blank, #read)?)
});
let param_fields = fields.iter().map(|field| {
let LayeredField { ident, ty, .. } = field;
let attributes = &field.attributes;
let keep_blank = attributes.keep_blank;
if attributes.litellm_param.is_empty() {
return quote!(#ident: ::core::option::Option::None);
}
let names = &attributes.litellm_param;
let decode = match &attributes.deserialize_with {
Some(path) => quote!(|value| #path(value).ok().flatten()),
None => quote!(|value| <#ty as #private::serde::Deserialize>::deserialize(value).ok()),
};
quote!(#ident: #private::from_litellm_params(params, &[#(#names),*], #keep_blank, #decode))
});
let sources = fields.iter().map(|field| {
let attributes = &field.attributes;
let setting = attributes.setting.clone().unwrap_or_else(|| {
LitStr::new(
&field.ident.to_string().replace('_', " "),
Span::call_site(),
)
});
let params = &attributes.litellm_param;
let env = &attributes.env;
let litellm_setting = match &attributes.litellm_setting {
Some(name) => quote!(::core::option::Option::Some(#name)),
None => quote!(::core::option::Option::None),
};
quote!(#layered::Sources {
setting: #setting,
litellm_params: &[#(#params),*],
litellm_setting: #litellm_setting,
env: &[#(#env),*],
})
});
let source_count = fields.len();
let source_refs = fields.iter().enumerate().map(|(index, field)| {
let constant = Ident::new(&field.ident.to_string().to_uppercase(), field.ident.span());
quote!(pub const #constant: &'static #layered::Sources = &Self::SOURCES[#index];)
});
let litellm_params = fields
.iter()
.flat_map(|field| field.attributes.litellm_param.iter());
let litellm_settings = fields.iter().filter_map(|field| {
let setting = field.attributes.litellm_setting.as_ref()?;
let label = field.ident.to_string();
Some(quote!((#setting, #label)))
});
let env_names = unique(
fields
.iter()
.flat_map(|field| field.attributes.env.iter().cloned()),
);
Ok(quote! {
#[derive(::core::clone::Clone, ::core::default::Default, ::core::cmp::PartialEq)]
#vis struct #partial {
#(#partial_fields,)*
}
impl ::core::fmt::Debug for #partial {
fn fmt(&self, formatter: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
formatter.debug_struct(stringify!(#partial))
#(#debug_fields)*
.finish()
}
}
impl #layered::Partial for #partial {
fn or(self, lower: Self) -> Self {
Self { #(#or_fields,)* }
}
}
impl #partial {
pub fn from_env(
env: &(impl #layered::Lookup + ?::core::marker::Sized),
) -> ::core::result::Result<Self, #layered::Error> {
::core::result::Result::Ok(Self { #(#env_fields,)* })
}
pub fn from_litellm_params(
params: &#private::serde_json::Map<::std::string::String, #private::serde_json::Value>,
) -> Self {
Self { #(#param_fields,)* }
}
}
impl #name {
pub const SOURCES: [#layered::Sources; #source_count] = [#(#sources),*];
pub const LITELLM_PARAMS: &'static [&'static str] = &[#(#litellm_params),*];
pub const LITELLM_SETTINGS: &'static [(&'static str, &'static str)] = &[#(#litellm_settings),*];
pub const ENV_NAMES: &'static [&'static str] = &[#(#env_names),*];
#(#source_refs)*
}
})
}
fn unique(names: impl Iterator<Item = LitStr>) -> Vec<LitStr> {
names.fold(Vec::new(), |mut seen, name| {
if !seen
.iter()
.any(|known: &LitStr| known.value() == name.value())
{
seen.push(name);
}
seen
})
}
fn field_attributes(field: &syn::Field) -> syn::Result<FieldAttributes> {
let mut attributes = FieldAttributes::default();
for attribute in field
.attrs
.iter()
.filter(|attribute| attribute.path().is_ident("layer"))
{
attribute.parse_nested_meta(|meta| {
let path = &meta.path;
if path.is_ident("env") {
attributes.env = names(&meta)?;
} else if path.is_ident("litellm_param") {
attributes.litellm_param = names(&meta)?;
} else if path.is_ident("read") {
attributes.read = Some(meta.value()?.parse()?);
} else if path.is_ident("from_env") {
attributes.from_env = Some(meta.value()?.parse()?);
} else if path.is_ident("deserialize_with") {
attributes.deserialize_with = Some(meta.value()?.parse()?);
} else if path.is_ident("litellm_setting") {
attributes.litellm_setting = Some(meta.value()?.parse()?);
} else if path.is_ident("setting") {
attributes.setting = Some(meta.value()?.parse()?);
} else if path.is_ident("secret") {
attributes.secret = true;
} else if path.is_ident("keep_blank") {
attributes.keep_blank = true;
} else {
return Err(meta.error("unknown `layer` attribute"));
}
Ok(())
})?;
}
if attributes.from_env.is_some() && (!attributes.env.is_empty() || attributes.read.is_some()) {
return Err(syn::Error::new_spanned(
field,
"`from_env` reads the environment itself, so it cannot be combined with `env` or `read`",
));
}
if attributes.read.is_some() && attributes.env.is_empty() {
return Err(syn::Error::new_spanned(field, "`read` needs `env`"));
}
if attributes.deserialize_with.is_some() && attributes.litellm_param.is_empty() {
return Err(syn::Error::new_spanned(
field,
"`deserialize_with` needs `litellm_param`",
));
}
Ok(attributes)
}
fn names(meta: &ParseNestedMeta<'_>) -> syn::Result<Vec<LitStr>> {
match meta.value()?.parse::<Expr>()? {
Expr::Lit(syn::ExprLit {
lit: Lit::Str(name),
..
}) => Ok(vec![name]),
Expr::Array(ExprArray { elems, .. }) => elems
.into_iter()
.map(|element| match element {
Expr::Lit(syn::ExprLit {
lit: Lit::Str(name),
..
}) => Ok(name),
other => Err(syn::Error::new_spanned(other, "expected a string literal")),
})
.collect(),
other => Err(syn::Error::new_spanned(
other,
"expected a string literal or an array of them",
)),
}
}

View file

@ -0,0 +1,27 @@
# layered
`#[derive(Layered)]` declares where each setting of a group comes from. The derive generates `<Name>Partial`, with `from_env` and `from_litellm_params`, and the name lists and `Sources` rows for errors and projection. `merge` stacks partials highest first, and the group's crate lowers the merged partial into its final type, where defaults live
## Runtimes stack different sources
| Runtime | Stack, highest first | Where config.yaml shows up |
|---|---|---|
| Python SDK | litellm params, env, `litellm.*` globals, defaults | nowhere |
| Python proxy | litellm params, env, `litellm.*` globals, defaults | already copied into env and globals by the proxy |
| Rust gateway | litellm params, env, config file, defaults | read by the gateway |
Each runtime writes its own `merge([..])` list per group, and a test per runtime pins it. Credentials put litellm params first and the global moves per provider, so never assume one order across groups
## Who builds which partial
- `from_env` and `from_litellm_params` are generated, and every runtime that has that source calls them
- A source holding typed values (the config file, a live Python global) gets a hand-written constructor in the crate that owns the source, never in `layered` or in the group's crate
- `layered` never depends on `litellm-config` or pyo3, and no library crate reads the process environment itself. It takes a `Lookup`: `Env` in the Rust gateway, `ProcessEnvironment` in `python-bridge`
## Hand-written constructors list every field
A hand-written constructor names every field of the partial and writes `None` where its source has no such key, never `..Default::default()`. Adding a field then fails to compile in each constructor until someone decides what that source does with it
## Blank means unset
An empty or whitespace-only env value or string litellm param counts as unset, so the lookup moves on to the next name and the next layer. Mark a field `keep_blank` only when an empty value is a deliberate choice, like `LITELLM_USER_AGENT=""` sending no user agent. Python's `os.getenv(X, litellm.x)` lets an empty env var shadow the global for the SSL settings, and we diverge from that on purpose

View file

@ -0,0 +1,16 @@
[package]
name = "litellm-layered"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
litellm-core-utils.workspace = true
litellm-layered-derive.workspace = true
serde.workspace = true
serde_json.workspace = true
thiserror.workspace = true
[dev-dependencies]
rstest.workspace = true

View file

@ -0,0 +1,70 @@
use std::{collections::BTreeMap, fmt};
use crate::Lookup;
#[derive(Clone, Default, PartialEq, Eq)]
pub struct Env(BTreeMap<String, String>);
impl Env {
#[allow(clippy::disallowed_methods)]
pub fn from_process() -> Self {
Self(
std::env::vars_os()
.filter_map(|(name, value)| {
Some((name.into_string().ok()?, value.into_string().ok()?))
})
.collect(),
)
}
pub fn overlaid(self, values: impl IntoIterator<Item = (String, String)>) -> Self {
Self(self.0.into_iter().chain(values).collect())
}
pub fn of(values: &[(&str, &str)]) -> Self {
Self(
values
.iter()
.map(|(name, value)| ((*name).to_owned(), (*value).to_owned()))
.collect(),
)
}
}
impl Lookup for Env {
fn get(&self, name: &str) -> Option<String> {
self.0.get(name).cloned()
}
}
impl fmt::Debug for Env {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_tuple("Env")
.field(&self.0.keys().collect::<Vec<_>>())
.finish()
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
#[rstest]
fn overlaid_values_win_over_the_snapshot() {
let env = Env::of(&[("SHARED", "process"), ("PROCESS_ONLY", "process")])
.overlaid([("SHARED".to_owned(), "config".to_owned())]);
assert_eq!(env.get("SHARED").as_deref(), Some("config"));
assert_eq!(env.get("PROCESS_ONLY").as_deref(), Some("process"));
assert_eq!(env.get("ABSENT"), None);
}
#[rstest]
fn debug_lists_names_without_values() {
let debug = format!("{:?}", Env::of(&[("API_KEY", "sk-hidden")]));
assert!(debug.contains("API_KEY"));
assert!(!debug.contains("sk-hidden"));
}
}

View file

@ -0,0 +1,18 @@
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
#[error("expected {expected}")]
pub struct Invalid {
expected: &'static str,
}
impl Invalid {
pub fn expected(expected: &'static str) -> Self {
Self { expected }
}
}
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
#[error("invalid environment variable {name}: {invalid}")]
pub struct Error {
pub name: &'static str,
pub invalid: Invalid,
}

View file

@ -0,0 +1,79 @@
mod env;
mod error;
pub mod read;
mod sources;
pub use env::Env;
pub use error::{Error, Invalid};
pub use litellm_core_utils::settings::Lookup;
pub use litellm_layered_derive::Layered;
pub use sources::Sources;
pub trait Partial: Default {
fn or(self, lower: Self) -> Self;
}
pub fn merge<P: Partial>(highest_precedence_first: impl IntoIterator<Item = P>) -> P {
highest_precedence_first
.into_iter()
.reduce(P::or)
.unwrap_or_default()
}
pub fn read_env<T>(
env: &(impl Lookup + ?Sized),
names: &'static [&'static str],
keep_blank: bool,
read: impl Fn(&str) -> Result<Option<T>, Invalid>,
) -> Result<Option<T>, Error> {
for name in names {
let Some(value) = env.get(name) else {
continue;
};
if !keep_blank && value.trim().is_empty() {
continue;
}
if let Some(value) = read(&value).map_err(|invalid| Error { name, invalid })? {
return Ok(Some(value));
}
}
Ok(None)
}
#[doc(hidden)]
pub mod __private {
use std::fmt;
use serde_json::{Map, Value};
pub use serde;
pub use serde_json;
pub fn from_litellm_params<T>(
params: &Map<String, Value>,
names: &[&str],
keep_blank: bool,
decode: impl Fn(&Value) -> Option<T>,
) -> Option<T> {
names
.iter()
.filter_map(|name| params.get(*name))
.filter(|value| keep_blank || !is_blank(value))
.find_map(decode)
}
fn is_blank(value: &Value) -> bool {
value.as_str().is_some_and(|text| text.trim().is_empty())
}
pub struct Redacted<'a, T>(pub &'a Option<T>);
impl<T> fmt::Debug for Redacted<'_, T> {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self.0 {
Some(_) => formatter.write_str("Some(\"[REDACTED]\")"),
None => formatter.write_str("None"),
}
}
}
}

View file

@ -0,0 +1,90 @@
use std::{any::type_name, str::FromStr, time::Duration};
use litellm_core_utils::serde_compat::parse_str_bool;
use crate::Invalid;
pub fn text<T: for<'a> From<&'a str>>(value: &str) -> Result<Option<T>, Invalid> {
Ok(Some(T::from(value)))
}
pub fn truthy<T: for<'a> From<&'a str>>(value: &str) -> Result<Option<T>, Invalid> {
Ok((!value.is_empty()).then(|| T::from(value)))
}
pub fn enabled(value: &str) -> Result<Option<bool>, Invalid> {
Ok((parse_str_bool(value) == Some(true)).then_some(true))
}
pub fn parsed<T: FromStr>(value: &str) -> Result<Option<T>, Invalid> {
Ok(value.trim().parse().ok())
}
pub fn required<T: FromStr>(value: &str) -> Result<Option<T>, Invalid> {
value
.trim()
.parse()
.map(Some)
.map_err(|_| Invalid::expected(type_name::<T>()))
}
pub fn seconds(value: &str) -> Result<Option<Duration>, Invalid> {
Ok(parsed::<u32>(value)?.map(|seconds| Duration::from_secs(u64::from(seconds))))
}
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use rstest::rstest;
use super::*;
#[rstest]
#[case::lower("true", Some(true))]
#[case::padded(" True ", Some(true))]
#[case::off("false", None)]
#[case::one("1", None)]
fn enabled_only_switches_on_and_never_forces_off(
#[case] value: &str,
#[case] expected: Option<bool>,
) {
assert_eq!(enabled(value), Ok(expected));
}
#[rstest]
#[case::empty("", None)]
#[case::kept_untrimmed(" value ", Some(" value "))]
fn truthy_drops_only_empty_values(#[case] value: &str, #[case] expected: Option<&str>) {
assert_eq!(truthy::<String>(value).unwrap().as_deref(), expected);
}
#[rstest]
fn text_keeps_the_value_as_written() {
assert_eq!(text::<PathBuf>(" /a b "), Ok(Some(PathBuf::from(" /a b "))));
}
#[rstest]
#[case::padded(" 45 ", Some(45))]
#[case::word("soon", None)]
#[case::fraction("0.5", None)]
fn parsed_trims_and_treats_junk_as_unset(#[case] value: &str, #[case] expected: Option<u32>) {
assert_eq!(parsed::<u32>(value), Ok(expected));
}
#[rstest]
#[case::padded(" 8080 ", Ok(Some(8080)))]
#[case::word("http", Err(Invalid::expected("u16")))]
#[case::empty("", Err(Invalid::expected("u16")))]
#[case::negative("-1", Err(Invalid::expected("u16")))]
fn required_fails_on_junk(#[case] value: &str, #[case] expected: Result<Option<u16>, Invalid>) {
assert_eq!(required::<u16>(value), expected);
}
#[rstest]
#[case::whole("45", Some(Duration::from_secs(45)))]
#[case::junk("later", None)]
fn seconds_reads_whole_seconds(#[case] value: &str, #[case] expected: Option<Duration>) {
assert_eq!(seconds(value), Ok(expected));
}
}

View file

@ -0,0 +1,26 @@
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Sources {
pub setting: &'static str,
pub litellm_params: &'static [&'static str],
pub litellm_setting: Option<&'static str>,
pub env: &'static [&'static str],
}
impl Sources {
pub fn guidance(&self) -> String {
let steps: Vec<String> = [
(!self.litellm_params.is_empty())
.then(|| format!("pass {}", self.litellm_params.join(" or "))),
self.litellm_setting
.map(|setting| format!("set litellm.{setting}")),
(!self.env.is_empty()).then(|| format!("set {}", self.env.join(" or "))),
]
.into_iter()
.flatten()
.collect();
match steps.as_slice() {
[first, second, third] => format!("{first}, {second}, or {third}"),
steps => steps.join(" or "),
}
}
}

View file

@ -0,0 +1,121 @@
use std::time::Duration;
use litellm_layered::{Env, Error, Layered, Lookup, read, read_env};
use rstest::rstest;
use serde_json::{Value, json};
#[derive(Layered)]
#[expect(dead_code, reason = "only the derived partial is built")]
struct Settings {
#[layer(env = ["FIRST_NAME", "SECOND_NAME"], read = read::text)]
name: String,
#[layer(env = "SWITCH", read = read::enabled)]
switch: bool,
#[layer(env = "COUNT", read = read::parsed::<u32>)]
count: u32,
#[layer(env = "PORT", read = read::required::<u16>)]
port: u16,
#[layer(env = "IDLE", read = read::seconds)]
idle: Duration,
#[layer(from_env = window_from_env)]
window: (u32, u32),
#[layer(litellm_param = "label", env = "LABEL", read = read::text, keep_blank)]
label: String,
untouched_by_env: bool,
}
fn window_from_env(env: &(impl Lookup + ?Sized)) -> Result<Option<(u32, u32)>, Error> {
let start = read_env(env, &["WINDOW_START"], false, read::required::<u32>)?;
let end = read_env(env, &["WINDOW_END"], false, read::required::<u32>)?;
Ok(start.zip(end))
}
fn params(value: Value) -> serde_json::Map<String, Value> {
serde_json::from_value(value).unwrap()
}
#[rstest]
fn each_field_reads_its_env_names_in_order() {
let partial = SettingsPartial::from_env(&Env::of(&[
("SECOND_NAME", "second"),
("SWITCH", "True"),
("COUNT", " 7 "),
("PORT", "8080"),
("IDLE", "45"),
("WINDOW_START", "1"),
("WINDOW_END", "2"),
]))
.unwrap();
assert_eq!(
partial,
SettingsPartial {
name: Some("second".into()),
switch: Some(true),
count: Some(7),
port: Some(8080),
idle: Some(Duration::from_secs(45)),
window: Some((1, 2)),
label: None,
untouched_by_env: None,
}
);
}
#[rstest]
#[case::empty("")]
#[case::whitespace(" ")]
fn blank_values_fall_through_to_the_next_name(#[case] blank: &str) {
let partial = SettingsPartial::from_env(&Env::of(&[
("FIRST_NAME", blank),
("SECOND_NAME", "second"),
]))
.unwrap();
assert_eq!(partial.name.as_deref(), Some("second"));
}
#[rstest]
fn keep_blank_fields_count_a_blank_value_as_set() {
assert_eq!(
SettingsPartial::from_env(&Env::of(&[("LABEL", "")]))
.unwrap()
.label
.as_deref(),
Some("")
);
assert_eq!(
SettingsPartial::from_litellm_params(&params(json!({"label": " "})))
.label
.as_deref(),
Some(" ")
);
}
#[rstest]
fn a_field_without_env_never_comes_from_env() {
let partial = SettingsPartial::from_env(&Env::of(&[("UNTOUCHED_BY_ENV", "true")])).unwrap();
assert_eq!(partial, SettingsPartial::default());
}
#[rstest]
#[case::switch_off("SWITCH", "false")]
#[case::switch_one("SWITCH", "1")]
#[case::lax_count("COUNT", "many")]
fn lax_values_that_do_not_read_are_unset(#[case] name: &str, #[case] value: &str) {
assert_eq!(
SettingsPartial::from_env(&Env::of(&[(name, value)])).unwrap(),
SettingsPartial::default()
);
}
#[rstest]
#[case::port("PORT", "abc")]
#[case::port_overflow("PORT", "65536")]
#[case::from_env_field("WINDOW_START", "soon")]
fn strict_values_that_do_not_read_fail_naming_the_variable(
#[case] name: &'static str,
#[case] value: &str,
) {
let error = SettingsPartial::from_env(&Env::of(&[(name, value)])).unwrap_err();
assert_eq!(error.name, name);
assert!(error.to_string().contains(name));
}

View file

@ -0,0 +1,137 @@
use litellm_layered::{Layered, Sources, merge, read};
use rstest::rstest;
use serde::{Deserialize, Deserializer};
use serde_json::{Value, json};
#[derive(Layered)]
#[expect(dead_code, reason = "only the derived partial is built")]
struct Credentials {
#[layer(
litellm_param = ["project", "legacy_project"],
litellm_setting = "credential_project",
env = ["PROJECT", "PROJECT_FALLBACK"],
read = read::text,
)]
project: String,
#[layer(litellm_param = "token", env = "TOKEN", read = read::text, secret, setting = "API token")]
token: String,
#[layer(litellm_param = "document", deserialize_with = document_text)]
document: String,
#[layer(env = "LIMIT")]
limit: u32,
}
fn document_text<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Option<String>, D::Error> {
Ok(match Value::deserialize(deserializer)? {
Value::String(text) => Some(text),
Value::Object(object) => Some(Value::Object(object).to_string()),
_ => None,
})
}
fn params(value: Value) -> serde_json::Map<String, Value> {
serde_json::from_value(value).unwrap()
}
#[rstest]
#[case::first_alias(json!({"project": "first", "legacy_project": "second"}), Some("first"))]
#[case::second_alias(json!({"legacy_project": "second"}), Some("second"))]
#[case::blank_falls_to_alias(json!({"project": " ", "legacy_project": "second"}), Some("second"))]
#[case::null_falls_to_alias(json!({"project": null, "legacy_project": "second"}), Some("second"))]
#[case::non_string_is_ignored(json!({"project": 5}), None)]
#[case::absent(json!({}), None)]
fn litellm_params_read_aliases_in_order(#[case] value: Value, #[case] expected: Option<&str>) {
assert_eq!(
CredentialsPartial::from_litellm_params(&params(value))
.project
.as_deref(),
expected
);
}
#[rstest]
fn litellm_params_use_the_field_decoder() {
let partial = CredentialsPartial::from_litellm_params(&params(json!({
"document": {"type": "service_account"},
"limit": 5,
})));
assert_eq!(
partial.document.as_deref(),
Some(r#"{"type":"service_account"}"#)
);
assert_eq!(partial.limit, None);
}
#[rstest]
fn merge_takes_each_field_from_the_highest_partial_that_sets_it() {
let merged = merge([
CredentialsPartial {
project: Some("params".into()),
..CredentialsPartial::default()
},
CredentialsPartial {
project: Some("env".into()),
token: Some("env-token".into()),
..CredentialsPartial::default()
},
]);
assert_eq!(merged.project.as_deref(), Some("params"));
assert_eq!(merged.token.as_deref(), Some("env-token"));
assert_eq!(
merge(Vec::<CredentialsPartial>::new()),
CredentialsPartial::default()
);
}
#[rstest]
fn debug_redacts_secret_fields() {
let debug = format!(
"{:?}",
CredentialsPartial {
project: Some("visible-project".into()),
token: Some("hidden-token".into()),
..CredentialsPartial::default()
}
);
assert!(debug.contains("visible-project"));
assert!(!debug.contains("hidden-token"));
assert!(debug.contains("REDACTED"));
}
#[rstest]
fn name_lists_follow_the_declaration() {
assert_eq!(
Credentials::LITELLM_PARAMS,
["project", "legacy_project", "token", "document"]
);
assert_eq!(
Credentials::LITELLM_SETTINGS,
[("credential_project", "project")]
);
assert_eq!(
Credentials::ENV_NAMES,
["PROJECT", "PROJECT_FALLBACK", "TOKEN", "LIMIT"]
);
assert_eq!(
Credentials::PROJECT,
&Sources {
setting: "project",
litellm_params: &["project", "legacy_project"],
litellm_setting: Some("credential_project"),
env: &["PROJECT", "PROJECT_FALLBACK"],
}
);
assert_eq!(Credentials::SOURCES[1].setting, "API token");
}
#[rstest]
#[case::every_source(
Credentials::PROJECT,
"pass project or legacy_project, set litellm.credential_project, or set PROJECT or PROJECT_FALLBACK"
)]
#[case::params_and_env(Credentials::TOKEN, "pass token or set TOKEN")]
#[case::params_only(Credentials::DOCUMENT, "pass document")]
#[case::env_only(Credentials::LIMIT, "set LIMIT")]
fn guidance_names_exactly_the_declared_sources(#[case] sources: &Sources, #[case] expected: &str) {
assert_eq!(sources.guidance(), expected);
}

View file

@ -42,6 +42,7 @@ litellm-inference-chat.workspace = true
litellm-inference-ocr.workspace = true
litellm-core-utils.workspace = true
litellm-http.workspace = true
litellm-layered.workspace = true
litellm-llms.workspace = true
litellm-secrets = { workspace = true, features = ["aws", "azure", "google", "hashicorp", "cyberark"] }
litellm-secrets-types.workspace = true

View file

@ -6,10 +6,11 @@ use std::{
use litellm_core_utils::settings::ProcessEnvironment;
use litellm_http::{
Client, ClientVariant, HttpClientConfig, HttpClientPool, HttpSettings, HttpSettingsLayer,
Client, ClientVariant, HttpClientConfig, HttpClientPool, HttpKeysPartial, HttpSettings,
Resolution, SslVerify, TlsSource, Unsupported,
media::{PublicDnsResolver, UrlPolicy},
};
use litellm_layered::merge;
use pyo3::{
exceptions::PyValueError,
prelude::*,
@ -101,14 +102,15 @@ pub(crate) fn call_config(
kwargs: &Bound<'_, PyDict>,
asynchronous: bool,
) -> PyResult<HttpClientConfig> {
let settings = HttpSettings::from_layers([
let settings = HttpSettings::from(merge([
for_call(call_ssl_verify(kwargs)?, asynchronous),
HttpSettingsLayer::from_environment(&ProcessEnvironment),
HttpKeysPartial::from_env(&ProcessEnvironment)
.map_err(|error| PyValueError::new_err(error.to_string()))?,
match PythonSettings::Http.read_or_unset(py)? {
Some(snapshot) => configured(&snapshot)?,
None => HttpSettingsLayer::default(),
None => HttpKeysPartial::default(),
},
])
]))
.without_missing_files(&|path: &Path| path.exists());
let resolution = Resolution::from(&settings);
for unsupported in unreported(&REPORTED_UNSUPPORTED, resolution.unsupported) {
@ -191,17 +193,29 @@ fn call_ssl_verify(kwargs: &Bound<'_, PyDict>) -> PyResult<Option<SslVerify>> {
}
}
fn for_call(call_ssl_verify: Option<SslVerify>, asynchronous: bool) -> HttpSettingsLayer {
HttpSettingsLayer {
fn for_call(call_ssl_verify: Option<SslVerify>, asynchronous: bool) -> HttpKeysPartial {
HttpKeysPartial {
ssl_verify: call_ssl_verify,
ssl_cert_file: None,
ssl_certificate: None,
ssl_security_level: None,
ssl_ecdh_curve: None,
force_ipv4: None,
http2: None,
aiohttp_trust_env: None,
disable_aiohttp_trust_env: None,
disable_aiohttp_transport: (!asynchronous).then_some(true),
..HttpSettingsLayer::default()
user_agent: None,
tcp_keepalive: None,
pool_idle_timeout: None,
proxies: None,
}
}
fn configured(snapshot: &Snapshot<'_>) -> PyResult<HttpSettingsLayer> {
Ok(HttpSettingsLayer {
fn configured(snapshot: &Snapshot<'_>) -> PyResult<HttpKeysPartial> {
Ok(HttpKeysPartial {
ssl_verify: snapshot.read(&SSL_VERIFY)?,
ssl_cert_file: None,
ssl_certificate: snapshot.read(&SSL_CERTIFICATE)?.map(PathBuf::from),
ssl_security_level: snapshot.read(&SSL_SECURITY_LEVEL)?,
ssl_ecdh_curve: snapshot.read(&SSL_ECDH_CURVE)?,
@ -211,7 +225,9 @@ fn configured(snapshot: &Snapshot<'_>) -> PyResult<HttpSettingsLayer> {
disable_aiohttp_trust_env: Some(snapshot.read(&DISABLE_AIOHTTP_TRUST_ENV)?),
disable_aiohttp_transport: Some(snapshot.read(&DISABLE_AIOHTTP_TRANSPORT)?),
user_agent: Some(snapshot.read(&USER_AGENT)?),
..HttpSettingsLayer::default()
tcp_keepalive: None,
pool_idle_timeout: None,
proxies: None,
})
}
@ -221,8 +237,14 @@ mod tests {
use pyo3::exceptions::PyRuntimeError;
use rstest::rstest;
use litellm_layered::Env;
use super::*;
fn stack(highest_first: impl IntoIterator<Item = HttpKeysPartial>) -> HttpSettings {
HttpSettings::from(merge(highest_first))
}
fn evaluate<'py>(py: Python<'py>, source: &str) -> Bound<'py, PyAny> {
py.eval(&std::ffi::CString::new(source).unwrap(), None, None)
.unwrap()
@ -260,7 +282,7 @@ settings = types.SimpleNamespace(**defaults)
Python::attach(|py| {
let layer = configured(&python_settings(py, "")).unwrap();
assert_eq!(
HttpSettings::from_layers([layer]),
stack([layer]),
HttpSettings {
ssl_verify: Some(SslVerify::Enabled),
user_agent: Some("litellm/test".into()),
@ -319,7 +341,7 @@ user_agent='litellm/9.9.9',
.unwrap();
assert_eq!(
layer,
HttpSettingsLayer {
HttpKeysPartial {
ssl_verify: Some(SslVerify::CaBundle("/etc/ssl/corp.pem".into())),
ssl_certificate: Some("/etc/ssl/client.pem".into()),
ssl_security_level: Some("2".into()),
@ -330,7 +352,7 @@ user_agent='litellm/9.9.9',
disable_aiohttp_trust_env: Some(true),
disable_aiohttp_transport: Some(true),
user_agent: Some("litellm/9.9.9".into()),
..HttpSettingsLayer::default()
..HttpKeysPartial::default()
}
);
});
@ -340,10 +362,9 @@ user_agent='litellm/9.9.9',
fn user_agent_environment_variable_beats_the_python_default() {
Python::initialize();
Python::attach(|py| {
let settings = HttpSettings::from_layers([
HttpSettingsLayer::from_environment(&|name: &str| {
(name == "LITELLM_USER_AGENT").then(|| "operator/1".to_string())
}),
let settings = stack([
HttpKeysPartial::from_env(&Env::of(&[("LITELLM_USER_AGENT", "operator/1")]))
.unwrap(),
configured(&python_settings(py, "")).unwrap(),
]);
assert_eq!(settings.user_agent.as_deref(), Some("operator/1"));
@ -362,7 +383,7 @@ user_agent='litellm/9.9.9',
Python::initialize();
Python::attach(|py| {
let layer = configured(&python_settings(py, overrides)).unwrap();
let config = Resolution::from(&HttpSettings::from_layers([layer])).config;
let config = Resolution::from(&stack([layer])).config;
assert_eq!(config.verify, expected);
});
}
@ -411,10 +432,10 @@ user_agent='litellm/9.9.9',
});
}
fn configured_ssl_verify(ssl_verify: SslVerify) -> HttpSettingsLayer {
HttpSettingsLayer {
fn configured_ssl_verify(ssl_verify: SslVerify) -> HttpKeysPartial {
HttpKeysPartial {
ssl_verify: Some(ssl_verify),
..HttpSettingsLayer::default()
..HttpKeysPartial::default()
}
}
@ -425,8 +446,7 @@ user_agent='litellm/9.9.9',
let kwargs = PyDict::new(py);
kwargs.set_item("ssl_verify", false).unwrap();
let call = for_call(call_ssl_verify(&kwargs).unwrap(), true);
let settings =
HttpSettings::from_layers([call, configured_ssl_verify(SslVerify::Enabled)]);
let settings = stack([call, configured_ssl_verify(SslVerify::Enabled)]);
assert_eq!(settings.ssl_verify, Some(SslVerify::Disabled));
});
}
@ -438,8 +458,7 @@ user_agent='litellm/9.9.9',
let kwargs = PyDict::new(py);
kwargs.set_item("ssl_verify", py.None()).unwrap();
let call = for_call(call_ssl_verify(&kwargs).unwrap(), true);
let settings =
HttpSettings::from_layers([call, configured_ssl_verify(SslVerify::Disabled)]);
let settings = stack([call, configured_ssl_verify(SslVerify::Disabled)]);
assert_eq!(settings.ssl_verify, Some(SslVerify::Disabled));
});
}
@ -485,12 +504,12 @@ user_agent='litellm/9.9.9',
#[case] asynchronous: bool,
#[case] expected: bool,
) {
let opted_out = HttpSettingsLayer {
let opted_out = HttpKeysPartial {
disable_aiohttp_trust_env: Some(true),
disable_aiohttp_transport: Some(false),
..HttpSettingsLayer::default()
..HttpKeysPartial::default()
};
let settings = HttpSettings::from_layers([for_call(None, asynchronous), opted_out]);
let settings = stack([for_call(None, asynchronous), opted_out]);
assert_eq!(settings.trust_proxy_env, expected);
}
#[rstest]