fix(gateway): read boot, OCR and config env settings through one injected Env
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled

The gateway takes one Env snapshot, overlays config environment_variables on it and injects it into every reader, so OCR REQUEST_TIMEOUT, Vertex credentials, Messages thinking budgets and AWS credentials now see config env vars. Boot and UI settings move to Layered and envy is dropped. clippy.toml bans std::env::var

ProcessEnvironment is private to python-bridge, so library constructors take the lookup as a required argument instead of defaulting to the process (MessagesTransformContext, RedisCache, EnvironmentSecrets, OcrConnection). auth-aws reads every env-backed credential source through the lookup before handing the rest to the SDK chain, so the assume-role shortcut and the credentials it returns agree

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yujong Lee 2026-10-26 09:54:31 -07:00
parent 9e3f422312
commit b61449e35d
78 changed files with 891 additions and 441 deletions

View file

@ -1911,15 +1911,6 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "envy"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f47e0157f2cb54f5ae1bd371b30a2ae4311e1c028f575cd4e81de7353215965"
dependencies = [
"serde",
]
[[package]]
name = "equivalent"
version = "1.0.2"
@ -3381,9 +3372,11 @@ dependencies = [
"serde",
"serde_json",
"sha2 0.10.9",
"tempfile",
"thiserror 2.0.19",
"tokio",
"veil",
"wiremock",
]
[[package]]
@ -3460,6 +3453,7 @@ dependencies = [
"litellm-cache-response",
"litellm-cache-testing",
"litellm-http",
"litellm-layered",
"reqwest 0.12.28",
"rstest",
"serde_json",
@ -3494,6 +3488,7 @@ dependencies = [
"litellm-cache",
"litellm-cache-testing",
"litellm-http",
"litellm-layered",
"percent-encoding",
"reqwest 0.12.28",
"rstest",
@ -3540,6 +3535,7 @@ version = "0.1.0"
dependencies = [
"litellm-cache",
"litellm-cache-testing",
"litellm-layered",
"r2d2",
"redis",
"redis-test",
@ -3598,6 +3594,7 @@ dependencies = [
"litellm-cache",
"litellm-cache-testing",
"litellm-http",
"litellm-layered",
"reqwest 0.12.28",
"rstest",
"serde_json",
@ -3732,11 +3729,11 @@ version = "0.1.0"
dependencies = [
"axum",
"base64 0.22.1",
"envy",
"futures-util",
"http-body-util",
"litellm-auth-types",
"litellm-config",
"litellm-core-utils",
"litellm-gateway-auth",
"litellm-gateway-inference",
"litellm-gateway-mcp",
@ -3759,6 +3756,7 @@ dependencies = [
"tower-sessions-moka-store",
"tracing",
"uuid",
"wiremock",
]
[[package]]
@ -3803,6 +3801,7 @@ dependencies = [
"litellm-inference-ocr",
"litellm-inference-responses",
"litellm-inference-transcription",
"litellm-layered",
"litellm-llms",
"litellm-llms-types",
"litellm-router",
@ -4037,6 +4036,7 @@ dependencies = [
"litellm-http",
"litellm-inference",
"litellm-inference-testing",
"litellm-layered",
"litellm-llms",
"litellm-llms-types",
"litellm-router-types",
@ -4184,6 +4184,7 @@ dependencies = [
"litellm-framer",
"litellm-host",
"litellm-http",
"litellm-layered",
"litellm-llms-types",
"litellm-python-compat",
"litellm-router-types",

View file

@ -19,6 +19,10 @@ disallowed-methods = [
{ path = "sqlx::query_as_with", reason = "use sqlx::query_as! or query_file_as! so the SQL is checked against the migrated schema" },
{ path = "sqlx::query_scalar_with", reason = "use sqlx::query_scalar! so the SQL is checked against the migrated schema" },
{ path = "sqlx::raw_sql", reason = "raw_sql is unchecked; use the checked query macros" },
{ path = "std::env::var", reason = "take a Lookup: Env in the Rust gateway, ProcessEnvironment in python-bridge" },
{ path = "std::env::var_os", reason = "take a Lookup: Env in the Rust gateway, ProcessEnvironment in python-bridge" },
{ path = "std::env::vars", reason = "take a Lookup: Env in the Rust gateway, ProcessEnvironment in python-bridge" },
{ path = "std::env::vars_os", reason = "take a Lookup: Env in the Rust gateway, ProcessEnvironment in python-bridge" },
]
# Every outbound client comes from litellm_http::HttpClientPool so it honors the host's TLS,

View file

@ -25,6 +25,8 @@ aws-smithy-runtime-api = "1.13.0"
[dev-dependencies]
rstest.workspace = true
tempfile.workspace = true
wiremock.workspace = true
litellm-http = { workspace = true, features = ["test-support"] }
reqwest.workspace = true
tokio.workspace = true

View file

@ -17,10 +17,11 @@ use litellm_auth_types::AwsParams;
use super::Error;
use super::constants::{
AWS_ACCESS_KEY_ID, AWS_EXTERNAL_ID, AWS_PROFILE_NAME, AWS_REGION_NAME, AWS_ROLE_ARN,
AWS_ROLE_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_NAME, AWS_SESSION_TOKEN,
AWS_SIGNED_HEADER_NAMES, AWS_STS_ENDPOINT, AWS_WEB_IDENTITY_TOKEN, AWS_WEB_IDENTITY_TOKEN_FILE,
DEFAULT_BEDROCK_REGION, DEFAULT_SESSION_NAME_PREFIX, SIGV4_COMPUTED_HEADER_NAMES,
AWS_ACCESS_KEY_ID, AWS_EXTERNAL_ID, AWS_PROFILE, AWS_PROFILE_NAME, AWS_REGION_NAME,
AWS_ROLE_ARN, AWS_ROLE_NAME, AWS_ROLE_SESSION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_NAME,
AWS_SESSION_TOKEN, AWS_SIGNED_HEADER_NAMES, AWS_STS_ENDPOINT, AWS_WEB_IDENTITY_TOKEN,
AWS_WEB_IDENTITY_TOKEN_FILE, DEFAULT_BEDROCK_REGION, DEFAULT_SESSION_NAME_PREFIX,
SIGV4_COMPUTED_HEADER_NAMES,
};
const STATIC_CREDENTIALS_TTL: Duration = Duration::from_secs(3600 - 60);
@ -274,46 +275,26 @@ impl AwsAuthService {
.map_err(|error| Error::AwsProfile(error.to_string()))
}
AwsAuthFlow::AssumeRole { role, session_name } => {
if is_already_running_as_role(&role, &resolved).await? {
let ambient_flow = AwsAuthFlow::DefaultChain;
let key = cache_key(&resolved, &ambient_flow);
if let Some(credentials) = self.get_cached_credentials(&key) {
return Ok(credentials);
}
let provider =
aws_config::default_provider::credentials::DefaultCredentialsChain::builder()
.build()
.await;
let credentials = provider
.provide_credentials()
.await
.map_err(|error| Error::AwsDefaultChain(error.to_string()))?;
self.set_cached_credentials(
key,
credentials.clone(),
credential_cache_ttl(&ambient_flow).unwrap_or(AMBIENT_CREDENTIALS_TTL),
);
return Ok(credentials);
if Box::pin(self.ambient_is_role(&role, &resolved, env_lookup)).await {
return Box::pin(self.ambient_credentials(&resolved, env_lookup)).await;
}
let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest());
if let Some(region) = resolved.region_name.clone() {
loader = loader.region(aws_types::region::Region::new(region));
}
if let Some(endpoint) = resolved.sts_endpoint.clone() {
loader = loader.endpoint_url(endpoint);
}
if let (Some(access_key_id), Some(secret_access_key)) =
(resolved.access_key_id, resolved.secret_access_key)
{
loader = loader.credentials_provider(Credentials::new(
let source = match (
resolved.access_key_id.clone(),
resolved.secret_access_key.clone(),
) {
(Some(access_key_id), Some(secret_access_key)) => Credentials::new(
access_key_id,
secret_access_key,
resolved.session_token,
resolved.session_token.clone(),
None,
"litellm-role-source",
));
}
let sdk_config = loader.load().await;
),
_ => Box::pin(self.ambient_credentials(&resolved, env_lookup)).await?,
};
let sdk_config = sts_config(&resolved)
.credentials_provider(source)
.load()
.await;
let builder = aws_config::sts::AssumeRoleProvider::builder(role);
let builder = match session_name {
Some(name) => builder.session_name(name),
@ -334,92 +315,175 @@ impl AwsAuthService {
role,
session_name,
} => {
let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest());
if let Some(region) = resolved.region_name {
loader = loader.region(aws_types::region::Region::new(region));
}
if let Some(endpoint) = resolved.sts_endpoint {
loader = loader.endpoint_url(endpoint);
}
let sdk_config = loader.load().await;
let client = aws_sdk_sts::Client::new(&sdk_config);
let response = client
.assume_role_with_web_identity()
.role_arn(role)
.role_session_name(session_name)
.web_identity_token(token)
.send()
.await
.map_err(|error| Error::AwsWebIdentity(error.to_string()))?;
let credentials = response
.credentials()
.ok_or(Error::AwsMissingWebIdentityCredentials)?;
let expiration = SystemTime::try_from(*credentials.expiration())
.map_err(|error| Error::AwsWebIdentityExpiration(error.to_string()))?;
Ok(Credentials::new(
credentials.access_key_id(),
credentials.secret_access_key(),
Some(credentials.session_token().to_string()),
Some(expiration),
"litellm-web-identity",
Box::pin(web_identity_credentials(
&resolved,
role,
session_name,
token,
))
.await
}
AwsAuthFlow::DefaultChain => {
let key = cache_key(&resolved, &AwsAuthFlow::DefaultChain);
if let Some(credentials) = self.get_cached_credentials(&key) {
return Ok(credentials);
}
let provider =
aws_config::default_provider::credentials::DefaultCredentialsChain::builder()
.build()
.await;
let credentials = provider
.provide_credentials()
.await
.map_err(|error| Error::AwsDefaultChain(error.to_string()))?;
self.set_cached_credentials(
key,
credentials.clone(),
credential_cache_ttl(&AwsAuthFlow::DefaultChain)
.unwrap_or(AMBIENT_CREDENTIALS_TTL),
);
Ok(credentials)
Box::pin(self.ambient_credentials(&resolved, env_lookup)).await
}
}
}
async fn ambient_credentials(
&self,
resolved: &AwsAuthConfig,
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
) -> Result<Credentials, Error> {
let key = cache_key(resolved, &AwsAuthFlow::DefaultChain);
if let Some(credentials) = self.get_cached_credentials(&key) {
return Ok(credentials);
}
let credentials = match ambient(env_lookup) {
Ambient::Keys(credentials) => credentials,
Ambient::WebIdentityFile {
role_arn,
token_file,
session_name,
} => {
let token = std::fs::read_to_string(&token_file)
.map_err(|error| Error::AwsWebIdentity(format!("{token_file}: {error}")))?;
web_identity_credentials(resolved, role_arn, session_name, token).await?
}
Ambient::Chain { profile_name } => {
let builder =
aws_config::default_provider::credentials::DefaultCredentialsChain::builder();
let builder = match profile_name {
Some(name) => builder.profile_name(&name),
None => builder,
};
builder
.build()
.await
.provide_credentials()
.await
.map_err(|error| Error::AwsDefaultChain(error.to_string()))?
}
};
self.set_cached_credentials(
key,
credentials.clone(),
credential_cache_ttl(&AwsAuthFlow::DefaultChain).unwrap_or(AMBIENT_CREDENTIALS_TTL),
);
Ok(credentials)
}
async fn ambient_is_role(
&self,
role: &str,
resolved: &AwsAuthConfig,
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
) -> bool {
if role_identity(role).is_none() {
return false;
}
if let Ambient::WebIdentityFile { role_arn, .. } = ambient(env_lookup) {
return same_role_arns(role, &role_arn);
}
let Ok(credentials) = self.ambient_credentials(resolved, env_lookup).await else {
return false;
};
let sdk_config = sts_config(resolved)
.credentials_provider(credentials)
.load()
.await;
aws_sdk_sts::Client::new(&sdk_config)
.get_caller_identity()
.send()
.await
.is_ok_and(|response| {
response
.arn()
.is_some_and(|caller| same_role_arns(role, caller))
})
}
}
async fn is_already_running_as_role(role: &str, config: &AwsAuthConfig) -> Result<bool, Error> {
if role_identity(role).is_none() {
return Ok(false);
}
if let (Ok(current_role), Ok(token_file)) = (
std::env::var(AWS_ROLE_ARN),
std::env::var(AWS_WEB_IDENTITY_TOKEN_FILE),
) && !token_file.is_empty()
{
return Ok(same_role_arns(role, &current_role));
}
/// What the SDK's default credentials chain would find, read through the injected lookup so a
/// config overlay reaches it. Only sources with no environment form are left to the SDK chain.
enum Ambient {
Keys(Credentials),
WebIdentityFile {
role_arn: String,
token_file: String,
session_name: String,
},
Chain {
profile_name: Option<String>,
},
}
let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest());
if let Some(region) = config.region_name.clone() {
loader = loader.region(aws_types::region::Region::new(region));
fn ambient(env_lookup: &(dyn Fn(&str) -> Option<String> + Sync)) -> Ambient {
if let (Some(access_key_id), Some(secret_access_key)) = (
env_lookup(AWS_ACCESS_KEY_ID),
env_lookup(AWS_SECRET_ACCESS_KEY),
) {
return Ambient::Keys(Credentials::new(
access_key_id,
secret_access_key,
env_lookup(AWS_SESSION_TOKEN),
None,
"litellm-environment",
));
}
if let Some(endpoint) = config.sts_endpoint.clone() {
loader = loader.endpoint_url(endpoint);
if let (Some(role_arn), Some(token_file)) = (
env_lookup(AWS_ROLE_ARN),
env_lookup(AWS_WEB_IDENTITY_TOKEN_FILE).filter(|file| !file.is_empty()),
) {
return Ambient::WebIdentityFile {
role_arn,
token_file,
session_name: env_lookup(AWS_ROLE_SESSION_NAME).unwrap_or_else(default_session_name),
};
}
let sdk_config = loader.load().await;
let response = match aws_sdk_sts::Client::new(&sdk_config)
.get_caller_identity()
Ambient::Chain {
profile_name: env_lookup(AWS_PROFILE),
}
}
fn sts_config(resolved: &AwsAuthConfig) -> aws_config::ConfigLoader {
let loader = aws_config::defaults(aws_config::BehaviorVersion::latest());
let loader = match resolved.region_name.clone() {
Some(region) => loader.region(aws_types::region::Region::new(region)),
None => loader,
};
match resolved.sts_endpoint.clone() {
Some(endpoint) => loader.endpoint_url(endpoint),
None => loader,
}
}
async fn web_identity_credentials(
resolved: &AwsAuthConfig,
role: String,
session_name: String,
token: String,
) -> Result<Credentials, Error> {
let sdk_config = sts_config(resolved).load().await;
let response = aws_sdk_sts::Client::new(&sdk_config)
.assume_role_with_web_identity()
.role_arn(role)
.role_session_name(session_name)
.web_identity_token(token)
.send()
.await
{
Ok(response) => response,
Err(_) => return Ok(false),
};
Ok(response
.arn()
.is_some_and(|caller| same_role_arns(role, caller)))
.map_err(|error| Error::AwsWebIdentity(error.to_string()))?;
let credentials = response
.credentials()
.ok_or(Error::AwsMissingWebIdentityCredentials)?;
let expiration = SystemTime::try_from(*credentials.expiration())
.map_err(|error| Error::AwsWebIdentityExpiration(error.to_string()))?;
Ok(Credentials::new(
credentials.access_key_id(),
credentials.secret_access_key(),
Some(credentials.session_token().to_string()),
Some(expiration),
"litellm-web-identity",
))
}
fn default_session_name() -> String {
@ -829,6 +893,66 @@ mod tests {
assert_eq!(credentials.session_token(), None);
}
#[tokio::test]
async fn lookup_keys_are_the_ambient_credentials_without_a_region() {
let env = |name: &str| match name {
AWS_ACCESS_KEY_ID => Some("lookup-ak".to_owned()),
AWS_SECRET_ACCESS_KEY => Some("lookup-sk".to_owned()),
_ => None,
};
let credentials = AwsAuthService::default()
.resolve_credentials(AwsAuthConfig::default(), &env)
.await
.unwrap();
assert_eq!(
(credentials.access_key_id(), credentials.secret_access_key()),
("lookup-ak", "lookup-sk")
);
}
#[tokio::test]
async fn a_role_the_lookup_web_identity_names_is_assumed_with_that_token() {
let role = "arn:aws:iam::123456789012:role/A";
let sts = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::body_string_contains(
"Action=AssumeRoleWithWebIdentity",
))
.and(wiremock::matchers::body_string_contains(
"WebIdentityToken=lookup-token",
))
.respond_with(wiremock::ResponseTemplate::new(200).set_body_string(
"<AssumeRoleWithWebIdentityResponse><AssumeRoleWithWebIdentityResult><Credentials>\
<AccessKeyId>sts-ak</AccessKeyId><SecretAccessKey>sts-sk</SecretAccessKey>\
<SessionToken>sts-token</SessionToken><Expiration>2099-01-01T00:00:00Z</Expiration>\
</Credentials></AssumeRoleWithWebIdentityResult></AssumeRoleWithWebIdentityResponse>",
))
.mount(&sts)
.await;
let token_file = tempfile::NamedTempFile::new().unwrap();
std::fs::write(token_file.path(), "lookup-token").unwrap();
let token_path = token_file.path().to_string_lossy().into_owned();
let env = |name: &str| match name {
AWS_ROLE_ARN => Some(role.to_owned()),
AWS_WEB_IDENTITY_TOKEN_FILE => Some(token_path.clone()),
_ => None,
};
let credentials = AwsAuthService::default()
.resolve_credentials(
AwsAuthConfig {
role_name: Some(role.into()),
region_name: Some("us-east-1".into()),
sts_endpoint: Some(sts.uri()),
..Default::default()
},
&env,
)
.await
.unwrap();
assert_eq!(credentials.access_key_id(), "sts-ak");
}
#[test]
fn cache_policy_matches_python_flows() {
assert_eq!(
@ -1063,6 +1187,7 @@ mod tests {
#[ignore]
#[tokio::test]
#[allow(clippy::disallowed_methods)] // the live test reads its own credentials
async fn live_bedrock_invoke_model_returns_200() -> Result<(), Box<dyn std::error::Error>> {
let access_key_id = std::env::var("AWS_BEDROCK_TEST_ACCESS_KEY_ID")?;
let secret_access_key = std::env::var("AWS_BEDROCK_TEST_SECRET_ACCESS_KEY")?;

View file

@ -12,6 +12,8 @@ pub const AWS_PROFILE_NAME: &str = AwsParams::PROFILE_NAME.env[0];
pub const AWS_ROLE_NAME: &str = AwsParams::ROLE_NAME.env[0];
pub const AWS_WEB_IDENTITY_TOKEN: &str = AwsParams::WEB_IDENTITY_TOKEN.env[0];
pub const AWS_ROLE_ARN: &str = "AWS_ROLE_ARN";
pub const AWS_ROLE_SESSION_NAME: &str = "AWS_ROLE_SESSION_NAME";
pub const AWS_PROFILE: &str = "AWS_PROFILE";
pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = "AWS_WEB_IDENTITY_TOKEN_FILE";
pub const AWS_STS_ENDPOINT: &str = AwsParams::STS_ENDPOINT.env[0];
pub const AWS_EXTERNAL_ID: &str = AwsParams::EXTERNAL_ID.env[0];

View file

@ -10,6 +10,7 @@ litellm-http.workspace = true
litellm-auth-azure.workspace = true
litellm-auth-types.workspace = true
litellm-cache.workspace = true
litellm-layered.workspace = true
async-trait = "0.1"
azure_core = "1.1.0"

View file

@ -10,9 +10,11 @@ use azure_storage_blob::{
models::{BlobClientUploadOptions, StorageErrorCode},
};
use futures_util::{TryStreamExt, future::try_join_all};
use litellm_auth_azure::AzureAuthService;
use litellm_cache::{
BaseCache, BatchCache, CacheCodec, DisconnectCache, Error, ExactCacheContext, FlushCache,
};
use litellm_layered::SharedLookup;
use tokio::runtime::Handle;
use url::Url;
@ -34,11 +36,15 @@ impl<C: CacheCodec> AzureBlobCache<C> {
http: litellm_http::Client,
codec: C,
runtime: Handle,
env: SharedLookup,
) -> Result<Self, Error> {
Self::connect_with_options(
account_url,
container,
Some(Arc::new(AzureBlobCredential::default())),
Some(Arc::new(AzureBlobCredential::new(
AzureAuthService::default(),
env,
))),
ClientOptions {
transport: Some(Transport::new(Arc::new(ReqwestTransport(http)))),
..ClientOptions::default()

View file

@ -1,6 +1,5 @@
use std::{
fmt,
sync::Arc,
time::{Duration, SystemTime},
};
@ -11,15 +10,14 @@ use azure_core::{
};
use litellm_auth_azure::{AzureAuthInputs, AzureAuthService};
use litellm_auth_types::ResolvedCredential;
use litellm_layered::SharedLookup;
const STATIC_TOKEN_LIFETIME: Duration = Duration::from_secs(300);
const LLM_TOKEN_ENV: &str = "AZURE_AD_TOKEN";
type EnvLookup = Arc<dyn Fn(&str) -> Option<String> + Send + Sync>;
pub struct AzureBlobCredential {
service: AzureAuthService,
env_lookup: EnvLookup,
env_lookup: SharedLookup,
}
impl fmt::Debug for AzureBlobCredential {
@ -28,17 +26,8 @@ impl fmt::Debug for AzureBlobCredential {
}
}
impl Default for AzureBlobCredential {
fn default() -> Self {
Self::new(
AzureAuthService::default(),
Arc::new(|name| std::env::var(name).ok()),
)
}
}
impl AzureBlobCredential {
pub fn new(service: AzureAuthService, env_lookup: EnvLookup) -> Self {
pub fn new(service: AzureAuthService, env_lookup: SharedLookup) -> Self {
Self {
service,
env_lookup,
@ -54,7 +43,11 @@ impl TokenCredential for AzureBlobCredential {
_options: Option<TokenRequestOptions<'_>>,
) -> azure_core::Result<AccessToken> {
let env_lookup = &self.env_lookup;
let lookup = move |name: &str| (name != LLM_TOKEN_ENV).then(|| env_lookup(name)).flatten();
let lookup = move |name: &str| {
(name != LLM_TOKEN_ENV)
.then(|| env_lookup.get(name))
.flatten()
};
let credential = self
.service
.get_azure_ad_token(

View file

@ -11,6 +11,7 @@ futures-util.workspace = true
litellm-auth-gcp.workspace = true
litellm-auth-types.workspace = true
litellm-cache.workspace = true
litellm-layered.workspace = true
percent-encoding.workspace = true
reqwest.workspace = true
tokio.workspace = true
@ -18,6 +19,7 @@ tokio.workspace = true
[dev-dependencies]
litellm-http = { workspace = true, features = ["test-support"] }
litellm-cache-testing.workspace = true
litellm-layered.workspace = true
rstest.workspace = true
serde_json.workspace = true
tokio.workspace = true

View file

@ -6,6 +6,7 @@ use litellm_cache::{
FlushCache,
};
use litellm_http::Client;
use litellm_layered::SharedLookup;
use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_encode};
use crate::{GcpTokenSource, TokenSource};
@ -53,8 +54,11 @@ pub struct GcsCache<S: CacheCodec> {
}
impl<S: CacheCodec> GcsCache<S> {
pub fn new(config: GcsConfig, client: Client, codec: S) -> Self {
let token = Arc::new(GcpTokenSource::new(config.path_service_account.clone()));
pub fn new(config: GcsConfig, client: Client, codec: S, env: SharedLookup) -> Self {
let token = Arc::new(GcpTokenSource::new(
config.path_service_account.clone(),
env,
));
Self::with_token_source(config, client, codec, token)
}

View file

@ -3,6 +3,7 @@ use std::{future::Future, pin::Pin};
use litellm_auth_gcp::{VertexAuth, VertexConfig};
use litellm_auth_types::{InputSource, SecretValue, Sourced};
use litellm_cache::Error;
use litellm_layered::SharedLookup;
pub trait TokenSource: Send + Sync + 'static {
fn bearer_token(&self) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + '_>>;
@ -11,15 +12,17 @@ pub trait TokenSource: Send + Sync + 'static {
pub struct GcpTokenSource {
auth: VertexAuth,
config: VertexConfig,
env: SharedLookup,
}
impl GcpTokenSource {
pub fn new(path_service_account: Option<String>) -> Self {
pub fn new(path_service_account: Option<String>, env: SharedLookup) -> Self {
let credentials = path_service_account
.map(|path| Sourced::new(SecretValue::new(path), InputSource::Deployment));
Self {
auth: VertexAuth::default(),
config: VertexConfig::new(credentials, None, None),
env,
}
}
}
@ -28,7 +31,7 @@ impl TokenSource for GcpTokenSource {
fn bearer_token(&self) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + '_>> {
Box::pin(async move {
self.auth
.access_token(&self.config, &|name| std::env::var(name).ok())
.access_token(&self.config, &|name| self.env.get(name))
.await
.map_err(|_| Error::Unavailable)
})

View file

@ -99,6 +99,7 @@ async fn cache_exposes_its_configuration(#[future(awt)] server: MockServer) {
},
litellm_http::Client::plain_for_test(),
litellm_cache::JsonCodec::<Value>::new(),
std::sync::Arc::new(litellm_layered::Env::default()),
);
assert_eq!(cache.bucket_name(), "bucket");
assert_eq!(cache.key_prefix(), "folder/");

View file

@ -1,3 +1,4 @@
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
mod support;
use std::time::Duration;

View file

@ -7,6 +7,7 @@ repository.workspace = true
[dependencies]
litellm-cache.workspace = true
litellm-layered.workspace = true
redis = { version = "1.7.0", features = ["cluster", "tls-rustls"] }
r2d2 = "0.8.10"
tokio.workspace = true

View file

@ -4,6 +4,7 @@ use std::{
};
use litellm_cache::{BatchEntry, CacheCodec, Error};
use litellm_layered::Lookup;
use crate::{
connection::{ConnectionRef, Connections},
@ -20,11 +21,17 @@ pub struct RedisCache<S, C = redis::Connection> {
pub(crate) topology: RedisTopology,
/// The server's major version, read from `INFO` once, like Python's `redis_version`.
pub(crate) major_version: Arc<OnceLock<u32>>,
default_major_version: u32,
}
impl<S: CacheCodec> RedisCache<S> {
pub fn new(url: &str, default_ttl: Option<Duration>, codec: S) -> Result<Self, Error> {
Self::connect(url, &RedisTopology::Standalone, default_ttl, codec)
pub fn new(
url: &str,
default_ttl: Option<Duration>,
codec: S,
env: &(impl Lookup + ?Sized),
) -> Result<Self, Error> {
Self::connect(url, &RedisTopology::Standalone, default_ttl, codec, env)
}
pub fn connect(
@ -32,6 +39,7 @@ impl<S: CacheCodec> RedisCache<S> {
topology: &RedisTopology,
default_ttl: Option<Duration>,
codec: S,
env: &(impl Lookup + ?Sized),
) -> Result<Self, Error> {
let connections = Connections::open(url, topology)?;
Ok(Self {
@ -41,6 +49,7 @@ impl<S: CacheCodec> RedisCache<S> {
namespace: None,
topology: topology.clone(),
major_version: Arc::default(),
default_major_version: default_major_version(env),
})
}
}
@ -50,7 +59,12 @@ where
S: CacheCodec,
C: redis::ConnectionLike + Send + 'static,
{
pub fn with_connection(connection: C, default_ttl: Option<Duration>, codec: S) -> Self {
pub fn with_connection(
connection: C,
default_ttl: Option<Duration>,
codec: S,
env: &(impl Lookup + ?Sized),
) -> Self {
Self {
connections: Arc::new(Connections::fixed(connection)),
default_ttl: default_ttl.unwrap_or(DEFAULT_TTL),
@ -58,6 +72,7 @@ where
namespace: None,
topology: RedisTopology::Standalone,
major_version: Arc::default(),
default_major_version: default_major_version(env),
}
}
@ -109,7 +124,7 @@ where
let version = info
.ok()
.and_then(|info| parse_major_version(&info))
.unwrap_or_else(default_major_version);
.unwrap_or(self.default_major_version);
*self.major_version.get_or_init(|| version)
}
@ -184,9 +199,9 @@ fn parse_major_version(info: &str) -> Option<u32> {
}
}
fn default_major_version() -> u32 {
std::env::var("DEFAULT_REDIS_MAJOR_VERSION")
.ok()
/// `DEFAULT_REDIS_MAJOR_VERSION`, the version assumed when `INFO` does not answer.
fn default_major_version(env: &(impl Lookup + ?Sized)) -> u32 {
env.get("DEFAULT_REDIS_MAJOR_VERSION")
.and_then(|value| value.parse().ok())
.unwrap_or(7)
}

View file

@ -25,11 +25,11 @@ fn mock(commands: Vec<MockCmd>) -> MockRedisConnection {
}
fn tagged(commands: Vec<MockCmd>) -> Mocked<TaggedByteCodec> {
RedisCache::with_connection(mock(commands), None, TaggedByteCodec(42))
RedisCache::with_connection(mock(commands), None, TaggedByteCodec(42), &|_: &str| None)
}
fn json_cache(commands: Vec<MockCmd>) -> Mocked<JsonCodec<serde_json::Value>> {
RedisCache::with_connection(mock(commands), None, JsonCodec::new())
RedisCache::with_connection(mock(commands), None, JsonCodec::new(), &|_: &str| None)
}
fn team(commands: Vec<MockCmd>) -> Mocked<JsonCodec<serde_json::Value>> {
@ -55,7 +55,15 @@ fn scan(pattern: &str, cursor: u64, count: usize, reply: redis::Value) -> MockCm
#[rstest]
fn constructor_rejects_invalid_urls() {
assert!(RedisCache::new("not a redis url", None, JsonCodec::<String>::new()).is_err());
assert!(
RedisCache::new(
"not a redis url",
None,
JsonCodec::<String>::new(),
&|_: &str| None
)
.is_err()
);
}
#[rstest]
@ -144,6 +152,7 @@ async fn async_operations_preserve_codec_ttl_and_missing_values(context: ExactCa
]),
Some(Duration::from_secs(9)),
TaggedByteCodec(42),
&|_: &str| None,
);
cache
.batch_cache_write("counter", 7, context.clone())
@ -342,6 +351,7 @@ async fn ttl_pipeline_keeps_each_entry_ttl_and_defaults_missing_ones() {
)]),
Some(Duration::from_secs(300)),
JsonCodec::<serde_json::Value>::new(),
&|_: &str| None,
)
.with_namespace(Some("ns".into()));
@ -616,6 +626,35 @@ async fn counted_lpop_assumes_redis_7_when_info_fails() {
);
}
#[rstest]
#[tokio::test]
async fn counted_lpop_pops_one_at_a_time_when_info_fails_and_the_env_names_an_older_version() {
let single_pop = || {
let mut pipeline = redis::pipe();
pipeline.cmd("LPOP").arg("team:queue");
pipeline
};
let cache = RedisCache::with_connection(
mock(vec![
MockCmd::new(
redis::cmd("INFO"),
Err::<String, _>(redis::RedisError::from((redis::ErrorKind::Io, "down"))),
),
MockCmd::with_values(single_pop(), Ok(vec![redis_test::redis_value!("a")])),
MockCmd::with_values(single_pop(), Ok(vec![redis_test::redis_value!("b")])),
]),
None,
JsonCodec::<serde_json::Value>::new(),
&litellm_layered::Env::of(&[("DEFAULT_REDIS_MAJOR_VERSION", "5")]),
)
.with_namespace(Some("team".into()));
assert_eq!(
cache.async_lpop("queue", Some(2)).await.unwrap(),
RedisLpopResult::Values(vec![b"a".to_vec(), b"b".to_vec()])
);
}
fn push_and_trim(start: i64) -> redis::Pipeline {
let mut pipeline = redis::pipe();
pipeline
@ -924,6 +963,7 @@ async fn increments_keep_an_existing_ttl_in_one_atomic_script(context: ExactCach
mock(vec![increment_script(2.5), increment_script(2.5)]),
None,
JsonCodec::<f64>::new(),
&|_: &str| None,
);
assert_eq!(
@ -1153,6 +1193,12 @@ async fn disconnect_keeps_a_caller_owned_connection_usable() {
#[rstest]
#[tokio::test]
async fn disconnect_drains_an_idle_pool_without_connecting() {
let cache = RedisCache::new("redis://127.0.0.1:1", None, JsonCodec::<String>::new()).unwrap();
let cache = RedisCache::new(
"redis://127.0.0.1:1",
None,
JsonCodec::<String>::new(),
&|_: &str| None,
)
.unwrap();
cache.disconnect().await.unwrap();
}

View file

@ -64,6 +64,7 @@ fn constructor_rejects_unusable_cluster_configs(
&RedisTopology::Cluster { startup_nodes },
None,
JsonCodec::new(),
&|_: &str| None,
)
.err();
assert!(matches!(error, Some(Error::Unavailable)));

View file

@ -1,3 +1,4 @@
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
#![allow(dead_code)]
use std::{
@ -183,8 +184,10 @@ impl redis::ConnectionLike for FakeRedis {
pub type JsonCache<C = redis::Connection> = RedisCache<JsonCodec<serde_json::Value>, C>;
pub fn fake_cache(namespace: &str) -> JsonCache<FakeRedis> {
RedisCache::with_connection(FakeRedis::default(), None, JsonCodec::new())
.with_namespace(Some(namespace.into()))
RedisCache::with_connection(FakeRedis::default(), None, JsonCodec::new(), &|_: &str| {
None
})
.with_namespace(Some(namespace.into()))
}
/// Startup nodes from `LITELLM_TEST_REDIS_CLUSTER_NODES` (`host:port,host:port`); tests that
@ -224,8 +227,14 @@ pub fn cluster_cache<S: CacheCodec>(
) -> Option<RedisCache<S>> {
let topology = cluster_topology()?;
Some(
RedisCache::connect(&cluster_url(), &topology, Some(default_ttl), codec)
.expect("cluster connection")
.with_namespace(Some(unique_namespace(label))),
RedisCache::connect(
&cluster_url(),
&topology,
Some(default_ttl),
codec,
&|_: &str| None,
)
.expect("cluster connection")
.with_namespace(Some(unique_namespace(label))),
)
}

View file

@ -34,7 +34,7 @@ pub fn keyed(key: &str) -> ResponseCacheRequest {
pub fn redis(commands: Vec<MockCmd>, namespace: Option<&str>) -> ResponseCache<MockedRedis> {
let connection = MockRedisConnection::new(commands).assert_all_commands_consumed();
ResponseCache::new(Arc::new(
RedisCache::with_connection(connection, None, ResponseCacheCodec)
RedisCache::with_connection(connection, None, ResponseCacheCodec, &|_: &str| None)
.with_namespace(namespace.map(str::to_owned)),
))
}

View file

@ -9,6 +9,7 @@ repository.workspace = true
litellm-http.workspace = true
litellm-cache.workspace = true
litellm-auth-aws.workspace = true
litellm-layered.workspace = true
aws-sdk-s3 = { version = "1.146.1", default-features = false, features = ["rustls", "rt-tokio"] }
aws-credential-types = "1.3.0"
aws-smithy-runtime-api = { version = "1.16.2", features = ["client", "http-1x"] }
@ -22,6 +23,7 @@ tokio.workspace = true
[dev-dependencies]
litellm-http = { workspace = true, features = ["test-support"] }
litellm-cache-testing.workspace = true
litellm-layered.workspace = true
rstest.workspace = true
wiremock.workspace = true
serde_json.workspace = true

View file

@ -3,20 +3,17 @@ use aws_credential_types::{
provider::{ProvideCredentials, error::CredentialsError, future},
};
use litellm_auth_aws::{AwsAuthConfig, AwsAuthService};
use litellm_layered::SharedLookup;
#[derive(Clone)]
pub struct S3Credentials {
auth: AwsAuthService,
config: AwsAuthConfig,
env: fn(&str) -> Option<String>,
env: SharedLookup,
}
impl S3Credentials {
pub fn new(config: AwsAuthConfig) -> Self {
Self::with_env(config, |name| std::env::var(name).ok())
}
pub fn with_env(config: AwsAuthConfig, env: fn(&str) -> Option<String>) -> Self {
pub fn new(config: AwsAuthConfig, env: SharedLookup) -> Self {
Self {
auth: AwsAuthService::default(),
config,
@ -44,7 +41,7 @@ impl ProvideCredentials for S3Credentials {
));
}
self.auth
.resolve_credentials(self.config.clone(), &self.env)
.resolve_credentials(self.config.clone(), &|name| self.env.get(name))
.await
.map_err(|_| CredentialsError::provider_error("S3 cache authentication failed"))
})

View file

@ -15,6 +15,7 @@ use litellm_auth_aws::AwsAuthConfig;
use litellm_cache::{
BaseCache, BatchCache, CacheCodec, DisconnectCache, Error, ExactCacheContext, FlushCache,
};
use litellm_layered::SharedLookup;
use tokio::runtime::Handle;
use crate::{auth::S3Credentials, transport::ReqwestHttpClient};
@ -47,13 +48,14 @@ impl<C: CacheCodec> S3Cache<C> {
http: litellm_http::Client,
codec: C,
runtime: Handle,
env: SharedLookup,
) -> Self {
let endpoint_url: Option<String> = config.endpoint.map(|endpoint| endpoint.url);
let base = aws_sdk_s3::Config::builder()
.behavior_version(BehaviorVersion::latest())
.region(Region::new(config.region.clone()))
.http_client(ReqwestHttpClient(http))
.credentials_provider(S3Credentials::new(config.auth))
.credentials_provider(S3Credentials::new(config.auth, env))
.request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
.response_checksum_validation(ResponseChecksumValidation::WhenRequired);
let builder = match &endpoint_url {

View file

@ -42,7 +42,7 @@ async fn credentials_resolve(
#[case] env: fn(&str) -> Option<String>,
#[case] expected: (&str, &str, Option<&str>),
) {
let credentials = S3Credentials::with_env(config, env)
let credentials = S3Credentials::new(config, std::sync::Arc::new(env))
.provide_credentials()
.await
.unwrap();

View file

@ -37,6 +37,7 @@ pub fn cache_with(config: S3CacheConfig, runtime: Handle) -> JsonS3Cache {
litellm_http::Client::plain_for_test(),
JsonCodec::new(),
runtime,
std::sync::Arc::new(litellm_layered::Env::default()),
)
}

View file

@ -1,26 +1,5 @@
use std::str::FromStr;
use crate::coercion::parse_str_bool;
pub trait Lookup {
fn get(&self, name: &str) -> Option<String>;
fn truthy(&self, name: &str) -> Option<String> {
self.get(name).filter(|value| !value.is_empty())
}
fn enabled(&self, name: &str) -> Option<bool> {
self.get(name)
.is_some_and(|value| parse_str_bool(&value) == Some(true))
.then_some(true)
}
fn parsed<T: FromStr>(&self, name: &str) -> Option<T>
where
Self: Sized,
{
self.get(name).and_then(|value| value.trim().parse().ok())
}
}
impl<F: Fn(&str) -> Option<String>> Lookup for F {
@ -29,14 +8,6 @@ impl<F: Fn(&str) -> Option<String>> Lookup for F {
}
}
pub struct ProcessEnvironment;
impl Lookup for ProcessEnvironment {
fn get(&self, name: &str) -> Option<String> {
std::env::var(name).ok()
}
}
pub fn resolve_non_empty(
value: Option<&str>,
env_lookup: &dyn Fn(&str) -> Option<String>,
@ -72,35 +43,4 @@ mod tests {
assert_eq!(env.get("EMPTY"), Some(String::new()));
assert_eq!(env.get("ABSENT"), None);
}
#[test]
fn truthy_drops_an_empty_value_like_a_python_or_chain() {
let env = env_of(&[("EMPTY", ""), ("SET", "value")]);
assert_eq!(env.truthy("EMPTY"), None);
assert_eq!(env.truthy("SET").as_deref(), Some("value"));
}
#[test]
fn enabled_only_switches_on_for_true_and_never_forces_off() {
let env = env_of(&[
("LOWER", "true"),
("PADDED", " True "),
("OFF", "false"),
("ONE", "1"),
]);
assert_eq!(env.enabled("LOWER"), Some(true));
assert_eq!(env.enabled("PADDED"), Some(true));
assert_eq!(env.enabled("OFF"), None);
assert_eq!(env.enabled("ONE"), None);
assert_eq!(env.enabled("ABSENT"), None);
}
#[test]
fn parsed_trims_and_skips_values_that_do_not_parse() {
let env = env_of(&[("PADDED", " 45 "), ("WORD", "soon"), ("FRACTION", "0.5")]);
assert_eq!(env.parsed::<u32>("PADDED"), Some(45));
assert_eq!(env.parsed::<u32>("WORD"), None);
assert_eq!(env.parsed::<f64>("FRACTION"), Some(0.5));
assert_eq!(env.parsed::<u32>("ABSENT"), None);
}
}

View file

@ -1,3 +1,4 @@
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
use std::{
env,
process::{Command, ExitCode},

View file

@ -22,6 +22,7 @@ litellm-host-http.workspace = true
litellm-host.workspace = true
litellm-http.workspace = true
litellm-llms.workspace = true
litellm-layered.workspace = true
litellm-router.workspace = true
litellm-secrets.workspace = true
litellm-llms-types.workspace = true

View file

@ -22,6 +22,7 @@ use litellm_inference_messages::MessagesRoute;
use litellm_inference_ocr::OcrRoute;
use litellm_inference_responses::ResponsesRoute;
use litellm_inference_transcription::AudioTranscriptionRoute;
use litellm_layered::SharedLookup;
use litellm_llms::base_llm::ocr::{handler::OcrClient, settings::OcrSettings};
use litellm_secrets::source::SecretSource;
@ -53,6 +54,8 @@ impl Gateway {
pub fn new(
resources: CoreResources,
http: HttpClientConfig,
ocr: OcrSettings,
env: SharedLookup,
secrets: Arc<dyn SecretSource>,
models: ModelRouter,
) -> Result<Self, litellm_http::Error> {
@ -70,14 +73,14 @@ impl Gateway {
auth.clone(),
secrets.clone(),
),
messages: MessagesRoute::new(provider.clone(), auth.clone(), secrets.clone()),
messages: MessagesRoute::new(provider.clone(), auth.clone(), secrets.clone(), env),
responses: ResponsesRoute::new(provider, auth.clone(), secrets.clone()),
ocr: OcrRoute::new(OcrClient::new(
&resources.pool,
&http,
UrlPolicy::default(),
auth,
OcrSettings::default(),
ocr,
secrets.clone(),
)?),
models,

View file

@ -13,6 +13,7 @@ use futures_util::future::BoxFuture;
use litellm_gateway_inference::{Deployment, Gateway, router};
use litellm_http::{HttpClientPool, HttpSettings, Resolution, media::PublicDnsResolver};
use litellm_inference::resources::CoreResources;
use litellm_llms::base_llm::ocr::settings::OcrSettings;
use litellm_secrets::{SecretValue, source::SecretSource};
use serde_json::Value;
use tower::ServiceExt;
@ -83,6 +84,8 @@ fn configured_app(
let gateway = Gateway::new(
resources,
http,
OcrSettings::default(),
Arc::new(litellm_layered::Env::default()),
secrets,
[(
"public/model".into(),

View file

@ -7,7 +7,6 @@ repository.workspace = true
[dependencies]
axum.workspace = true
envy = "0.4.2"
http-body-util = "0.1"
litellm-inference.workspace = true
litellm-gateway-inference.workspace = true
@ -19,6 +18,7 @@ litellm-gateway-auth.workspace = true
litellm-gateway-ui.workspace = true
litellm-auth-types.workspace = true
litellm-config.workspace = true
litellm-core-utils.workspace = true
litellm-http = { workspace = true, features = ["mcp"] }
litellm-layered.workspace = true
litellm-llms.workspace = true
@ -38,3 +38,4 @@ serde_yaml_ng = "0.10.0"
tempfile.workspace = true
tokio = { workspace = true, features = ["sync"] }
tower = { version = "0.5", features = ["util"] }
wiremock.workspace = true

View file

@ -20,30 +20,35 @@ use litellm_gateway_auth::Auth;
use litellm_gateway_inference::{Gateway, ModelRouter};
use litellm_http::{ClientVariant, HttpClientPool, Resolution, media::PublicDnsResolver};
use litellm_inference::resources::CoreResources;
use litellm_layered::Env;
use litellm_llms::base_llm::ocr::settings::{OcrKeysPartial, OcrSettings};
use litellm_secrets::source::EnvironmentSecrets;
use litellm_tracing::ByteChunk;
use uuid::Uuid;
pub fn build_inference(config: &Config) -> Result<Arc<Gateway>, Error> {
pub fn build_inference(config: &Config, process: &Env) -> Result<Arc<Gateway>, Error> {
let pool = Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver)));
let environment = secrets::environment_values(&config.environment_variables)?;
let lookup = |name: &str| {
let env = process.clone().overlaid(
environment
.get(name)
.map(|value| value.expose().to_owned())
.or_else(|| std::env::var(name).ok())
};
let http_settings = http::http_settings(&lookup, &config.litellm_settings)?;
.iter()
.map(|(name, value)| (name.clone(), value.expose().to_owned())),
);
let http_settings = http::http_settings(&env, &config.litellm_settings)?;
let ocr = OcrSettings::from(OcrKeysPartial::from_env(&env)?);
let http = Resolution::from(&http_settings).config;
let client = pool.client(&http, ClientVariant::Provider)?;
let env = Arc::new(env);
let secrets = Arc::new(secrets::ConfigSecrets::new(
environment,
Arc::new(EnvironmentSecrets::python_compatible(client)),
Arc::new(EnvironmentSecrets::python_compatible(client, env.clone())),
));
let resources = CoreResources::new(pool);
Ok(Arc::new(Gateway::new(
resources,
http,
ocr,
env,
secrets,
ModelRouter::from_model_list(&config.model_list),
)?))

View file

@ -10,7 +10,8 @@ use serde_json::json;
mod settings;
use settings::{Settings, UiSettings};
use litellm_layered::Env;
use settings::{BootKeysPartial, Settings, UiKeysPartial, UiSettings};
struct StderrSink {
level: Level,
@ -41,14 +42,15 @@ impl Sink for StderrSink {
#[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
let settings: Settings = settings::from_iter(std::env::vars_os())?;
let env = Env::from_process();
let settings = Settings::from(BootKeysPartial::from_env(&env)?);
let level = settings.log_level();
Logger::new(StderrSink { level }).install_global()?;
let config = Config::load(settings.litellm_config)?;
let inference = litellm_gateway::build_inference(&config)?;
let ui = match std::env::var_os("LITELLM_UI_PATH") {
let inference = litellm_gateway::build_inference(&config, &env)?;
let ui = match &settings.ui_path {
Some(directory) => {
let ui_settings: UiSettings = settings::from_iter(std::env::vars_os())?;
let ui_settings = UiSettings::try_from(UiKeysPartial::from_env(&env)?)?;
let backend = UiBackend::new(ui_settings.ui_username, ui_settings.ui_password)?;
Some(
litellm_gateway_ui::router(

View file

@ -1,25 +1,39 @@
use std::ffi::OsString;
use litellm_auth_types::SecretValue;
use litellm_core_utils::coercion;
use litellm_layered::{Invalid, Layered};
use litellm_tracing::Level;
use serde::{Deserialize, de::DeserializeOwned};
#[derive(Deserialize)]
#[serde(default)]
#[derive(Layered)]
#[expect(dead_code, reason = "only BootKeysPartial is built")]
pub(super) struct BootKeys {
#[layer(env = "HOST")]
host: String,
#[layer(env = "PORT", read = coercion::required::<u16>)]
port: u16,
#[layer(env = "LITELLM_CONFIG")]
litellm_config: String,
#[layer(env = "RUST_LOG")]
rust_log: String,
#[layer(env = "LITELLM_UI_PATH")]
ui_path: String,
}
pub(super) struct Settings {
pub host: String,
pub port: u16,
pub litellm_config: String,
pub ui_path: Option<String>,
rust_log: Option<String>,
}
impl Default for Settings {
fn default() -> Self {
impl From<BootKeysPartial> for Settings {
fn from(keys: BootKeysPartial) -> Self {
Self {
host: "0.0.0.0".into(),
port: 4000,
litellm_config: "config.yaml".into(),
rust_log: None,
host: keys.host.unwrap_or_else(|| "0.0.0.0".into()),
port: keys.port.unwrap_or(4000),
litellm_config: keys.litellm_config.unwrap_or_else(|| "config.yaml".into()),
ui_path: keys.ui_path,
rust_log: keys.rust_log,
}
}
}
@ -33,64 +47,81 @@ impl Settings {
}
}
#[derive(Deserialize)]
#[derive(Layered)]
#[expect(dead_code, reason = "only UiKeysPartial is built")]
pub(super) struct UiKeys {
#[layer(env = "UI_USERNAME")]
ui_username: String,
#[layer(env = "UI_PASSWORD", read = secret, secret)]
ui_password: SecretValue,
#[layer(env = "LITELLM_UI_SECURE_COOKIES", read = coercion::required::<bool>)]
litellm_ui_secure_cookies: bool,
}
fn secret(value: &str) -> Result<Option<SecretValue>, Invalid> {
Ok(Some(SecretValue::new(value)))
}
pub(super) struct UiSettings {
#[serde(default = "default_username")]
pub ui_username: String,
pub ui_password: SecretValue,
#[serde(default = "default_secure_cookies")]
pub litellm_ui_secure_cookies: bool,
}
fn default_username() -> String {
"admin".into()
}
#[derive(Debug, thiserror::Error)]
#[error("UI_PASSWORD must be set when LITELLM_UI_PATH serves the UI")]
pub(super) struct MissingUiPassword;
fn default_secure_cookies() -> bool {
true
}
impl TryFrom<UiKeysPartial> for UiSettings {
type Error = MissingUiPassword;
pub(super) fn from_iter<T: DeserializeOwned>(
variables: impl IntoIterator<Item = (OsString, OsString)>,
) -> envy::Result<T> {
envy::from_iter(
variables
.into_iter()
.filter_map(|(key, value)| Some((key.into_string().ok()?, value.into_string().ok()?))),
)
fn try_from(keys: UiKeysPartial) -> Result<Self, Self::Error> {
Ok(Self {
ui_username: keys.ui_username.unwrap_or_else(|| "admin".into()),
ui_password: keys.ui_password.ok_or(MissingUiPassword)?,
litellm_ui_secure_cookies: keys.litellm_ui_secure_cookies.unwrap_or(true),
})
}
}
#[cfg(test)]
mod tests {
use std::ffi::OsString;
use litellm_layered::Env;
use rstest::rstest;
use super::*;
fn parse<T: DeserializeOwned>(variables: &[(&str, &str)]) -> envy::Result<T> {
from_iter(
variables
.iter()
.map(|(key, value)| (OsString::from(key), OsString::from(value))),
)
fn boot(variables: &[(&str, &str)]) -> Result<Settings, litellm_layered::Error> {
BootKeysPartial::from_env(&Env::of(variables)).map(Settings::from)
}
fn ui(variables: &[(&str, &str)]) -> Result<UiSettings, Box<dyn std::error::Error>> {
Ok(UiSettings::try_from(UiKeysPartial::from_env(&Env::of(
variables,
))?)?)
}
#[rstest]
fn defaults_allow_startup_without_ui_credentials() {
let settings: Settings = parse(&[]).unwrap();
let settings = boot(&[]).unwrap();
assert_eq!(settings.host, "0.0.0.0");
assert_eq!(settings.port, 4000);
assert_eq!(settings.litellm_config, "config.yaml");
assert_eq!(settings.ui_path, None);
assert_eq!(settings.log_level(), Level::INFO);
}
#[rstest]
fn environment_overrides_startup_settings() {
let settings: Settings = parse(&[
let settings = boot(&[
("HOST", "127.0.0.1"),
("PORT", "8080"),
("LITELLM_CONFIG", "/tmp/custom.yaml"),
("RUST_LOG", "debug"),
("LITELLM_UI_PATH", "/srv/ui"),
("LITELLM_UI_SECURE_COOKIES", "invalid-but-ui-disabled"),
])
.unwrap();
@ -98,6 +129,7 @@ mod tests {
assert_eq!(settings.host, "127.0.0.1");
assert_eq!(settings.port, 8080);
assert_eq!(settings.litellm_config, "/tmp/custom.yaml");
assert_eq!(settings.ui_path.as_deref(), Some("/srv/ui"));
assert_eq!(settings.log_level(), Level::DEBUG);
}
@ -106,23 +138,42 @@ mod tests {
#[case::filter("litellm=debug")]
#[case::invalid("invalid")]
fn unrecognized_log_levels_fall_back_to_info(#[case] value: &str) {
let settings: Settings = parse(&[("RUST_LOG", value)]).unwrap();
assert_eq!(settings.log_level(), Level::INFO);
assert_eq!(
boot(&[("RUST_LOG", value)]).unwrap().log_level(),
Level::INFO
);
}
#[rstest]
#[case::empty("")]
#[case::negative("-1")]
#[case::overflow("65536")]
#[case::invalid("http")]
fn invalid_ports_fail_startup(#[case] value: &str) {
assert!(parse::<Settings>(&[("PORT", value)]).is_err());
#[case::invalid("abc")]
#[case::empty("")]
#[case::blank(" ")]
fn invalid_ports_fail_startup_naming_the_variable(#[case] value: &str) {
let error = boot(&[("PORT", value)]).err().unwrap();
assert_eq!(error.name, "PORT");
assert!(error.to_string().contains("PORT"));
}
#[rstest]
fn blank_text_settings_remain_set() {
let settings = boot(&[
("HOST", ""),
("LITELLM_CONFIG", ""),
("LITELLM_UI_PATH", ""),
])
.unwrap();
assert_eq!(settings.host, "");
assert_eq!(settings.litellm_config, "");
assert_eq!(settings.port, 4000);
assert_eq!(settings.ui_path.as_deref(), Some(""));
}
#[rstest]
fn ui_defaults_keep_secure_cookies_and_preserve_password() {
let settings: UiSettings = parse(&[("UI_PASSWORD", " secret,with spaces ")]).unwrap();
let settings = ui(&[("UI_PASSWORD", " secret,with spaces ")]).unwrap();
assert_eq!(settings.ui_username, "admin");
assert_eq!(settings.ui_password.expose(), " secret,with spaces ");
@ -133,7 +184,7 @@ mod tests {
#[case::secure("true", true)]
#[case::insecure("false", false)]
fn ui_environment_overrides(#[case] value: &str, #[case] expected: bool) {
let settings: UiSettings = parse(&[
let settings = ui(&[
("UI_USERNAME", "operator"),
("UI_PASSWORD", "password"),
("LITELLM_UI_SECURE_COOKIES", value),
@ -148,7 +199,16 @@ mod tests {
#[case::missing_password(&[])]
#[case::invalid_cookie_flag(&[("UI_PASSWORD", "password"), ("LITELLM_UI_SECURE_COOKIES", "invalid")])]
fn invalid_ui_settings_fail(#[case] variables: &[(&str, &str)]) {
assert!(parse::<UiSettings>(variables).is_err());
assert!(ui(variables).is_err());
}
#[rstest]
#[case::empty("")]
#[case::whitespace(" ")]
fn blank_ui_credentials_remain_set(#[case] value: &str) {
let settings = ui(&[("UI_PASSWORD", value), ("UI_USERNAME", value)]).unwrap();
assert_eq!(settings.ui_password.expose(), value);
assert_eq!(settings.ui_username, value);
}
#[cfg(unix)]
@ -158,12 +218,12 @@ mod tests {
fn non_unicode_environment_values_are_ignored(#[case] key: &str) {
use std::os::unix::ffi::OsStringExt;
let settings: Settings = from_iter([
let env = Env::from_vars([
(OsString::from(key), OsString::from_vec(vec![0xff])),
(OsString::from_vec(vec![0xff]), OsString::from("value")),
(OsString::from("PORT"), OsString::from("8080")),
])
.unwrap();
]);
let settings = Settings::from(BootKeysPartial::from_env(&env).unwrap());
assert_eq!(settings.host, "0.0.0.0");
assert_eq!(settings.port, 8080);

View file

@ -126,7 +126,8 @@ mcp_servers:
upstream.url
))
.unwrap();
let inference = litellm_gateway::build_inference(&config).unwrap();
let inference =
litellm_gateway::build_inference(&config, &litellm_layered::Env::default()).unwrap();
let shutdown = CancellationToken::new();
let mcp = litellm_gateway::build_mcp(
&config,
@ -243,7 +244,8 @@ async fn rejects_unsupported_or_invalid_settings_before_serving(
#[case] expected: &str,
) {
let config = Config::from_yaml(&format!("general_settings: {{master_key: gateway-key}}\nmcp_servers:\n docs:\n url: http://127.0.0.1:1/mcp\n {settings}\n")).unwrap();
let inference = litellm_gateway::build_inference(&config).unwrap();
let inference =
litellm_gateway::build_inference(&config, &litellm_layered::Env::default()).unwrap();
let result = litellm_gateway::build_mcp(
&config,
Arc::new(NoSecrets),
@ -261,7 +263,8 @@ async fn rejects_unsupported_or_invalid_settings_before_serving(
#[tokio::test]
async fn mcp_is_optional_and_requires_admission_credentials_when_enabled() {
let empty = Config::from_yaml("{}").unwrap();
let inference = litellm_gateway::build_inference(&empty).unwrap();
let inference =
litellm_gateway::build_inference(&empty, &litellm_layered::Env::default()).unwrap();
assert!(
litellm_gateway::build_mcp(
&empty,
@ -311,7 +314,9 @@ fn applies_configured_http_environment_before_building_clients() {
missing.display(),
))
.unwrap();
let error = litellm_gateway::build_inference(&config).err().unwrap();
let error = litellm_gateway::build_inference(&config, &litellm_layered::Env::default())
.err()
.unwrap();
assert!(matches!(
error,
litellm_gateway::Error::Http(litellm_http::Error::Read { path, .. })
@ -329,7 +334,8 @@ async fn refuses_unimplemented_global_mcp_policy(#[case] settings: &str) {
"{settings}\nmcp_servers: {{docs: {{url: 'http://127.0.0.1:1/mcp'}}}}\n"
))
.unwrap();
let inference = litellm_gateway::build_inference(&config).unwrap();
let inference =
litellm_gateway::build_inference(&config, &litellm_layered::Env::default()).unwrap();
let error = litellm_gateway::build_mcp(
&config,
Arc::new(NoSecrets),

View file

@ -13,6 +13,7 @@ use rstest::{fixture, rstest};
use serde_json::{Value, json};
use tokio::{net::TcpListener, sync::oneshot, time::timeout};
use tower::ServiceExt;
use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method};
struct LogSink(mpsc::Sender<Value>);
@ -30,7 +31,11 @@ impl Sink for LogSink {
#[fixture]
fn inference() -> Arc<Gateway> {
litellm_gateway::build_inference(&Config::from_yaml("model_list: []").unwrap()).unwrap()
litellm_gateway::build_inference(
&Config::from_yaml("model_list: []").unwrap(),
&litellm_layered::Env::default(),
)
.unwrap()
}
#[rstest]
@ -113,6 +118,174 @@ async fn authenticates_before_serving_mounted_inference_routes(
.unwrap();
}
#[rstest]
#[tokio::test]
async fn ocr_gives_up_at_the_request_timeout_from_config_environment_variables() {
let upstream = MockServer::start().await;
Mock::given(method("POST"))
.respond_with(
ResponseTemplate::new(200)
.set_delay(Duration::from_secs(3))
.set_body_json(json!({"pages": []})),
)
.mount(&upstream)
.await;
let config = Config::from_yaml(&format!(
"model_list:
- model_name: public/ocr
litellm_params:
model: mistral/test-ocr
api_base: {}
api_key: test-key
general_settings:
master_key: gateway-key
environment_variables:
REQUEST_TIMEOUT: '0.2'
",
upstream.uri()
))
.unwrap();
let inference =
litellm_gateway::build_inference(&config, &litellm_layered::Env::default()).unwrap();
let request = Request::post("/ocr")
.header("authorization", "Bearer gateway-key")
.header("content-type", "application/json")
.body(Body::from(
json!({
"model": "public/ocr",
"document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"},
})
.to_string(),
))
.unwrap();
let response = timeout(
Duration::from_secs(2),
litellm_gateway::router(inference, &config, None, None).oneshot(request),
)
.await
.expect("OCR must give up at REQUEST_TIMEOUT instead of the default")
.unwrap();
assert_eq!(response.status().as_u16(), 408);
}
#[rstest]
#[tokio::test]
async fn messages_thinking_budgets_read_config_environment_variables() {
let upstream = MockServer::start().await;
Mock::given(method("POST"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-5",
"content": [{"type": "text", "text": "hi"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 1, "output_tokens": 1},
})))
.mount(&upstream)
.await;
let config = Config::from_yaml(&format!(
"model_list:
- model_name: public/claude
litellm_params:
model: anthropic/claude-sonnet-4-5
api_key: sk-ant-test
api_base: {}
general_settings:
master_key: gateway-key
environment_variables:
DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET: '2000'
",
upstream.uri()
))
.unwrap();
let inference =
litellm_gateway::build_inference(&config, &litellm_layered::Env::default()).unwrap();
let request = Request::post("/v1/messages")
.header("authorization", "Bearer gateway-key")
.header("content-type", "application/json")
.body(Body::from(
json!({
"model": "public/claude",
"max_tokens": 4096,
"reasoning_effort": "low",
"messages": [{"role": "user", "content": "hi"}],
})
.to_string(),
))
.unwrap();
let response = timeout(
Duration::from_secs(10),
litellm_gateway::router(inference, &config, None, None).oneshot(request),
)
.await
.unwrap()
.unwrap();
assert_eq!(response.status().as_u16(), 200);
let sent: Value = upstream.received_requests().await.unwrap()[0]
.body_json()
.unwrap();
assert_eq!(sent["thinking"]["budget_tokens"], 2000);
}
#[rstest]
#[tokio::test]
async fn messages_auth_reads_credentials_from_config_environment_variables() {
let directory = tempfile::tempdir().unwrap();
let credentials = directory.path().join("service-account.json");
std::fs::write(&credentials, "not a service account").unwrap();
let config = Config::from_yaml(&format!(
"model_list:
- model_name: public/vertex
litellm_params:
model: vertex_ai/claude-sonnet-4-5
vertex_project: test-project
vertex_location: us-east5
general_settings:
master_key: gateway-key
environment_variables:
GOOGLE_APPLICATION_CREDENTIALS: '{}'
",
credentials.display()
))
.unwrap();
let inference =
litellm_gateway::build_inference(&config, &litellm_layered::Env::default()).unwrap();
let request = Request::post("/v1/messages")
.header("authorization", "Bearer gateway-key")
.header("content-type", "application/json")
.body(Body::from(
json!({
"model": "public/vertex",
"max_tokens": 16,
"messages": [{"role": "user", "content": "hi"}],
})
.to_string(),
))
.unwrap();
let response = timeout(
Duration::from_secs(10),
litellm_gateway::router(inference, &config, None, None).oneshot(request),
)
.await
.unwrap()
.unwrap();
let body = axum::body::to_bytes(response.into_body(), 64 * 1024)
.await
.unwrap();
assert!(
String::from_utf8_lossy(&body).contains("ApplicationCredentials"),
"{}",
String::from_utf8_lossy(&body)
);
}
#[rstest]
#[tokio::test]
async fn logs_request_outcome_without_credentials_or_query(inference: Arc<Gateway>) {

View file

@ -6,6 +6,7 @@ license.workspace = true
repository.workspace = true
[dependencies]
litellm-layered.workspace = true
bytes.workspace = true
futures-util.workspace = true
litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] }

View file

@ -62,6 +62,7 @@ pub(super) async fn execute(
model,
call,
secrets.as_ref(),
route.env.as_ref(),
context,
)
.boxed()
@ -100,6 +101,7 @@ async fn provider_call(
model: String,
call: MessagesCall,
secrets: &(dyn Lookup + Send + Sync),
env: &(dyn Lookup + Send + Sync),
context: &CallContext<'_, impl Interceptors<Error>>,
) -> Result<ProviderCall, Error> {
let MessagesCall {
@ -143,7 +145,7 @@ async fn provider_call(
let body = config.transform_anthropic_messages_request(
trimmed,
&MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
&MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params, env),
)?;
let environment = ValidatedEnvironment {
headers: config.request_headers(
@ -168,7 +170,10 @@ async fn provider_call(
secret_fields: Vec::new(),
api_key: api_key.map(SecretValue::new),
};
let authenticated = resolve_auth(auth, environment, &|key| std::env::var(key).ok()).await?;
let authenticated = resolve_auth(auth, environment, &|key| {
secrets.get(key).or_else(|| env.get(key))
})
.await?;
let identity = ProviderIdentity {
model: request_context.model.clone(),
provider: request_context.custom_llm_provider.clone(),
@ -436,6 +441,7 @@ mod tests {
model,
call,
secrets,
&|_: &str| None,
&context,
)
.await

View file

@ -9,6 +9,7 @@ use litellm_auth::AuthServices;
use litellm_host::interceptors::Interceptors;
use litellm_inference::context::CallContext;
use litellm_inference::provider::resolve_llm_provider;
use litellm_layered::SharedLookup;
use litellm_secrets::source::SecretSource;
use std::sync::Arc;
@ -25,6 +26,7 @@ pub struct MessagesRoute {
http: litellm_http::Client,
auth: Arc<AuthServices>,
secrets: Arc<dyn SecretSource>,
env: SharedLookup,
cache: Option<litellm_cache_response::ScopedCache>,
}
@ -33,11 +35,13 @@ impl MessagesRoute {
http: litellm_http::Client,
auth: Arc<AuthServices>,
secrets: Arc<dyn SecretSource>,
env: SharedLookup,
) -> Self {
Self {
http,
auth,
secrets,
env,
cache: None,
}
}

View file

@ -69,7 +69,12 @@ impl Interceptors<Error> for LiveCall {
#[rstest::fixture]
pub(super) fn route() -> MessagesRoute {
let resources = LiveResources::default();
MessagesRoute::new(resources.http, resources.auth, resources.secrets)
MessagesRoute::new(
resources.http,
resources.auth,
resources.secrets,
std::sync::Arc::new(litellm_layered::Env::from_process()),
)
}
pub(super) async fn request(

View file

@ -277,6 +277,7 @@ async fn the_facade_sends_through_the_injected_http_pool_configuration(call: Mes
provider_http(&resources, &Resolution::from(&settings).config),
resources.auth,
no_secrets(),
std::sync::Arc::new(litellm_layered::Env::of(&[])),
)
.execute(
MessagesCall {
@ -360,6 +361,7 @@ async fn route_uses_injected_dependencies_and_optional_cache(
provider_http(&resources, &http_config()),
resources.auth.clone(),
Arc::new(RecordingSecrets::new([("ANTHROPIC_API_KEY", "route-key")])),
std::sync::Arc::new(litellm_layered::Env::of(&[])),
);
let route = if caching {
route.with_cache(ScopedCache::new(

View file

@ -22,6 +22,7 @@ pub fn messages_route(secrets: Arc<dyn SecretSource>) -> litellm_inference_messa
provider_http(&resources, &http_config()),
resources.auth,
secrets,
std::sync::Arc::new(litellm_layered::Env::of(&[])),
)
}

View file

@ -155,7 +155,7 @@ impl OcrConfigKind {
LlmProviders::AzureAi => (None, Some("AZURE_AI_API_BASE")),
_ => (None, None),
};
let secret = |name: &str| secrets.truthy(name);
let secret = |name: &str| secrets.get(name).filter(|value| !value.is_empty());
let dynamic_api_key = credentials.dynamic_api_key.or_else(|| {
credentials.api_key.clone().or_else(|| {
preferred_api_key_env

View file

@ -1,3 +1,4 @@
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
use std::{sync::Arc, time::Duration};
use futures_util::TryStreamExt;

View file

@ -32,11 +32,15 @@ impl Default for LiveResources {
Self {
http,
auth: resources.auth,
secrets: Arc::new(EnvironmentSecrets::python_compatible(secrets_http)),
secrets: Arc::new(EnvironmentSecrets::python_compatible(
secrets_http,
Arc::new(Env::from_process()),
)),
}
}
}
#[allow(clippy::disallowed_methods)] // live tests read their own model names
pub fn model(route: &str, provider: &str) -> String {
let name = format!("LITELLM_LIVE_{route}_{provider}_MODEL").to_ascii_uppercase();
std::env::var(&name)

View file

@ -17,6 +17,8 @@ Each runtime writes its own `merge([..])` list per group, and a test per runtime
- `from_env` and `from_litellm_params` are generated, and every runtime that has that source calls them
- A source holding typed values (the config file, a live Python global) gets a hand-written constructor in the crate that owns the source, never in `layered` or in the group's crate
- `layered` never depends on `litellm-config` or pyo3, and no library crate reads the process environment itself. It takes a `Lookup`: `Env` in the Rust gateway, `ProcessEnvironment` in `python-bridge`
- `ProcessEnvironment` is private to `python-bridge`, so a library constructor takes the `Lookup` as a required argument and never falls back to the process through `new()`, `Default` or an optional builder step
- An SDK that reads the process environment itself, like `aws-config`, gets every value that has an environment form from the `Lookup` first. Only sources with no environment form (profiles, ECS, IMDS) are left to the SDK
## Hand-written constructors list every field

View file

@ -1,4 +1,4 @@
use std::{collections::BTreeMap, fmt};
use std::{collections::BTreeMap, ffi::OsString, fmt};
use crate::Lookup;
@ -8,8 +8,12 @@ pub struct Env(BTreeMap<String, String>);
impl Env {
#[allow(clippy::disallowed_methods)]
pub fn from_process() -> Self {
Self::from_vars(std::env::vars_os())
}
pub fn from_vars(vars: impl IntoIterator<Item = (OsString, OsString)>) -> Self {
Self(
std::env::vars_os()
vars.into_iter()
.filter_map(|(name, value)| {
Some((name.into_string().ok()?, value.into_string().ok()?))
})

View file

@ -8,6 +8,8 @@ pub use litellm_core_utils::{Invalid, coercion::FromEnv, settings::Lookup};
pub use litellm_layered_derive::Layered;
pub use sources::Sources;
pub type SharedLookup = std::sync::Arc<dyn Lookup + Send + Sync>;
pub trait Partial: Default {
fn or(self, lower: Self) -> Self;
}

View file

@ -11,6 +11,7 @@ test-support = ["litellm-http/test-support"]
[dependencies]
litellm-llms-types.workspace = true
litellm-core-utils.workspace = true
litellm-layered.workspace = true
litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] }
litellm-auth-aws.workspace = true
litellm-auth-azure.workspace = true

View file

@ -371,10 +371,7 @@ pub fn map_openai_context_management_to_anthropic(
#[cfg(test)]
mod tests {
use crate::base_llm::messages::context::{
MessagesModelCapabilities, ThinkingBudgets, ThinkingContext,
};
use std::process::Command;
use crate::base_llm::messages::context::MessagesModelCapabilities;
use rstest::{fixture, rstest};
@ -388,7 +385,6 @@ mod tests {
const OAUTH_BETA: &str = "oauth-2025-04-20";
const BROWSER_ACCESS: (&str, &str) = ("anthropic-dangerous-direct-browser-access", "true");
const LOW_BUDGET_ENV: &str = "DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET";
const PROCESS_ENV_PROBE: &str = "LITELLM_MESSAGES_TRANSFORM_CONTEXT_PROBE";
fn merged(base: Value, fields: Value) -> Value {
Value::Object(
@ -443,7 +439,7 @@ mod tests {
ANTHROPIC_MESSAGES_CONFIG
.transform_anthropic_messages_request(
request(fields),
&MessagesTransformContext::with_lookup(capabilities, drop_params, &no_env),
&MessagesTransformContext::new(capabilities, drop_params, &no_env),
)
.map(|transformed| serde_json::to_value(transformed).unwrap())
}
@ -768,7 +764,7 @@ mod tests {
#[test]
fn thinking_is_translated_with_the_context_budgets() {
let context = MessagesTransformContext::with_lookup(
let context = MessagesTransformContext::new(
MessagesModelCapabilities {
supports_reasoning: true,
..Default::default()
@ -791,54 +787,6 @@ mod tests {
);
}
#[test]
fn new_reads_thinking_budgets_from_the_process_environment() {
if std::env::var_os(PROCESS_ENV_PROBE).is_some() {
assert_eq!(
MessagesTransformContext::new(sampling_removed(), true),
MessagesTransformContext {
thinking: ThinkingContext {
capabilities: sampling_removed(),
budgets: ThinkingBudgets {
low: 2000,
..ThinkingBudgets::default()
},
},
drop_params: true,
}
);
return;
}
let (_, test_path) = concat!(
module_path!(),
"::new_reads_thinking_budgets_from_the_process_environment"
)
.split_once("::")
.unwrap();
let other_tiers = ["MINIMAL", "MEDIUM", "HIGH", "XHIGH", "MAX"]
.map(|tier| format!("DEFAULT_REASONING_EFFORT_{tier}_THINKING_BUDGET"));
let output = other_tiers
.iter()
.fold(
Command::new(std::env::current_exe().unwrap()),
|mut command, name| {
command.env_remove(name);
command
},
)
.args([test_path, "--exact"])
.env(PROCESS_ENV_PROBE, "1")
.env(LOW_BUDGET_ENV, "2000")
.output()
.unwrap();
let stdout = String::from_utf8_lossy(&output.stdout);
assert!(
output.status.success() && stdout.contains("1 passed"),
"{stdout}{}",
String::from_utf8_lossy(&output.stderr)
);
}
#[rstest]
#[case::public_endpoint(None, &[], "https://api.anthropic.com/v1/messages")]
#[case::base_url_env(

View file

@ -379,7 +379,7 @@ mod tests {
"inference_geo": "us",
"litellm_metadata": {"trace": "abc"}
});
let context = MessagesTransformContext::with_lookup(
let context = MessagesTransformContext::new(
MessagesModelCapabilities {
supports_reasoning: true,
supports_adaptive_thinking: true,
@ -406,6 +406,7 @@ mod tests {
..MessagesModelCapabilities::default()
},
false,
&|_: &str| None,
)
}
@ -507,7 +508,7 @@ mod tests {
{"role": "assistant", "content": "hello"}
]
});
let context = MessagesTransformContext::with_lookup(
let context = MessagesTransformContext::new(
MessagesModelCapabilities {
supports_reasoning: true,
supports_adaptive_thinking: true,

View file

@ -1,4 +1,4 @@
use litellm_core_utils::settings::{Lookup, ProcessEnvironment};
use litellm_core_utils::settings::Lookup;
use serde::{Deserialize, Serialize};
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
@ -89,10 +89,11 @@ impl Default for ThinkingBudgets {
}
impl ThinkingBudgets {
pub fn from_lookup(env: &impl Lookup) -> Self {
pub fn from_lookup(env: &(impl Lookup + ?Sized)) -> Self {
let defaults = Self::default();
let tier = |name: &str, default: u64| {
env.parsed::<u64>(&format!("DEFAULT_REASONING_EFFORT_{name}_THINKING_BUDGET"))
env.get(&format!("DEFAULT_REASONING_EFFORT_{name}_THINKING_BUDGET"))
.and_then(|value| value.trim().parse().ok())
.unwrap_or(default)
};
Self {
@ -117,14 +118,10 @@ pub struct MessagesTransformContext {
pub drop_params: bool,
}
impl MessagesTransformContext {
pub fn new(capabilities: MessagesModelCapabilities, drop_params: bool) -> Self {
Self::with_lookup(capabilities, drop_params, &ProcessEnvironment)
}
pub fn with_lookup(
pub fn new(
capabilities: MessagesModelCapabilities,
drop_params: bool,
env: &impl Lookup,
env: &(impl Lookup + ?Sized),
) -> Self {
Self {
thinking: ThinkingContext {

View file

@ -91,6 +91,7 @@ impl OcrClient {
secrets: Arc::new(
litellm_secrets::source::EnvironmentSecrets::python_compatible(
provider_http.clone(),
Arc::new(|_: &str| None),
),
),
provider_http,

View file

@ -1,6 +1,7 @@
use std::time::Duration;
use litellm_core_utils::settings::Lookup;
use litellm_core_utils::coercion;
use litellm_layered::{Invalid, Layered};
#[derive(Clone, Debug, PartialEq)]
pub struct OcrSettings {
@ -25,70 +26,88 @@ impl Default for OcrSettings {
}
}
impl OcrSettings {
pub fn from_environment(env: &impl Lookup) -> Self {
#[derive(Layered)]
pub struct OcrKeys {
#[layer(env = "REQUEST_TIMEOUT", read = timeout_seconds)]
pub request_timeout: Duration,
#[layer(env = "MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", read = download_megabytes)]
pub max_download_bytes: u64,
#[layer(env = "AZURE_OPERATION_POLLING_TIMEOUT", read = polling_seconds)]
pub poll_timeout: Duration,
#[layer(env = "AZURE_DOCUMENT_INTELLIGENCE_API_VERSION")]
pub document_intelligence_api_version: String,
#[layer(env = "AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI")]
pub document_intelligence_dpi: i64,
pub enable_azure_ad_token_refresh: bool,
}
impl From<OcrKeysPartial> for OcrSettings {
fn from(merged: OcrKeysPartial) -> Self {
let defaults = Self::default();
Self {
request_timeout: env
.parsed::<f64>("REQUEST_TIMEOUT")
.and_then(|seconds| Duration::try_from_secs_f64(seconds).ok())
.unwrap_or(defaults.request_timeout),
max_download_bytes: env
.parsed::<f64>("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB")
.filter(|size| size.is_finite())
.map_or(defaults.max_download_bytes, megabytes),
poll_timeout: env
.parsed::<i64>("AZURE_OPERATION_POLLING_TIMEOUT")
.map_or(defaults.poll_timeout, |seconds| {
Duration::from_secs(seconds.max(0).unsigned_abs())
}),
document_intelligence_api_version: env
.get("AZURE_DOCUMENT_INTELLIGENCE_API_VERSION")
request_timeout: merged.request_timeout.unwrap_or(defaults.request_timeout),
max_download_bytes: merged
.max_download_bytes
.unwrap_or(defaults.max_download_bytes),
poll_timeout: merged.poll_timeout.unwrap_or(defaults.poll_timeout),
document_intelligence_api_version: merged
.document_intelligence_api_version
.unwrap_or(defaults.document_intelligence_api_version),
document_intelligence_dpi: env
.parsed("AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI")
document_intelligence_dpi: merged
.document_intelligence_dpi
.unwrap_or(defaults.document_intelligence_dpi),
..defaults
enable_azure_ad_token_refresh: merged
.enable_azure_ad_token_refresh
.unwrap_or(defaults.enable_azure_ad_token_refresh),
}
}
}
fn timeout_seconds(value: &str) -> Result<Option<Duration>, Invalid> {
Ok(coercion::parsed::<f64>(value)?
.and_then(|seconds| Duration::try_from_secs_f64(seconds).ok()))
}
fn download_megabytes(value: &str) -> Result<Option<u64>, Invalid> {
Ok(coercion::parsed::<f64>(value)?
.filter(|size| size.is_finite())
.map(megabytes))
}
fn polling_seconds(value: &str) -> Result<Option<Duration>, Invalid> {
Ok(coercion::parsed::<i64>(value)?
.map(|seconds| Duration::from_secs(seconds.max(0).unsigned_abs())))
}
fn megabytes(size: f64) -> u64 {
(size * 1024.0 * 1024.0) as u64
}
#[cfg(test)]
mod tests {
use litellm_layered::Env;
use rstest::rstest;
use super::*;
fn env_of(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
move |name| {
values
.iter()
.find(|(key, _)| *key == name)
.map(|(_, value)| value.to_string())
}
fn from_env(values: &[(&str, &str)]) -> OcrSettings {
OcrSettings::from(OcrKeysPartial::from_env(&Env::of(values)).unwrap())
}
#[test]
fn an_empty_environment_keeps_the_python_defaults() {
assert_eq!(
OcrSettings::from_environment(&env_of(&[])),
OcrSettings::default()
);
assert_eq!(from_env(&[]), OcrSettings::default());
}
#[test]
fn every_setting_follows_its_environment_variable() {
let settings = OcrSettings::from_environment(&env_of(&[
let settings = from_env(&[
("REQUEST_TIMEOUT", "30.5"),
("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", "0.5"),
("AZURE_OPERATION_POLLING_TIMEOUT", " 600 "),
("AZURE_DOCUMENT_INTELLIGENCE_API_VERSION", "2025-01-01"),
("AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI", "72"),
]));
]);
assert_eq!(
settings,
OcrSettings {
@ -111,30 +130,25 @@ mod tests {
#[case] value: &'static str,
#[case] bytes: u64,
) {
let env =
move |name: &str| (name == "MAX_IMAGE_URL_DOWNLOAD_SIZE_MB").then(|| value.to_string());
assert_eq!(
OcrSettings::from_environment(&env).max_download_bytes,
from_env(&[("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", value)]).max_download_bytes,
bytes
);
}
#[test]
fn a_negative_polling_timeout_expires_immediately() {
let env =
|name: &str| (name == "AZURE_OPERATION_POLLING_TIMEOUT").then(|| "-5".to_string());
assert_eq!(
OcrSettings::from_environment(&env).poll_timeout,
from_env(&[("AZURE_OPERATION_POLLING_TIMEOUT", "-5")]).poll_timeout,
Duration::ZERO
);
}
#[test]
fn an_empty_api_version_is_sent_as_is_like_python_str_of_getenv() {
let env =
|name: &str| (name == "AZURE_DOCUMENT_INTELLIGENCE_API_VERSION").then(String::new);
fn an_empty_api_version_shadows_the_default() {
assert_eq!(
OcrSettings::from_environment(&env).document_intelligence_api_version,
from_env(&[("AZURE_DOCUMENT_INTELLIGENCE_API_VERSION", "")])
.document_intelligence_api_version,
""
);
}

View file

@ -2,7 +2,7 @@ use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrDocument, OcrRespo
use std::{collections::BTreeMap, future::Future, sync::Arc, time::Duration};
use litellm_auth::{InputSource, SecretValue, Sourced, TokenProviderHandle};
use litellm_core_utils::{call_arguments::CallArguments, settings::ProcessEnvironment};
use litellm_core_utils::call_arguments::CallArguments;
use litellm_http::outbound::{OutboundRequest, RequestSigner};
use litellm_secrets::source::Secrets;
use serde::{
@ -147,7 +147,7 @@ impl Default for OcrConnection {
ResolvedOcrCredentials::default(),
OcrTransportConfig::default(),
OcrSettings::default(),
Arc::new(ProcessEnvironment),
Arc::new(|_: &str| None),
)
}
}
@ -455,7 +455,7 @@ mod tests {
..OcrTransportConfig::default()
},
settings.clone(),
Arc::new(ProcessEnvironment),
Arc::new(|_: &str| None),
)
.timeout
};

View file

@ -1,3 +1,4 @@
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
fn main() {
if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos") {
println!("cargo:rustc-cdylib-link-arg=-undefined");

View file

@ -100,6 +100,7 @@ impl NativeCacheHandle {
&RedisTopology::Standalone,
Some(ttl),
ResponseCacheCodec,
&crate::environment::ProcessEnvironment,
)
})
.map_err(cache_error)?

View file

@ -0,0 +1,10 @@
use litellm_core_utils::settings::Lookup;
pub(crate) struct ProcessEnvironment;
impl Lookup for ProcessEnvironment {
#[allow(clippy::disallowed_methods)] // the Python host's settings live in the process environment
fn get(&self, name: &str) -> Option<String> {
std::env::var(name).ok()
}
}

View file

@ -4,7 +4,7 @@ use std::{
sync::{Arc, LazyLock, Mutex, PoisonError},
};
use litellm_core_utils::settings::ProcessEnvironment;
use crate::environment::ProcessEnvironment;
use litellm_http::{
Client, ClientVariant, HttpClientConfig, HttpClientPool, HttpKeysPartial, HttpSettings,
Resolution, SslVerify, TlsSource, Unsupported,

View file

@ -3,6 +3,7 @@ mod callable;
mod coercion;
mod credentials;
mod diagnostics;
mod environment;
mod errors;
mod execution;
mod http;

View file

@ -1,3 +1,4 @@
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
use std::{process::Command, task::Poll};
use litellm_host::{

View file

@ -18,6 +18,7 @@ fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyR
.map_err(crate::http::client_error)?,
crate::http::resources().auth.clone(),
crate::secrets::source(py)?,
std::sync::Arc::new(crate::environment::ProcessEnvironment),
);
Ok(litellm_host::call::hosted_call(
request,

View file

@ -3,13 +3,14 @@ mod errors;
mod host;
mod project;
use crate::environment::ProcessEnvironment;
use host::OcrPythonHost;
use litellm_callbacks_legacy_python::LoggingOperation;
use litellm_core_utils::settings::ProcessEnvironment;
use litellm_host_python::to_py;
use litellm_inference_ocr::provider_config;
use litellm_llms::base_llm::ocr::settings::OcrSettings;
use pyo3::prelude::*;
use litellm_layered::merge;
use litellm_llms::base_llm::ocr::settings::{OcrKeysPartial, OcrSettings};
use pyo3::{exceptions::PyValueError, prelude::*};
use super::NativeCall;
@ -55,9 +56,21 @@ fn ocr_settings(py: Python<'_>) -> PyResult<OcrSettings> {
}
fn project_provider_defaults(snapshot: &Snapshot<'_>) -> PyResult<OcrSettings> {
Ok(OcrSettings {
enable_azure_ad_token_refresh: snapshot.read(&ENABLE_AZURE_AD_TOKEN_REFRESH)?,
..OcrSettings::from_environment(&ProcessEnvironment)
Ok(OcrSettings::from(merge([
OcrKeysPartial::from_env(&ProcessEnvironment)
.map_err(|error| PyValueError::new_err(error.to_string()))?,
globals(snapshot)?,
])))
}
fn globals(snapshot: &Snapshot<'_>) -> PyResult<OcrKeysPartial> {
Ok(OcrKeysPartial {
request_timeout: None,
max_download_bytes: None,
poll_timeout: None,
document_intelligence_api_version: None,
document_intelligence_dpi: None,
enable_azure_ad_token_refresh: Some(snapshot.read(&ENABLE_AZURE_AD_TOKEN_REFRESH)?),
})
}

View file

@ -29,6 +29,7 @@ pub(crate) fn source(py: Python<'_>) -> PyResult<Arc<dyn SecretSource>> {
let Some(snapshot) = PythonSettings::SecretManager.read_or_unset(py)? else {
return Ok(Arc::new(EnvironmentSecrets::python_compatible(
crate::http::host_client(py, litellm_http::ClientVariant::Provider)?,
Arc::new(crate::environment::ProcessEnvironment),
)));
};
if snapshot.read(&NATIVE)? {

View file

@ -1,7 +1,7 @@
use std::sync::Arc;
use crate::environment::ProcessEnvironment;
use futures_util::future::BoxFuture;
use litellm_core_utils::settings::ProcessEnvironment;
use litellm_host_python::PythonContext;
use litellm_http::Client;
use litellm_secrets::source::SecretSource;

View file

@ -1,6 +1,8 @@
use std::{collections::BTreeMap, sync::Arc};
use litellm_core_utils::settings::{Lookup, ProcessEnvironment};
use litellm_core_utils::settings::Lookup;
use crate::environment::ProcessEnvironment;
use litellm_host_python::{from_py, json_object_field, to_py};
use litellm_secrets::{
KeyManagementSettings, KeyManagementSystem, Secret, SecretManager, load_native_manager,

View file

@ -1,3 +1,4 @@
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
//! Replays `generated/values.json`, which CPython wrote with `scripts/generate_fixtures.py`.
use std::{collections::BTreeMap, fs::File, io::Write};

View file

@ -1,3 +1,4 @@
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
use super::*;
#[rstest]

View file

@ -1,6 +1,6 @@
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
use std::sync::Arc;
use litellm_core_utils::settings::ProcessEnvironment;
use litellm_secrets_azure::AzureKeyVault;
use litellm_secrets_types::Secret;
use rstest::rstest;
@ -9,7 +9,7 @@ use rstest::rstest;
#[tokio::test]
#[ignore]
async fn reads_a_real_secret() {
let environment = Arc::new(ProcessEnvironment);
let environment = Arc::new(|name: &str| std::env::var(name).ok());
let manager = AzureKeyVault::new(litellm_http::Client::plain_for_test(), environment).unwrap();
let name = std::env::var("AZURE_KEY_VAULT_LIVE_SECRET_NAME").unwrap();
let secret = manager.get_secret(&name).await.unwrap().unwrap();

View file

@ -1,3 +1,4 @@
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
use super::*;
#[rstest]

View file

@ -1,3 +1,4 @@
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
use super::*;
#[rstest]
@ -282,7 +283,7 @@ async fn rotation_applies_timeout_to_each_request(token_values: Vec<(&str, &str)
#[ignore]
async fn live_vault_round_trip() {
let environment: Arc<dyn Lookup + Send + Sync> =
Arc::new(litellm_core_utils::settings::ProcessEnvironment);
Arc::new(|name: &str| std::env::var(name).ok());
let manager: HashicorpVault = HashicorpVault::new(environment, true).unwrap();
let name: String = std::env::var("LITELLM_VAULT_LIVE_SECRET_NAME").unwrap();
let value: SecretValue = SecretValue::new("native-live-value");

View file

@ -40,10 +40,10 @@ impl SecretSource for SecretResolver {
pub struct EnvironmentSecrets(SecretResolver);
impl EnvironmentSecrets {
pub fn python_compatible(client: litellm_http::Client) -> Self {
pub fn python_compatible(client: litellm_http::Client, env: Secrets) -> Self {
Self(SecretResolver::new_python_compatible(
Arc::new(crate::SecretManagerState::default()),
Arc::new(litellm_core_utils::settings::ProcessEnvironment),
env,
crate::OidcResolver::new(client),
))
}

View file

@ -11,16 +11,17 @@ mod tests {
#[tokio::test]
async fn python_environment_values_are_absent_like_get_secret_str(
#[case] name: &'static str,
#[case] value: &str,
#[case] value: &'static str,
#[case] expected: Option<&str>,
) {
unsafe { std::env::set_var(name, value) };
let secret = EnvironmentSecrets::python_compatible(litellm_http::Client::plain_for_test())
.resolve(&[name])
.await
.unwrap()
.get(name);
unsafe { std::env::remove_var(name) };
let secret = EnvironmentSecrets::python_compatible(
litellm_http::Client::plain_for_test(),
std::sync::Arc::new(move |key: &str| (key == name).then(|| value.to_owned())),
)
.resolve(&[name])
.await
.unwrap()
.get(name);
assert_eq!(secret.as_deref(), expected);
}
}

View file

@ -15,6 +15,7 @@ use time::{OffsetDateTime, format_description::well_known::Rfc3339};
use crate::Error;
use litellm_storage_clickhouse::Connection;
#[allow(clippy::disallowed_methods)] // the ClickHouse sink is not built from an injected Lookup yet
fn max_insert_bytes() -> Result<usize, Error> {
let name = "CLICKHOUSE_TRACE_MAX_INSERT_BYTES";
match std::env::var(name) {

View file

@ -1,3 +1,4 @@
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
use std::{collections::BTreeMap, time::Duration};
use litellm_http::Client;

View file

@ -5,6 +5,7 @@ use litellm_http::Client;
use crate::{Connection, Error, valid_identifier};
#[allow(clippy::disallowed_methods)] // the ClickHouse sink is not built from an injected Lookup yet
fn insert_timeout() -> Result<Duration, Error> {
let name = "CLICKHOUSE_INSERT_TIMEOUT_SECONDS";
match std::env::var(name) {

View file

@ -1,3 +1,4 @@
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
use std::collections::BTreeMap;
use litellm_http::Client;

View file

@ -19,6 +19,7 @@ impl HttpFetch {
}
}
#[allow(clippy::disallowed_methods)] // test tooling reads its own token
pub fn from_env() -> Self {
Self::new(std::env::var("GITHUB_TOKEN").ok())
}

View file

@ -1,3 +1,4 @@
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
//! Drives the real agents through a real gateway. Run with `cargo test -p litellm-testkit --test live -- --ignored`
//! after exporting `TESTKIT_GATEWAY_URL`, `TESTKIT_GATEWAY_KEY`, one `TESTKIT_MODEL_<WIRE>` per wire
//! (`MESSAGES`, `RESPONSES`, `CHAT_COMPLETIONS`) and one `TESTKIT_<AGENT>_VERSION` per agent