mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(gateway): read boot, OCR and config env settings through one injected Env
The gateway takes one Env snapshot, overlays config environment_variables on it and injects it into every reader, so OCR REQUEST_TIMEOUT, Vertex credentials, Messages thinking budgets and AWS credentials now see config env vars. Boot and UI settings move to Layered and envy is dropped. clippy.toml bans std::env::var ProcessEnvironment is private to python-bridge, so library constructors take the lookup as a required argument instead of defaulting to the process (MessagesTransformContext, RedisCache, EnvironmentSecrets, OcrConnection). auth-aws reads every env-backed credential source through the lookup before handing the rest to the SDK chain, so the assume-role shortcut and the credentials it returns agree Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
9e3f422312
commit
b61449e35d
78 changed files with 891 additions and 441 deletions
21
litellm-rust/Cargo.lock
generated
21
litellm-rust/Cargo.lock
generated
|
|
@ -1911,15 +1911,6 @@ dependencies = [
|
|||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "envy"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f47e0157f2cb54f5ae1bd371b30a2ae4311e1c028f575cd4e81de7353215965"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "equivalent"
|
||||
version = "1.0.2"
|
||||
|
|
@ -3381,9 +3372,11 @@ dependencies = [
|
|||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"veil",
|
||||
"wiremock",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3460,6 +3453,7 @@ dependencies = [
|
|||
"litellm-cache-response",
|
||||
"litellm-cache-testing",
|
||||
"litellm-http",
|
||||
"litellm-layered",
|
||||
"reqwest 0.12.28",
|
||||
"rstest",
|
||||
"serde_json",
|
||||
|
|
@ -3494,6 +3488,7 @@ dependencies = [
|
|||
"litellm-cache",
|
||||
"litellm-cache-testing",
|
||||
"litellm-http",
|
||||
"litellm-layered",
|
||||
"percent-encoding",
|
||||
"reqwest 0.12.28",
|
||||
"rstest",
|
||||
|
|
@ -3540,6 +3535,7 @@ version = "0.1.0"
|
|||
dependencies = [
|
||||
"litellm-cache",
|
||||
"litellm-cache-testing",
|
||||
"litellm-layered",
|
||||
"r2d2",
|
||||
"redis",
|
||||
"redis-test",
|
||||
|
|
@ -3598,6 +3594,7 @@ dependencies = [
|
|||
"litellm-cache",
|
||||
"litellm-cache-testing",
|
||||
"litellm-http",
|
||||
"litellm-layered",
|
||||
"reqwest 0.12.28",
|
||||
"rstest",
|
||||
"serde_json",
|
||||
|
|
@ -3732,11 +3729,11 @@ version = "0.1.0"
|
|||
dependencies = [
|
||||
"axum",
|
||||
"base64 0.22.1",
|
||||
"envy",
|
||||
"futures-util",
|
||||
"http-body-util",
|
||||
"litellm-auth-types",
|
||||
"litellm-config",
|
||||
"litellm-core-utils",
|
||||
"litellm-gateway-auth",
|
||||
"litellm-gateway-inference",
|
||||
"litellm-gateway-mcp",
|
||||
|
|
@ -3759,6 +3756,7 @@ dependencies = [
|
|||
"tower-sessions-moka-store",
|
||||
"tracing",
|
||||
"uuid",
|
||||
"wiremock",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3803,6 +3801,7 @@ dependencies = [
|
|||
"litellm-inference-ocr",
|
||||
"litellm-inference-responses",
|
||||
"litellm-inference-transcription",
|
||||
"litellm-layered",
|
||||
"litellm-llms",
|
||||
"litellm-llms-types",
|
||||
"litellm-router",
|
||||
|
|
@ -4037,6 +4036,7 @@ dependencies = [
|
|||
"litellm-http",
|
||||
"litellm-inference",
|
||||
"litellm-inference-testing",
|
||||
"litellm-layered",
|
||||
"litellm-llms",
|
||||
"litellm-llms-types",
|
||||
"litellm-router-types",
|
||||
|
|
@ -4184,6 +4184,7 @@ dependencies = [
|
|||
"litellm-framer",
|
||||
"litellm-host",
|
||||
"litellm-http",
|
||||
"litellm-layered",
|
||||
"litellm-llms-types",
|
||||
"litellm-python-compat",
|
||||
"litellm-router-types",
|
||||
|
|
|
|||
|
|
@ -19,6 +19,10 @@ disallowed-methods = [
|
|||
{ path = "sqlx::query_as_with", reason = "use sqlx::query_as! or query_file_as! so the SQL is checked against the migrated schema" },
|
||||
{ path = "sqlx::query_scalar_with", reason = "use sqlx::query_scalar! so the SQL is checked against the migrated schema" },
|
||||
{ path = "sqlx::raw_sql", reason = "raw_sql is unchecked; use the checked query macros" },
|
||||
{ path = "std::env::var", reason = "take a Lookup: Env in the Rust gateway, ProcessEnvironment in python-bridge" },
|
||||
{ path = "std::env::var_os", reason = "take a Lookup: Env in the Rust gateway, ProcessEnvironment in python-bridge" },
|
||||
{ path = "std::env::vars", reason = "take a Lookup: Env in the Rust gateway, ProcessEnvironment in python-bridge" },
|
||||
{ path = "std::env::vars_os", reason = "take a Lookup: Env in the Rust gateway, ProcessEnvironment in python-bridge" },
|
||||
]
|
||||
|
||||
# Every outbound client comes from litellm_http::HttpClientPool so it honors the host's TLS,
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ aws-smithy-runtime-api = "1.13.0"
|
|||
|
||||
[dev-dependencies]
|
||||
rstest.workspace = true
|
||||
tempfile.workspace = true
|
||||
wiremock.workspace = true
|
||||
litellm-http = { workspace = true, features = ["test-support"] }
|
||||
reqwest.workspace = true
|
||||
tokio.workspace = true
|
||||
|
|
|
|||
|
|
@ -17,10 +17,11 @@ use litellm_auth_types::AwsParams;
|
|||
|
||||
use super::Error;
|
||||
use super::constants::{
|
||||
AWS_ACCESS_KEY_ID, AWS_EXTERNAL_ID, AWS_PROFILE_NAME, AWS_REGION_NAME, AWS_ROLE_ARN,
|
||||
AWS_ROLE_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_NAME, AWS_SESSION_TOKEN,
|
||||
AWS_SIGNED_HEADER_NAMES, AWS_STS_ENDPOINT, AWS_WEB_IDENTITY_TOKEN, AWS_WEB_IDENTITY_TOKEN_FILE,
|
||||
DEFAULT_BEDROCK_REGION, DEFAULT_SESSION_NAME_PREFIX, SIGV4_COMPUTED_HEADER_NAMES,
|
||||
AWS_ACCESS_KEY_ID, AWS_EXTERNAL_ID, AWS_PROFILE, AWS_PROFILE_NAME, AWS_REGION_NAME,
|
||||
AWS_ROLE_ARN, AWS_ROLE_NAME, AWS_ROLE_SESSION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_NAME,
|
||||
AWS_SESSION_TOKEN, AWS_SIGNED_HEADER_NAMES, AWS_STS_ENDPOINT, AWS_WEB_IDENTITY_TOKEN,
|
||||
AWS_WEB_IDENTITY_TOKEN_FILE, DEFAULT_BEDROCK_REGION, DEFAULT_SESSION_NAME_PREFIX,
|
||||
SIGV4_COMPUTED_HEADER_NAMES,
|
||||
};
|
||||
|
||||
const STATIC_CREDENTIALS_TTL: Duration = Duration::from_secs(3600 - 60);
|
||||
|
|
@ -274,46 +275,26 @@ impl AwsAuthService {
|
|||
.map_err(|error| Error::AwsProfile(error.to_string()))
|
||||
}
|
||||
AwsAuthFlow::AssumeRole { role, session_name } => {
|
||||
if is_already_running_as_role(&role, &resolved).await? {
|
||||
let ambient_flow = AwsAuthFlow::DefaultChain;
|
||||
let key = cache_key(&resolved, &ambient_flow);
|
||||
if let Some(credentials) = self.get_cached_credentials(&key) {
|
||||
return Ok(credentials);
|
||||
}
|
||||
let provider =
|
||||
aws_config::default_provider::credentials::DefaultCredentialsChain::builder()
|
||||
.build()
|
||||
.await;
|
||||
let credentials = provider
|
||||
.provide_credentials()
|
||||
.await
|
||||
.map_err(|error| Error::AwsDefaultChain(error.to_string()))?;
|
||||
self.set_cached_credentials(
|
||||
key,
|
||||
credentials.clone(),
|
||||
credential_cache_ttl(&ambient_flow).unwrap_or(AMBIENT_CREDENTIALS_TTL),
|
||||
);
|
||||
return Ok(credentials);
|
||||
if Box::pin(self.ambient_is_role(&role, &resolved, env_lookup)).await {
|
||||
return Box::pin(self.ambient_credentials(&resolved, env_lookup)).await;
|
||||
}
|
||||
let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest());
|
||||
if let Some(region) = resolved.region_name.clone() {
|
||||
loader = loader.region(aws_types::region::Region::new(region));
|
||||
}
|
||||
if let Some(endpoint) = resolved.sts_endpoint.clone() {
|
||||
loader = loader.endpoint_url(endpoint);
|
||||
}
|
||||
if let (Some(access_key_id), Some(secret_access_key)) =
|
||||
(resolved.access_key_id, resolved.secret_access_key)
|
||||
{
|
||||
loader = loader.credentials_provider(Credentials::new(
|
||||
let source = match (
|
||||
resolved.access_key_id.clone(),
|
||||
resolved.secret_access_key.clone(),
|
||||
) {
|
||||
(Some(access_key_id), Some(secret_access_key)) => Credentials::new(
|
||||
access_key_id,
|
||||
secret_access_key,
|
||||
resolved.session_token,
|
||||
resolved.session_token.clone(),
|
||||
None,
|
||||
"litellm-role-source",
|
||||
));
|
||||
}
|
||||
let sdk_config = loader.load().await;
|
||||
),
|
||||
_ => Box::pin(self.ambient_credentials(&resolved, env_lookup)).await?,
|
||||
};
|
||||
let sdk_config = sts_config(&resolved)
|
||||
.credentials_provider(source)
|
||||
.load()
|
||||
.await;
|
||||
let builder = aws_config::sts::AssumeRoleProvider::builder(role);
|
||||
let builder = match session_name {
|
||||
Some(name) => builder.session_name(name),
|
||||
|
|
@ -334,92 +315,175 @@ impl AwsAuthService {
|
|||
role,
|
||||
session_name,
|
||||
} => {
|
||||
let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest());
|
||||
if let Some(region) = resolved.region_name {
|
||||
loader = loader.region(aws_types::region::Region::new(region));
|
||||
}
|
||||
if let Some(endpoint) = resolved.sts_endpoint {
|
||||
loader = loader.endpoint_url(endpoint);
|
||||
}
|
||||
let sdk_config = loader.load().await;
|
||||
let client = aws_sdk_sts::Client::new(&sdk_config);
|
||||
let response = client
|
||||
.assume_role_with_web_identity()
|
||||
.role_arn(role)
|
||||
.role_session_name(session_name)
|
||||
.web_identity_token(token)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| Error::AwsWebIdentity(error.to_string()))?;
|
||||
let credentials = response
|
||||
.credentials()
|
||||
.ok_or(Error::AwsMissingWebIdentityCredentials)?;
|
||||
let expiration = SystemTime::try_from(*credentials.expiration())
|
||||
.map_err(|error| Error::AwsWebIdentityExpiration(error.to_string()))?;
|
||||
Ok(Credentials::new(
|
||||
credentials.access_key_id(),
|
||||
credentials.secret_access_key(),
|
||||
Some(credentials.session_token().to_string()),
|
||||
Some(expiration),
|
||||
"litellm-web-identity",
|
||||
Box::pin(web_identity_credentials(
|
||||
&resolved,
|
||||
role,
|
||||
session_name,
|
||||
token,
|
||||
))
|
||||
.await
|
||||
}
|
||||
AwsAuthFlow::DefaultChain => {
|
||||
let key = cache_key(&resolved, &AwsAuthFlow::DefaultChain);
|
||||
if let Some(credentials) = self.get_cached_credentials(&key) {
|
||||
return Ok(credentials);
|
||||
}
|
||||
let provider =
|
||||
aws_config::default_provider::credentials::DefaultCredentialsChain::builder()
|
||||
.build()
|
||||
.await;
|
||||
let credentials = provider
|
||||
.provide_credentials()
|
||||
.await
|
||||
.map_err(|error| Error::AwsDefaultChain(error.to_string()))?;
|
||||
self.set_cached_credentials(
|
||||
key,
|
||||
credentials.clone(),
|
||||
credential_cache_ttl(&AwsAuthFlow::DefaultChain)
|
||||
.unwrap_or(AMBIENT_CREDENTIALS_TTL),
|
||||
);
|
||||
Ok(credentials)
|
||||
Box::pin(self.ambient_credentials(&resolved, env_lookup)).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn ambient_credentials(
|
||||
&self,
|
||||
resolved: &AwsAuthConfig,
|
||||
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
|
||||
) -> Result<Credentials, Error> {
|
||||
let key = cache_key(resolved, &AwsAuthFlow::DefaultChain);
|
||||
if let Some(credentials) = self.get_cached_credentials(&key) {
|
||||
return Ok(credentials);
|
||||
}
|
||||
let credentials = match ambient(env_lookup) {
|
||||
Ambient::Keys(credentials) => credentials,
|
||||
Ambient::WebIdentityFile {
|
||||
role_arn,
|
||||
token_file,
|
||||
session_name,
|
||||
} => {
|
||||
let token = std::fs::read_to_string(&token_file)
|
||||
.map_err(|error| Error::AwsWebIdentity(format!("{token_file}: {error}")))?;
|
||||
web_identity_credentials(resolved, role_arn, session_name, token).await?
|
||||
}
|
||||
Ambient::Chain { profile_name } => {
|
||||
let builder =
|
||||
aws_config::default_provider::credentials::DefaultCredentialsChain::builder();
|
||||
let builder = match profile_name {
|
||||
Some(name) => builder.profile_name(&name),
|
||||
None => builder,
|
||||
};
|
||||
builder
|
||||
.build()
|
||||
.await
|
||||
.provide_credentials()
|
||||
.await
|
||||
.map_err(|error| Error::AwsDefaultChain(error.to_string()))?
|
||||
}
|
||||
};
|
||||
self.set_cached_credentials(
|
||||
key,
|
||||
credentials.clone(),
|
||||
credential_cache_ttl(&AwsAuthFlow::DefaultChain).unwrap_or(AMBIENT_CREDENTIALS_TTL),
|
||||
);
|
||||
Ok(credentials)
|
||||
}
|
||||
|
||||
async fn ambient_is_role(
|
||||
&self,
|
||||
role: &str,
|
||||
resolved: &AwsAuthConfig,
|
||||
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
|
||||
) -> bool {
|
||||
if role_identity(role).is_none() {
|
||||
return false;
|
||||
}
|
||||
if let Ambient::WebIdentityFile { role_arn, .. } = ambient(env_lookup) {
|
||||
return same_role_arns(role, &role_arn);
|
||||
}
|
||||
let Ok(credentials) = self.ambient_credentials(resolved, env_lookup).await else {
|
||||
return false;
|
||||
};
|
||||
let sdk_config = sts_config(resolved)
|
||||
.credentials_provider(credentials)
|
||||
.load()
|
||||
.await;
|
||||
aws_sdk_sts::Client::new(&sdk_config)
|
||||
.get_caller_identity()
|
||||
.send()
|
||||
.await
|
||||
.is_ok_and(|response| {
|
||||
response
|
||||
.arn()
|
||||
.is_some_and(|caller| same_role_arns(role, caller))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async fn is_already_running_as_role(role: &str, config: &AwsAuthConfig) -> Result<bool, Error> {
|
||||
if role_identity(role).is_none() {
|
||||
return Ok(false);
|
||||
}
|
||||
if let (Ok(current_role), Ok(token_file)) = (
|
||||
std::env::var(AWS_ROLE_ARN),
|
||||
std::env::var(AWS_WEB_IDENTITY_TOKEN_FILE),
|
||||
) && !token_file.is_empty()
|
||||
{
|
||||
return Ok(same_role_arns(role, ¤t_role));
|
||||
}
|
||||
/// What the SDK's default credentials chain would find, read through the injected lookup so a
|
||||
/// config overlay reaches it. Only sources with no environment form are left to the SDK chain.
|
||||
enum Ambient {
|
||||
Keys(Credentials),
|
||||
WebIdentityFile {
|
||||
role_arn: String,
|
||||
token_file: String,
|
||||
session_name: String,
|
||||
},
|
||||
Chain {
|
||||
profile_name: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest());
|
||||
if let Some(region) = config.region_name.clone() {
|
||||
loader = loader.region(aws_types::region::Region::new(region));
|
||||
fn ambient(env_lookup: &(dyn Fn(&str) -> Option<String> + Sync)) -> Ambient {
|
||||
if let (Some(access_key_id), Some(secret_access_key)) = (
|
||||
env_lookup(AWS_ACCESS_KEY_ID),
|
||||
env_lookup(AWS_SECRET_ACCESS_KEY),
|
||||
) {
|
||||
return Ambient::Keys(Credentials::new(
|
||||
access_key_id,
|
||||
secret_access_key,
|
||||
env_lookup(AWS_SESSION_TOKEN),
|
||||
None,
|
||||
"litellm-environment",
|
||||
));
|
||||
}
|
||||
if let Some(endpoint) = config.sts_endpoint.clone() {
|
||||
loader = loader.endpoint_url(endpoint);
|
||||
if let (Some(role_arn), Some(token_file)) = (
|
||||
env_lookup(AWS_ROLE_ARN),
|
||||
env_lookup(AWS_WEB_IDENTITY_TOKEN_FILE).filter(|file| !file.is_empty()),
|
||||
) {
|
||||
return Ambient::WebIdentityFile {
|
||||
role_arn,
|
||||
token_file,
|
||||
session_name: env_lookup(AWS_ROLE_SESSION_NAME).unwrap_or_else(default_session_name),
|
||||
};
|
||||
}
|
||||
let sdk_config = loader.load().await;
|
||||
let response = match aws_sdk_sts::Client::new(&sdk_config)
|
||||
.get_caller_identity()
|
||||
Ambient::Chain {
|
||||
profile_name: env_lookup(AWS_PROFILE),
|
||||
}
|
||||
}
|
||||
|
||||
fn sts_config(resolved: &AwsAuthConfig) -> aws_config::ConfigLoader {
|
||||
let loader = aws_config::defaults(aws_config::BehaviorVersion::latest());
|
||||
let loader = match resolved.region_name.clone() {
|
||||
Some(region) => loader.region(aws_types::region::Region::new(region)),
|
||||
None => loader,
|
||||
};
|
||||
match resolved.sts_endpoint.clone() {
|
||||
Some(endpoint) => loader.endpoint_url(endpoint),
|
||||
None => loader,
|
||||
}
|
||||
}
|
||||
|
||||
async fn web_identity_credentials(
|
||||
resolved: &AwsAuthConfig,
|
||||
role: String,
|
||||
session_name: String,
|
||||
token: String,
|
||||
) -> Result<Credentials, Error> {
|
||||
let sdk_config = sts_config(resolved).load().await;
|
||||
let response = aws_sdk_sts::Client::new(&sdk_config)
|
||||
.assume_role_with_web_identity()
|
||||
.role_arn(role)
|
||||
.role_session_name(session_name)
|
||||
.web_identity_token(token)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(response) => response,
|
||||
Err(_) => return Ok(false),
|
||||
};
|
||||
Ok(response
|
||||
.arn()
|
||||
.is_some_and(|caller| same_role_arns(role, caller)))
|
||||
.map_err(|error| Error::AwsWebIdentity(error.to_string()))?;
|
||||
let credentials = response
|
||||
.credentials()
|
||||
.ok_or(Error::AwsMissingWebIdentityCredentials)?;
|
||||
let expiration = SystemTime::try_from(*credentials.expiration())
|
||||
.map_err(|error| Error::AwsWebIdentityExpiration(error.to_string()))?;
|
||||
Ok(Credentials::new(
|
||||
credentials.access_key_id(),
|
||||
credentials.secret_access_key(),
|
||||
Some(credentials.session_token().to_string()),
|
||||
Some(expiration),
|
||||
"litellm-web-identity",
|
||||
))
|
||||
}
|
||||
|
||||
fn default_session_name() -> String {
|
||||
|
|
@ -829,6 +893,66 @@ mod tests {
|
|||
assert_eq!(credentials.session_token(), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lookup_keys_are_the_ambient_credentials_without_a_region() {
|
||||
let env = |name: &str| match name {
|
||||
AWS_ACCESS_KEY_ID => Some("lookup-ak".to_owned()),
|
||||
AWS_SECRET_ACCESS_KEY => Some("lookup-sk".to_owned()),
|
||||
_ => None,
|
||||
};
|
||||
let credentials = AwsAuthService::default()
|
||||
.resolve_credentials(AwsAuthConfig::default(), &env)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
(credentials.access_key_id(), credentials.secret_access_key()),
|
||||
("lookup-ak", "lookup-sk")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_role_the_lookup_web_identity_names_is_assumed_with_that_token() {
|
||||
let role = "arn:aws:iam::123456789012:role/A";
|
||||
let sts = wiremock::MockServer::start().await;
|
||||
wiremock::Mock::given(wiremock::matchers::body_string_contains(
|
||||
"Action=AssumeRoleWithWebIdentity",
|
||||
))
|
||||
.and(wiremock::matchers::body_string_contains(
|
||||
"WebIdentityToken=lookup-token",
|
||||
))
|
||||
.respond_with(wiremock::ResponseTemplate::new(200).set_body_string(
|
||||
"<AssumeRoleWithWebIdentityResponse><AssumeRoleWithWebIdentityResult><Credentials>\
|
||||
<AccessKeyId>sts-ak</AccessKeyId><SecretAccessKey>sts-sk</SecretAccessKey>\
|
||||
<SessionToken>sts-token</SessionToken><Expiration>2099-01-01T00:00:00Z</Expiration>\
|
||||
</Credentials></AssumeRoleWithWebIdentityResult></AssumeRoleWithWebIdentityResponse>",
|
||||
))
|
||||
.mount(&sts)
|
||||
.await;
|
||||
let token_file = tempfile::NamedTempFile::new().unwrap();
|
||||
std::fs::write(token_file.path(), "lookup-token").unwrap();
|
||||
let token_path = token_file.path().to_string_lossy().into_owned();
|
||||
let env = |name: &str| match name {
|
||||
AWS_ROLE_ARN => Some(role.to_owned()),
|
||||
AWS_WEB_IDENTITY_TOKEN_FILE => Some(token_path.clone()),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
let credentials = AwsAuthService::default()
|
||||
.resolve_credentials(
|
||||
AwsAuthConfig {
|
||||
role_name: Some(role.into()),
|
||||
region_name: Some("us-east-1".into()),
|
||||
sts_endpoint: Some(sts.uri()),
|
||||
..Default::default()
|
||||
},
|
||||
&env,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(credentials.access_key_id(), "sts-ak");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_policy_matches_python_flows() {
|
||||
assert_eq!(
|
||||
|
|
@ -1063,6 +1187,7 @@ mod tests {
|
|||
|
||||
#[ignore]
|
||||
#[tokio::test]
|
||||
#[allow(clippy::disallowed_methods)] // the live test reads its own credentials
|
||||
async fn live_bedrock_invoke_model_returns_200() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let access_key_id = std::env::var("AWS_BEDROCK_TEST_ACCESS_KEY_ID")?;
|
||||
let secret_access_key = std::env::var("AWS_BEDROCK_TEST_SECRET_ACCESS_KEY")?;
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ pub const AWS_PROFILE_NAME: &str = AwsParams::PROFILE_NAME.env[0];
|
|||
pub const AWS_ROLE_NAME: &str = AwsParams::ROLE_NAME.env[0];
|
||||
pub const AWS_WEB_IDENTITY_TOKEN: &str = AwsParams::WEB_IDENTITY_TOKEN.env[0];
|
||||
pub const AWS_ROLE_ARN: &str = "AWS_ROLE_ARN";
|
||||
pub const AWS_ROLE_SESSION_NAME: &str = "AWS_ROLE_SESSION_NAME";
|
||||
pub const AWS_PROFILE: &str = "AWS_PROFILE";
|
||||
pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = "AWS_WEB_IDENTITY_TOKEN_FILE";
|
||||
pub const AWS_STS_ENDPOINT: &str = AwsParams::STS_ENDPOINT.env[0];
|
||||
pub const AWS_EXTERNAL_ID: &str = AwsParams::EXTERNAL_ID.env[0];
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ litellm-http.workspace = true
|
|||
litellm-auth-azure.workspace = true
|
||||
litellm-auth-types.workspace = true
|
||||
litellm-cache.workspace = true
|
||||
litellm-layered.workspace = true
|
||||
|
||||
async-trait = "0.1"
|
||||
azure_core = "1.1.0"
|
||||
|
|
|
|||
|
|
@ -10,9 +10,11 @@ use azure_storage_blob::{
|
|||
models::{BlobClientUploadOptions, StorageErrorCode},
|
||||
};
|
||||
use futures_util::{TryStreamExt, future::try_join_all};
|
||||
use litellm_auth_azure::AzureAuthService;
|
||||
use litellm_cache::{
|
||||
BaseCache, BatchCache, CacheCodec, DisconnectCache, Error, ExactCacheContext, FlushCache,
|
||||
};
|
||||
use litellm_layered::SharedLookup;
|
||||
use tokio::runtime::Handle;
|
||||
use url::Url;
|
||||
|
||||
|
|
@ -34,11 +36,15 @@ impl<C: CacheCodec> AzureBlobCache<C> {
|
|||
http: litellm_http::Client,
|
||||
codec: C,
|
||||
runtime: Handle,
|
||||
env: SharedLookup,
|
||||
) -> Result<Self, Error> {
|
||||
Self::connect_with_options(
|
||||
account_url,
|
||||
container,
|
||||
Some(Arc::new(AzureBlobCredential::default())),
|
||||
Some(Arc::new(AzureBlobCredential::new(
|
||||
AzureAuthService::default(),
|
||||
env,
|
||||
))),
|
||||
ClientOptions {
|
||||
transport: Some(Transport::new(Arc::new(ReqwestTransport(http)))),
|
||||
..ClientOptions::default()
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
use std::{
|
||||
fmt,
|
||||
sync::Arc,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
|
|
@ -11,15 +10,14 @@ use azure_core::{
|
|||
};
|
||||
use litellm_auth_azure::{AzureAuthInputs, AzureAuthService};
|
||||
use litellm_auth_types::ResolvedCredential;
|
||||
use litellm_layered::SharedLookup;
|
||||
|
||||
const STATIC_TOKEN_LIFETIME: Duration = Duration::from_secs(300);
|
||||
const LLM_TOKEN_ENV: &str = "AZURE_AD_TOKEN";
|
||||
|
||||
type EnvLookup = Arc<dyn Fn(&str) -> Option<String> + Send + Sync>;
|
||||
|
||||
pub struct AzureBlobCredential {
|
||||
service: AzureAuthService,
|
||||
env_lookup: EnvLookup,
|
||||
env_lookup: SharedLookup,
|
||||
}
|
||||
|
||||
impl fmt::Debug for AzureBlobCredential {
|
||||
|
|
@ -28,17 +26,8 @@ impl fmt::Debug for AzureBlobCredential {
|
|||
}
|
||||
}
|
||||
|
||||
impl Default for AzureBlobCredential {
|
||||
fn default() -> Self {
|
||||
Self::new(
|
||||
AzureAuthService::default(),
|
||||
Arc::new(|name| std::env::var(name).ok()),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl AzureBlobCredential {
|
||||
pub fn new(service: AzureAuthService, env_lookup: EnvLookup) -> Self {
|
||||
pub fn new(service: AzureAuthService, env_lookup: SharedLookup) -> Self {
|
||||
Self {
|
||||
service,
|
||||
env_lookup,
|
||||
|
|
@ -54,7 +43,11 @@ impl TokenCredential for AzureBlobCredential {
|
|||
_options: Option<TokenRequestOptions<'_>>,
|
||||
) -> azure_core::Result<AccessToken> {
|
||||
let env_lookup = &self.env_lookup;
|
||||
let lookup = move |name: &str| (name != LLM_TOKEN_ENV).then(|| env_lookup(name)).flatten();
|
||||
let lookup = move |name: &str| {
|
||||
(name != LLM_TOKEN_ENV)
|
||||
.then(|| env_lookup.get(name))
|
||||
.flatten()
|
||||
};
|
||||
let credential = self
|
||||
.service
|
||||
.get_azure_ad_token(
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ futures-util.workspace = true
|
|||
litellm-auth-gcp.workspace = true
|
||||
litellm-auth-types.workspace = true
|
||||
litellm-cache.workspace = true
|
||||
litellm-layered.workspace = true
|
||||
percent-encoding.workspace = true
|
||||
reqwest.workspace = true
|
||||
tokio.workspace = true
|
||||
|
|
@ -18,6 +19,7 @@ tokio.workspace = true
|
|||
[dev-dependencies]
|
||||
litellm-http = { workspace = true, features = ["test-support"] }
|
||||
litellm-cache-testing.workspace = true
|
||||
litellm-layered.workspace = true
|
||||
rstest.workspace = true
|
||||
serde_json.workspace = true
|
||||
tokio.workspace = true
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ use litellm_cache::{
|
|||
FlushCache,
|
||||
};
|
||||
use litellm_http::Client;
|
||||
use litellm_layered::SharedLookup;
|
||||
use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_encode};
|
||||
|
||||
use crate::{GcpTokenSource, TokenSource};
|
||||
|
|
@ -53,8 +54,11 @@ pub struct GcsCache<S: CacheCodec> {
|
|||
}
|
||||
|
||||
impl<S: CacheCodec> GcsCache<S> {
|
||||
pub fn new(config: GcsConfig, client: Client, codec: S) -> Self {
|
||||
let token = Arc::new(GcpTokenSource::new(config.path_service_account.clone()));
|
||||
pub fn new(config: GcsConfig, client: Client, codec: S, env: SharedLookup) -> Self {
|
||||
let token = Arc::new(GcpTokenSource::new(
|
||||
config.path_service_account.clone(),
|
||||
env,
|
||||
));
|
||||
Self::with_token_source(config, client, codec, token)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ use std::{future::Future, pin::Pin};
|
|||
use litellm_auth_gcp::{VertexAuth, VertexConfig};
|
||||
use litellm_auth_types::{InputSource, SecretValue, Sourced};
|
||||
use litellm_cache::Error;
|
||||
use litellm_layered::SharedLookup;
|
||||
|
||||
pub trait TokenSource: Send + Sync + 'static {
|
||||
fn bearer_token(&self) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + '_>>;
|
||||
|
|
@ -11,15 +12,17 @@ pub trait TokenSource: Send + Sync + 'static {
|
|||
pub struct GcpTokenSource {
|
||||
auth: VertexAuth,
|
||||
config: VertexConfig,
|
||||
env: SharedLookup,
|
||||
}
|
||||
|
||||
impl GcpTokenSource {
|
||||
pub fn new(path_service_account: Option<String>) -> Self {
|
||||
pub fn new(path_service_account: Option<String>, env: SharedLookup) -> Self {
|
||||
let credentials = path_service_account
|
||||
.map(|path| Sourced::new(SecretValue::new(path), InputSource::Deployment));
|
||||
Self {
|
||||
auth: VertexAuth::default(),
|
||||
config: VertexConfig::new(credentials, None, None),
|
||||
env,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -28,7 +31,7 @@ impl TokenSource for GcpTokenSource {
|
|||
fn bearer_token(&self) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + '_>> {
|
||||
Box::pin(async move {
|
||||
self.auth
|
||||
.access_token(&self.config, &|name| std::env::var(name).ok())
|
||||
.access_token(&self.config, &|name| self.env.get(name))
|
||||
.await
|
||||
.map_err(|_| Error::Unavailable)
|
||||
})
|
||||
|
|
|
|||
|
|
@ -99,6 +99,7 @@ async fn cache_exposes_its_configuration(#[future(awt)] server: MockServer) {
|
|||
},
|
||||
litellm_http::Client::plain_for_test(),
|
||||
litellm_cache::JsonCodec::<Value>::new(),
|
||||
std::sync::Arc::new(litellm_layered::Env::default()),
|
||||
);
|
||||
assert_eq!(cache.bucket_name(), "bucket");
|
||||
assert_eq!(cache.key_prefix(), "folder/");
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
|
||||
mod support;
|
||||
|
||||
use std::time::Duration;
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ repository.workspace = true
|
|||
|
||||
[dependencies]
|
||||
litellm-cache.workspace = true
|
||||
litellm-layered.workspace = true
|
||||
redis = { version = "1.7.0", features = ["cluster", "tls-rustls"] }
|
||||
r2d2 = "0.8.10"
|
||||
tokio.workspace = true
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ use std::{
|
|||
};
|
||||
|
||||
use litellm_cache::{BatchEntry, CacheCodec, Error};
|
||||
use litellm_layered::Lookup;
|
||||
|
||||
use crate::{
|
||||
connection::{ConnectionRef, Connections},
|
||||
|
|
@ -20,11 +21,17 @@ pub struct RedisCache<S, C = redis::Connection> {
|
|||
pub(crate) topology: RedisTopology,
|
||||
/// The server's major version, read from `INFO` once, like Python's `redis_version`.
|
||||
pub(crate) major_version: Arc<OnceLock<u32>>,
|
||||
default_major_version: u32,
|
||||
}
|
||||
|
||||
impl<S: CacheCodec> RedisCache<S> {
|
||||
pub fn new(url: &str, default_ttl: Option<Duration>, codec: S) -> Result<Self, Error> {
|
||||
Self::connect(url, &RedisTopology::Standalone, default_ttl, codec)
|
||||
pub fn new(
|
||||
url: &str,
|
||||
default_ttl: Option<Duration>,
|
||||
codec: S,
|
||||
env: &(impl Lookup + ?Sized),
|
||||
) -> Result<Self, Error> {
|
||||
Self::connect(url, &RedisTopology::Standalone, default_ttl, codec, env)
|
||||
}
|
||||
|
||||
pub fn connect(
|
||||
|
|
@ -32,6 +39,7 @@ impl<S: CacheCodec> RedisCache<S> {
|
|||
topology: &RedisTopology,
|
||||
default_ttl: Option<Duration>,
|
||||
codec: S,
|
||||
env: &(impl Lookup + ?Sized),
|
||||
) -> Result<Self, Error> {
|
||||
let connections = Connections::open(url, topology)?;
|
||||
Ok(Self {
|
||||
|
|
@ -41,6 +49,7 @@ impl<S: CacheCodec> RedisCache<S> {
|
|||
namespace: None,
|
||||
topology: topology.clone(),
|
||||
major_version: Arc::default(),
|
||||
default_major_version: default_major_version(env),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -50,7 +59,12 @@ where
|
|||
S: CacheCodec,
|
||||
C: redis::ConnectionLike + Send + 'static,
|
||||
{
|
||||
pub fn with_connection(connection: C, default_ttl: Option<Duration>, codec: S) -> Self {
|
||||
pub fn with_connection(
|
||||
connection: C,
|
||||
default_ttl: Option<Duration>,
|
||||
codec: S,
|
||||
env: &(impl Lookup + ?Sized),
|
||||
) -> Self {
|
||||
Self {
|
||||
connections: Arc::new(Connections::fixed(connection)),
|
||||
default_ttl: default_ttl.unwrap_or(DEFAULT_TTL),
|
||||
|
|
@ -58,6 +72,7 @@ where
|
|||
namespace: None,
|
||||
topology: RedisTopology::Standalone,
|
||||
major_version: Arc::default(),
|
||||
default_major_version: default_major_version(env),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -109,7 +124,7 @@ where
|
|||
let version = info
|
||||
.ok()
|
||||
.and_then(|info| parse_major_version(&info))
|
||||
.unwrap_or_else(default_major_version);
|
||||
.unwrap_or(self.default_major_version);
|
||||
*self.major_version.get_or_init(|| version)
|
||||
}
|
||||
|
||||
|
|
@ -184,9 +199,9 @@ fn parse_major_version(info: &str) -> Option<u32> {
|
|||
}
|
||||
}
|
||||
|
||||
fn default_major_version() -> u32 {
|
||||
std::env::var("DEFAULT_REDIS_MAJOR_VERSION")
|
||||
.ok()
|
||||
/// `DEFAULT_REDIS_MAJOR_VERSION`, the version assumed when `INFO` does not answer.
|
||||
fn default_major_version(env: &(impl Lookup + ?Sized)) -> u32 {
|
||||
env.get("DEFAULT_REDIS_MAJOR_VERSION")
|
||||
.and_then(|value| value.parse().ok())
|
||||
.unwrap_or(7)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,11 +25,11 @@ fn mock(commands: Vec<MockCmd>) -> MockRedisConnection {
|
|||
}
|
||||
|
||||
fn tagged(commands: Vec<MockCmd>) -> Mocked<TaggedByteCodec> {
|
||||
RedisCache::with_connection(mock(commands), None, TaggedByteCodec(42))
|
||||
RedisCache::with_connection(mock(commands), None, TaggedByteCodec(42), &|_: &str| None)
|
||||
}
|
||||
|
||||
fn json_cache(commands: Vec<MockCmd>) -> Mocked<JsonCodec<serde_json::Value>> {
|
||||
RedisCache::with_connection(mock(commands), None, JsonCodec::new())
|
||||
RedisCache::with_connection(mock(commands), None, JsonCodec::new(), &|_: &str| None)
|
||||
}
|
||||
|
||||
fn team(commands: Vec<MockCmd>) -> Mocked<JsonCodec<serde_json::Value>> {
|
||||
|
|
@ -55,7 +55,15 @@ fn scan(pattern: &str, cursor: u64, count: usize, reply: redis::Value) -> MockCm
|
|||
|
||||
#[rstest]
|
||||
fn constructor_rejects_invalid_urls() {
|
||||
assert!(RedisCache::new("not a redis url", None, JsonCodec::<String>::new()).is_err());
|
||||
assert!(
|
||||
RedisCache::new(
|
||||
"not a redis url",
|
||||
None,
|
||||
JsonCodec::<String>::new(),
|
||||
&|_: &str| None
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
|
|
@ -144,6 +152,7 @@ async fn async_operations_preserve_codec_ttl_and_missing_values(context: ExactCa
|
|||
]),
|
||||
Some(Duration::from_secs(9)),
|
||||
TaggedByteCodec(42),
|
||||
&|_: &str| None,
|
||||
);
|
||||
cache
|
||||
.batch_cache_write("counter", 7, context.clone())
|
||||
|
|
@ -342,6 +351,7 @@ async fn ttl_pipeline_keeps_each_entry_ttl_and_defaults_missing_ones() {
|
|||
)]),
|
||||
Some(Duration::from_secs(300)),
|
||||
JsonCodec::<serde_json::Value>::new(),
|
||||
&|_: &str| None,
|
||||
)
|
||||
.with_namespace(Some("ns".into()));
|
||||
|
||||
|
|
@ -616,6 +626,35 @@ async fn counted_lpop_assumes_redis_7_when_info_fails() {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn counted_lpop_pops_one_at_a_time_when_info_fails_and_the_env_names_an_older_version() {
|
||||
let single_pop = || {
|
||||
let mut pipeline = redis::pipe();
|
||||
pipeline.cmd("LPOP").arg("team:queue");
|
||||
pipeline
|
||||
};
|
||||
let cache = RedisCache::with_connection(
|
||||
mock(vec![
|
||||
MockCmd::new(
|
||||
redis::cmd("INFO"),
|
||||
Err::<String, _>(redis::RedisError::from((redis::ErrorKind::Io, "down"))),
|
||||
),
|
||||
MockCmd::with_values(single_pop(), Ok(vec![redis_test::redis_value!("a")])),
|
||||
MockCmd::with_values(single_pop(), Ok(vec![redis_test::redis_value!("b")])),
|
||||
]),
|
||||
None,
|
||||
JsonCodec::<serde_json::Value>::new(),
|
||||
&litellm_layered::Env::of(&[("DEFAULT_REDIS_MAJOR_VERSION", "5")]),
|
||||
)
|
||||
.with_namespace(Some("team".into()));
|
||||
|
||||
assert_eq!(
|
||||
cache.async_lpop("queue", Some(2)).await.unwrap(),
|
||||
RedisLpopResult::Values(vec![b"a".to_vec(), b"b".to_vec()])
|
||||
);
|
||||
}
|
||||
|
||||
fn push_and_trim(start: i64) -> redis::Pipeline {
|
||||
let mut pipeline = redis::pipe();
|
||||
pipeline
|
||||
|
|
@ -924,6 +963,7 @@ async fn increments_keep_an_existing_ttl_in_one_atomic_script(context: ExactCach
|
|||
mock(vec![increment_script(2.5), increment_script(2.5)]),
|
||||
None,
|
||||
JsonCodec::<f64>::new(),
|
||||
&|_: &str| None,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
|
|
@ -1153,6 +1193,12 @@ async fn disconnect_keeps_a_caller_owned_connection_usable() {
|
|||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn disconnect_drains_an_idle_pool_without_connecting() {
|
||||
let cache = RedisCache::new("redis://127.0.0.1:1", None, JsonCodec::<String>::new()).unwrap();
|
||||
let cache = RedisCache::new(
|
||||
"redis://127.0.0.1:1",
|
||||
None,
|
||||
JsonCodec::<String>::new(),
|
||||
&|_: &str| None,
|
||||
)
|
||||
.unwrap();
|
||||
cache.disconnect().await.unwrap();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -64,6 +64,7 @@ fn constructor_rejects_unusable_cluster_configs(
|
|||
&RedisTopology::Cluster { startup_nodes },
|
||||
None,
|
||||
JsonCodec::new(),
|
||||
&|_: &str| None,
|
||||
)
|
||||
.err();
|
||||
assert!(matches!(error, Some(Error::Unavailable)));
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::{
|
||||
|
|
@ -183,8 +184,10 @@ impl redis::ConnectionLike for FakeRedis {
|
|||
pub type JsonCache<C = redis::Connection> = RedisCache<JsonCodec<serde_json::Value>, C>;
|
||||
|
||||
pub fn fake_cache(namespace: &str) -> JsonCache<FakeRedis> {
|
||||
RedisCache::with_connection(FakeRedis::default(), None, JsonCodec::new())
|
||||
.with_namespace(Some(namespace.into()))
|
||||
RedisCache::with_connection(FakeRedis::default(), None, JsonCodec::new(), &|_: &str| {
|
||||
None
|
||||
})
|
||||
.with_namespace(Some(namespace.into()))
|
||||
}
|
||||
|
||||
/// Startup nodes from `LITELLM_TEST_REDIS_CLUSTER_NODES` (`host:port,host:port`); tests that
|
||||
|
|
@ -224,8 +227,14 @@ pub fn cluster_cache<S: CacheCodec>(
|
|||
) -> Option<RedisCache<S>> {
|
||||
let topology = cluster_topology()?;
|
||||
Some(
|
||||
RedisCache::connect(&cluster_url(), &topology, Some(default_ttl), codec)
|
||||
.expect("cluster connection")
|
||||
.with_namespace(Some(unique_namespace(label))),
|
||||
RedisCache::connect(
|
||||
&cluster_url(),
|
||||
&topology,
|
||||
Some(default_ttl),
|
||||
codec,
|
||||
&|_: &str| None,
|
||||
)
|
||||
.expect("cluster connection")
|
||||
.with_namespace(Some(unique_namespace(label))),
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ pub fn keyed(key: &str) -> ResponseCacheRequest {
|
|||
pub fn redis(commands: Vec<MockCmd>, namespace: Option<&str>) -> ResponseCache<MockedRedis> {
|
||||
let connection = MockRedisConnection::new(commands).assert_all_commands_consumed();
|
||||
ResponseCache::new(Arc::new(
|
||||
RedisCache::with_connection(connection, None, ResponseCacheCodec)
|
||||
RedisCache::with_connection(connection, None, ResponseCacheCodec, &|_: &str| None)
|
||||
.with_namespace(namespace.map(str::to_owned)),
|
||||
))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ repository.workspace = true
|
|||
litellm-http.workspace = true
|
||||
litellm-cache.workspace = true
|
||||
litellm-auth-aws.workspace = true
|
||||
litellm-layered.workspace = true
|
||||
aws-sdk-s3 = { version = "1.146.1", default-features = false, features = ["rustls", "rt-tokio"] }
|
||||
aws-credential-types = "1.3.0"
|
||||
aws-smithy-runtime-api = { version = "1.16.2", features = ["client", "http-1x"] }
|
||||
|
|
@ -22,6 +23,7 @@ tokio.workspace = true
|
|||
[dev-dependencies]
|
||||
litellm-http = { workspace = true, features = ["test-support"] }
|
||||
litellm-cache-testing.workspace = true
|
||||
litellm-layered.workspace = true
|
||||
rstest.workspace = true
|
||||
wiremock.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
|
|
|||
|
|
@ -3,20 +3,17 @@ use aws_credential_types::{
|
|||
provider::{ProvideCredentials, error::CredentialsError, future},
|
||||
};
|
||||
use litellm_auth_aws::{AwsAuthConfig, AwsAuthService};
|
||||
use litellm_layered::SharedLookup;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct S3Credentials {
|
||||
auth: AwsAuthService,
|
||||
config: AwsAuthConfig,
|
||||
env: fn(&str) -> Option<String>,
|
||||
env: SharedLookup,
|
||||
}
|
||||
|
||||
impl S3Credentials {
|
||||
pub fn new(config: AwsAuthConfig) -> Self {
|
||||
Self::with_env(config, |name| std::env::var(name).ok())
|
||||
}
|
||||
|
||||
pub fn with_env(config: AwsAuthConfig, env: fn(&str) -> Option<String>) -> Self {
|
||||
pub fn new(config: AwsAuthConfig, env: SharedLookup) -> Self {
|
||||
Self {
|
||||
auth: AwsAuthService::default(),
|
||||
config,
|
||||
|
|
@ -44,7 +41,7 @@ impl ProvideCredentials for S3Credentials {
|
|||
));
|
||||
}
|
||||
self.auth
|
||||
.resolve_credentials(self.config.clone(), &self.env)
|
||||
.resolve_credentials(self.config.clone(), &|name| self.env.get(name))
|
||||
.await
|
||||
.map_err(|_| CredentialsError::provider_error("S3 cache authentication failed"))
|
||||
})
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ use litellm_auth_aws::AwsAuthConfig;
|
|||
use litellm_cache::{
|
||||
BaseCache, BatchCache, CacheCodec, DisconnectCache, Error, ExactCacheContext, FlushCache,
|
||||
};
|
||||
use litellm_layered::SharedLookup;
|
||||
use tokio::runtime::Handle;
|
||||
|
||||
use crate::{auth::S3Credentials, transport::ReqwestHttpClient};
|
||||
|
|
@ -47,13 +48,14 @@ impl<C: CacheCodec> S3Cache<C> {
|
|||
http: litellm_http::Client,
|
||||
codec: C,
|
||||
runtime: Handle,
|
||||
env: SharedLookup,
|
||||
) -> Self {
|
||||
let endpoint_url: Option<String> = config.endpoint.map(|endpoint| endpoint.url);
|
||||
let base = aws_sdk_s3::Config::builder()
|
||||
.behavior_version(BehaviorVersion::latest())
|
||||
.region(Region::new(config.region.clone()))
|
||||
.http_client(ReqwestHttpClient(http))
|
||||
.credentials_provider(S3Credentials::new(config.auth))
|
||||
.credentials_provider(S3Credentials::new(config.auth, env))
|
||||
.request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
|
||||
.response_checksum_validation(ResponseChecksumValidation::WhenRequired);
|
||||
let builder = match &endpoint_url {
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ async fn credentials_resolve(
|
|||
#[case] env: fn(&str) -> Option<String>,
|
||||
#[case] expected: (&str, &str, Option<&str>),
|
||||
) {
|
||||
let credentials = S3Credentials::with_env(config, env)
|
||||
let credentials = S3Credentials::new(config, std::sync::Arc::new(env))
|
||||
.provide_credentials()
|
||||
.await
|
||||
.unwrap();
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ pub fn cache_with(config: S3CacheConfig, runtime: Handle) -> JsonS3Cache {
|
|||
litellm_http::Client::plain_for_test(),
|
||||
JsonCodec::new(),
|
||||
runtime,
|
||||
std::sync::Arc::new(litellm_layered::Env::default()),
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,26 +1,5 @@
|
|||
use std::str::FromStr;
|
||||
|
||||
use crate::coercion::parse_str_bool;
|
||||
|
||||
pub trait Lookup {
|
||||
fn get(&self, name: &str) -> Option<String>;
|
||||
|
||||
fn truthy(&self, name: &str) -> Option<String> {
|
||||
self.get(name).filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
fn enabled(&self, name: &str) -> Option<bool> {
|
||||
self.get(name)
|
||||
.is_some_and(|value| parse_str_bool(&value) == Some(true))
|
||||
.then_some(true)
|
||||
}
|
||||
|
||||
fn parsed<T: FromStr>(&self, name: &str) -> Option<T>
|
||||
where
|
||||
Self: Sized,
|
||||
{
|
||||
self.get(name).and_then(|value| value.trim().parse().ok())
|
||||
}
|
||||
}
|
||||
|
||||
impl<F: Fn(&str) -> Option<String>> Lookup for F {
|
||||
|
|
@ -29,14 +8,6 @@ impl<F: Fn(&str) -> Option<String>> Lookup for F {
|
|||
}
|
||||
}
|
||||
|
||||
pub struct ProcessEnvironment;
|
||||
|
||||
impl Lookup for ProcessEnvironment {
|
||||
fn get(&self, name: &str) -> Option<String> {
|
||||
std::env::var(name).ok()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn resolve_non_empty(
|
||||
value: Option<&str>,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
|
|
@ -72,35 +43,4 @@ mod tests {
|
|||
assert_eq!(env.get("EMPTY"), Some(String::new()));
|
||||
assert_eq!(env.get("ABSENT"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truthy_drops_an_empty_value_like_a_python_or_chain() {
|
||||
let env = env_of(&[("EMPTY", ""), ("SET", "value")]);
|
||||
assert_eq!(env.truthy("EMPTY"), None);
|
||||
assert_eq!(env.truthy("SET").as_deref(), Some("value"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enabled_only_switches_on_for_true_and_never_forces_off() {
|
||||
let env = env_of(&[
|
||||
("LOWER", "true"),
|
||||
("PADDED", " True "),
|
||||
("OFF", "false"),
|
||||
("ONE", "1"),
|
||||
]);
|
||||
assert_eq!(env.enabled("LOWER"), Some(true));
|
||||
assert_eq!(env.enabled("PADDED"), Some(true));
|
||||
assert_eq!(env.enabled("OFF"), None);
|
||||
assert_eq!(env.enabled("ONE"), None);
|
||||
assert_eq!(env.enabled("ABSENT"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parsed_trims_and_skips_values_that_do_not_parse() {
|
||||
let env = env_of(&[("PADDED", " 45 "), ("WORD", "soon"), ("FRACTION", "0.5")]);
|
||||
assert_eq!(env.parsed::<u32>("PADDED"), Some(45));
|
||||
assert_eq!(env.parsed::<u32>("WORD"), None);
|
||||
assert_eq!(env.parsed::<f64>("FRACTION"), Some(0.5));
|
||||
assert_eq!(env.parsed::<u32>("ABSENT"), None);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
|
||||
use std::{
|
||||
env,
|
||||
process::{Command, ExitCode},
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ litellm-host-http.workspace = true
|
|||
litellm-host.workspace = true
|
||||
litellm-http.workspace = true
|
||||
litellm-llms.workspace = true
|
||||
litellm-layered.workspace = true
|
||||
litellm-router.workspace = true
|
||||
litellm-secrets.workspace = true
|
||||
litellm-llms-types.workspace = true
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ use litellm_inference_messages::MessagesRoute;
|
|||
use litellm_inference_ocr::OcrRoute;
|
||||
use litellm_inference_responses::ResponsesRoute;
|
||||
use litellm_inference_transcription::AudioTranscriptionRoute;
|
||||
use litellm_layered::SharedLookup;
|
||||
use litellm_llms::base_llm::ocr::{handler::OcrClient, settings::OcrSettings};
|
||||
use litellm_secrets::source::SecretSource;
|
||||
|
||||
|
|
@ -53,6 +54,8 @@ impl Gateway {
|
|||
pub fn new(
|
||||
resources: CoreResources,
|
||||
http: HttpClientConfig,
|
||||
ocr: OcrSettings,
|
||||
env: SharedLookup,
|
||||
secrets: Arc<dyn SecretSource>,
|
||||
models: ModelRouter,
|
||||
) -> Result<Self, litellm_http::Error> {
|
||||
|
|
@ -70,14 +73,14 @@ impl Gateway {
|
|||
auth.clone(),
|
||||
secrets.clone(),
|
||||
),
|
||||
messages: MessagesRoute::new(provider.clone(), auth.clone(), secrets.clone()),
|
||||
messages: MessagesRoute::new(provider.clone(), auth.clone(), secrets.clone(), env),
|
||||
responses: ResponsesRoute::new(provider, auth.clone(), secrets.clone()),
|
||||
ocr: OcrRoute::new(OcrClient::new(
|
||||
&resources.pool,
|
||||
&http,
|
||||
UrlPolicy::default(),
|
||||
auth,
|
||||
OcrSettings::default(),
|
||||
ocr,
|
||||
secrets.clone(),
|
||||
)?),
|
||||
models,
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ use futures_util::future::BoxFuture;
|
|||
use litellm_gateway_inference::{Deployment, Gateway, router};
|
||||
use litellm_http::{HttpClientPool, HttpSettings, Resolution, media::PublicDnsResolver};
|
||||
use litellm_inference::resources::CoreResources;
|
||||
use litellm_llms::base_llm::ocr::settings::OcrSettings;
|
||||
use litellm_secrets::{SecretValue, source::SecretSource};
|
||||
use serde_json::Value;
|
||||
use tower::ServiceExt;
|
||||
|
|
@ -83,6 +84,8 @@ fn configured_app(
|
|||
let gateway = Gateway::new(
|
||||
resources,
|
||||
http,
|
||||
OcrSettings::default(),
|
||||
Arc::new(litellm_layered::Env::default()),
|
||||
secrets,
|
||||
[(
|
||||
"public/model".into(),
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ repository.workspace = true
|
|||
|
||||
[dependencies]
|
||||
axum.workspace = true
|
||||
envy = "0.4.2"
|
||||
http-body-util = "0.1"
|
||||
litellm-inference.workspace = true
|
||||
litellm-gateway-inference.workspace = true
|
||||
|
|
@ -19,6 +18,7 @@ litellm-gateway-auth.workspace = true
|
|||
litellm-gateway-ui.workspace = true
|
||||
litellm-auth-types.workspace = true
|
||||
litellm-config.workspace = true
|
||||
litellm-core-utils.workspace = true
|
||||
litellm-http = { workspace = true, features = ["mcp"] }
|
||||
litellm-layered.workspace = true
|
||||
litellm-llms.workspace = true
|
||||
|
|
@ -38,3 +38,4 @@ serde_yaml_ng = "0.10.0"
|
|||
tempfile.workspace = true
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
tower = { version = "0.5", features = ["util"] }
|
||||
wiremock.workspace = true
|
||||
|
|
|
|||
|
|
@ -20,30 +20,35 @@ use litellm_gateway_auth::Auth;
|
|||
use litellm_gateway_inference::{Gateway, ModelRouter};
|
||||
use litellm_http::{ClientVariant, HttpClientPool, Resolution, media::PublicDnsResolver};
|
||||
use litellm_inference::resources::CoreResources;
|
||||
use litellm_layered::Env;
|
||||
use litellm_llms::base_llm::ocr::settings::{OcrKeysPartial, OcrSettings};
|
||||
use litellm_secrets::source::EnvironmentSecrets;
|
||||
use litellm_tracing::ByteChunk;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub fn build_inference(config: &Config) -> Result<Arc<Gateway>, Error> {
|
||||
pub fn build_inference(config: &Config, process: &Env) -> Result<Arc<Gateway>, Error> {
|
||||
let pool = Arc::new(HttpClientPool::new(Arc::new(PublicDnsResolver)));
|
||||
let environment = secrets::environment_values(&config.environment_variables)?;
|
||||
let lookup = |name: &str| {
|
||||
let env = process.clone().overlaid(
|
||||
environment
|
||||
.get(name)
|
||||
.map(|value| value.expose().to_owned())
|
||||
.or_else(|| std::env::var(name).ok())
|
||||
};
|
||||
let http_settings = http::http_settings(&lookup, &config.litellm_settings)?;
|
||||
.iter()
|
||||
.map(|(name, value)| (name.clone(), value.expose().to_owned())),
|
||||
);
|
||||
let http_settings = http::http_settings(&env, &config.litellm_settings)?;
|
||||
let ocr = OcrSettings::from(OcrKeysPartial::from_env(&env)?);
|
||||
let http = Resolution::from(&http_settings).config;
|
||||
let client = pool.client(&http, ClientVariant::Provider)?;
|
||||
let env = Arc::new(env);
|
||||
let secrets = Arc::new(secrets::ConfigSecrets::new(
|
||||
environment,
|
||||
Arc::new(EnvironmentSecrets::python_compatible(client)),
|
||||
Arc::new(EnvironmentSecrets::python_compatible(client, env.clone())),
|
||||
));
|
||||
let resources = CoreResources::new(pool);
|
||||
Ok(Arc::new(Gateway::new(
|
||||
resources,
|
||||
http,
|
||||
ocr,
|
||||
env,
|
||||
secrets,
|
||||
ModelRouter::from_model_list(&config.model_list),
|
||||
)?))
|
||||
|
|
|
|||
|
|
@ -10,7 +10,8 @@ use serde_json::json;
|
|||
|
||||
mod settings;
|
||||
|
||||
use settings::{Settings, UiSettings};
|
||||
use litellm_layered::Env;
|
||||
use settings::{BootKeysPartial, Settings, UiKeysPartial, UiSettings};
|
||||
|
||||
struct StderrSink {
|
||||
level: Level,
|
||||
|
|
@ -41,14 +42,15 @@ impl Sink for StderrSink {
|
|||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn Error>> {
|
||||
let settings: Settings = settings::from_iter(std::env::vars_os())?;
|
||||
let env = Env::from_process();
|
||||
let settings = Settings::from(BootKeysPartial::from_env(&env)?);
|
||||
let level = settings.log_level();
|
||||
Logger::new(StderrSink { level }).install_global()?;
|
||||
let config = Config::load(settings.litellm_config)?;
|
||||
let inference = litellm_gateway::build_inference(&config)?;
|
||||
let ui = match std::env::var_os("LITELLM_UI_PATH") {
|
||||
let inference = litellm_gateway::build_inference(&config, &env)?;
|
||||
let ui = match &settings.ui_path {
|
||||
Some(directory) => {
|
||||
let ui_settings: UiSettings = settings::from_iter(std::env::vars_os())?;
|
||||
let ui_settings = UiSettings::try_from(UiKeysPartial::from_env(&env)?)?;
|
||||
let backend = UiBackend::new(ui_settings.ui_username, ui_settings.ui_password)?;
|
||||
Some(
|
||||
litellm_gateway_ui::router(
|
||||
|
|
|
|||
|
|
@ -1,25 +1,39 @@
|
|||
use std::ffi::OsString;
|
||||
|
||||
use litellm_auth_types::SecretValue;
|
||||
use litellm_core_utils::coercion;
|
||||
use litellm_layered::{Invalid, Layered};
|
||||
use litellm_tracing::Level;
|
||||
use serde::{Deserialize, de::DeserializeOwned};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(default)]
|
||||
#[derive(Layered)]
|
||||
#[expect(dead_code, reason = "only BootKeysPartial is built")]
|
||||
pub(super) struct BootKeys {
|
||||
#[layer(env = "HOST")]
|
||||
host: String,
|
||||
#[layer(env = "PORT", read = coercion::required::<u16>)]
|
||||
port: u16,
|
||||
#[layer(env = "LITELLM_CONFIG")]
|
||||
litellm_config: String,
|
||||
#[layer(env = "RUST_LOG")]
|
||||
rust_log: String,
|
||||
#[layer(env = "LITELLM_UI_PATH")]
|
||||
ui_path: String,
|
||||
}
|
||||
|
||||
pub(super) struct Settings {
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
pub litellm_config: String,
|
||||
pub ui_path: Option<String>,
|
||||
rust_log: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for Settings {
|
||||
fn default() -> Self {
|
||||
impl From<BootKeysPartial> for Settings {
|
||||
fn from(keys: BootKeysPartial) -> Self {
|
||||
Self {
|
||||
host: "0.0.0.0".into(),
|
||||
port: 4000,
|
||||
litellm_config: "config.yaml".into(),
|
||||
rust_log: None,
|
||||
host: keys.host.unwrap_or_else(|| "0.0.0.0".into()),
|
||||
port: keys.port.unwrap_or(4000),
|
||||
litellm_config: keys.litellm_config.unwrap_or_else(|| "config.yaml".into()),
|
||||
ui_path: keys.ui_path,
|
||||
rust_log: keys.rust_log,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -33,64 +47,81 @@ impl Settings {
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[derive(Layered)]
|
||||
#[expect(dead_code, reason = "only UiKeysPartial is built")]
|
||||
pub(super) struct UiKeys {
|
||||
#[layer(env = "UI_USERNAME")]
|
||||
ui_username: String,
|
||||
#[layer(env = "UI_PASSWORD", read = secret, secret)]
|
||||
ui_password: SecretValue,
|
||||
#[layer(env = "LITELLM_UI_SECURE_COOKIES", read = coercion::required::<bool>)]
|
||||
litellm_ui_secure_cookies: bool,
|
||||
}
|
||||
|
||||
fn secret(value: &str) -> Result<Option<SecretValue>, Invalid> {
|
||||
Ok(Some(SecretValue::new(value)))
|
||||
}
|
||||
|
||||
pub(super) struct UiSettings {
|
||||
#[serde(default = "default_username")]
|
||||
pub ui_username: String,
|
||||
pub ui_password: SecretValue,
|
||||
#[serde(default = "default_secure_cookies")]
|
||||
pub litellm_ui_secure_cookies: bool,
|
||||
}
|
||||
|
||||
fn default_username() -> String {
|
||||
"admin".into()
|
||||
}
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("UI_PASSWORD must be set when LITELLM_UI_PATH serves the UI")]
|
||||
pub(super) struct MissingUiPassword;
|
||||
|
||||
fn default_secure_cookies() -> bool {
|
||||
true
|
||||
}
|
||||
impl TryFrom<UiKeysPartial> for UiSettings {
|
||||
type Error = MissingUiPassword;
|
||||
|
||||
pub(super) fn from_iter<T: DeserializeOwned>(
|
||||
variables: impl IntoIterator<Item = (OsString, OsString)>,
|
||||
) -> envy::Result<T> {
|
||||
envy::from_iter(
|
||||
variables
|
||||
.into_iter()
|
||||
.filter_map(|(key, value)| Some((key.into_string().ok()?, value.into_string().ok()?))),
|
||||
)
|
||||
fn try_from(keys: UiKeysPartial) -> Result<Self, Self::Error> {
|
||||
Ok(Self {
|
||||
ui_username: keys.ui_username.unwrap_or_else(|| "admin".into()),
|
||||
ui_password: keys.ui_password.ok_or(MissingUiPassword)?,
|
||||
litellm_ui_secure_cookies: keys.litellm_ui_secure_cookies.unwrap_or(true),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::ffi::OsString;
|
||||
|
||||
use litellm_layered::Env;
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn parse<T: DeserializeOwned>(variables: &[(&str, &str)]) -> envy::Result<T> {
|
||||
from_iter(
|
||||
variables
|
||||
.iter()
|
||||
.map(|(key, value)| (OsString::from(key), OsString::from(value))),
|
||||
)
|
||||
fn boot(variables: &[(&str, &str)]) -> Result<Settings, litellm_layered::Error> {
|
||||
BootKeysPartial::from_env(&Env::of(variables)).map(Settings::from)
|
||||
}
|
||||
|
||||
fn ui(variables: &[(&str, &str)]) -> Result<UiSettings, Box<dyn std::error::Error>> {
|
||||
Ok(UiSettings::try_from(UiKeysPartial::from_env(&Env::of(
|
||||
variables,
|
||||
))?)?)
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn defaults_allow_startup_without_ui_credentials() {
|
||||
let settings: Settings = parse(&[]).unwrap();
|
||||
let settings = boot(&[]).unwrap();
|
||||
|
||||
assert_eq!(settings.host, "0.0.0.0");
|
||||
assert_eq!(settings.port, 4000);
|
||||
assert_eq!(settings.litellm_config, "config.yaml");
|
||||
assert_eq!(settings.ui_path, None);
|
||||
assert_eq!(settings.log_level(), Level::INFO);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn environment_overrides_startup_settings() {
|
||||
let settings: Settings = parse(&[
|
||||
let settings = boot(&[
|
||||
("HOST", "127.0.0.1"),
|
||||
("PORT", "8080"),
|
||||
("LITELLM_CONFIG", "/tmp/custom.yaml"),
|
||||
("RUST_LOG", "debug"),
|
||||
("LITELLM_UI_PATH", "/srv/ui"),
|
||||
("LITELLM_UI_SECURE_COOKIES", "invalid-but-ui-disabled"),
|
||||
])
|
||||
.unwrap();
|
||||
|
|
@ -98,6 +129,7 @@ mod tests {
|
|||
assert_eq!(settings.host, "127.0.0.1");
|
||||
assert_eq!(settings.port, 8080);
|
||||
assert_eq!(settings.litellm_config, "/tmp/custom.yaml");
|
||||
assert_eq!(settings.ui_path.as_deref(), Some("/srv/ui"));
|
||||
assert_eq!(settings.log_level(), Level::DEBUG);
|
||||
}
|
||||
|
||||
|
|
@ -106,23 +138,42 @@ mod tests {
|
|||
#[case::filter("litellm=debug")]
|
||||
#[case::invalid("invalid")]
|
||||
fn unrecognized_log_levels_fall_back_to_info(#[case] value: &str) {
|
||||
let settings: Settings = parse(&[("RUST_LOG", value)]).unwrap();
|
||||
|
||||
assert_eq!(settings.log_level(), Level::INFO);
|
||||
assert_eq!(
|
||||
boot(&[("RUST_LOG", value)]).unwrap().log_level(),
|
||||
Level::INFO
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::empty("")]
|
||||
#[case::negative("-1")]
|
||||
#[case::overflow("65536")]
|
||||
#[case::invalid("http")]
|
||||
fn invalid_ports_fail_startup(#[case] value: &str) {
|
||||
assert!(parse::<Settings>(&[("PORT", value)]).is_err());
|
||||
#[case::invalid("abc")]
|
||||
#[case::empty("")]
|
||||
#[case::blank(" ")]
|
||||
fn invalid_ports_fail_startup_naming_the_variable(#[case] value: &str) {
|
||||
let error = boot(&[("PORT", value)]).err().unwrap();
|
||||
assert_eq!(error.name, "PORT");
|
||||
assert!(error.to_string().contains("PORT"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn blank_text_settings_remain_set() {
|
||||
let settings = boot(&[
|
||||
("HOST", ""),
|
||||
("LITELLM_CONFIG", ""),
|
||||
("LITELLM_UI_PATH", ""),
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(settings.host, "");
|
||||
assert_eq!(settings.litellm_config, "");
|
||||
assert_eq!(settings.port, 4000);
|
||||
assert_eq!(settings.ui_path.as_deref(), Some(""));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn ui_defaults_keep_secure_cookies_and_preserve_password() {
|
||||
let settings: UiSettings = parse(&[("UI_PASSWORD", " secret,with spaces ")]).unwrap();
|
||||
let settings = ui(&[("UI_PASSWORD", " secret,with spaces ")]).unwrap();
|
||||
|
||||
assert_eq!(settings.ui_username, "admin");
|
||||
assert_eq!(settings.ui_password.expose(), " secret,with spaces ");
|
||||
|
|
@ -133,7 +184,7 @@ mod tests {
|
|||
#[case::secure("true", true)]
|
||||
#[case::insecure("false", false)]
|
||||
fn ui_environment_overrides(#[case] value: &str, #[case] expected: bool) {
|
||||
let settings: UiSettings = parse(&[
|
||||
let settings = ui(&[
|
||||
("UI_USERNAME", "operator"),
|
||||
("UI_PASSWORD", "password"),
|
||||
("LITELLM_UI_SECURE_COOKIES", value),
|
||||
|
|
@ -148,7 +199,16 @@ mod tests {
|
|||
#[case::missing_password(&[])]
|
||||
#[case::invalid_cookie_flag(&[("UI_PASSWORD", "password"), ("LITELLM_UI_SECURE_COOKIES", "invalid")])]
|
||||
fn invalid_ui_settings_fail(#[case] variables: &[(&str, &str)]) {
|
||||
assert!(parse::<UiSettings>(variables).is_err());
|
||||
assert!(ui(variables).is_err());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::empty("")]
|
||||
#[case::whitespace(" ")]
|
||||
fn blank_ui_credentials_remain_set(#[case] value: &str) {
|
||||
let settings = ui(&[("UI_PASSWORD", value), ("UI_USERNAME", value)]).unwrap();
|
||||
assert_eq!(settings.ui_password.expose(), value);
|
||||
assert_eq!(settings.ui_username, value);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
|
|
@ -158,12 +218,12 @@ mod tests {
|
|||
fn non_unicode_environment_values_are_ignored(#[case] key: &str) {
|
||||
use std::os::unix::ffi::OsStringExt;
|
||||
|
||||
let settings: Settings = from_iter([
|
||||
let env = Env::from_vars([
|
||||
(OsString::from(key), OsString::from_vec(vec![0xff])),
|
||||
(OsString::from_vec(vec![0xff]), OsString::from("value")),
|
||||
(OsString::from("PORT"), OsString::from("8080")),
|
||||
])
|
||||
.unwrap();
|
||||
]);
|
||||
let settings = Settings::from(BootKeysPartial::from_env(&env).unwrap());
|
||||
|
||||
assert_eq!(settings.host, "0.0.0.0");
|
||||
assert_eq!(settings.port, 8080);
|
||||
|
|
|
|||
|
|
@ -126,7 +126,8 @@ mcp_servers:
|
|||
upstream.url
|
||||
))
|
||||
.unwrap();
|
||||
let inference = litellm_gateway::build_inference(&config).unwrap();
|
||||
let inference =
|
||||
litellm_gateway::build_inference(&config, &litellm_layered::Env::default()).unwrap();
|
||||
let shutdown = CancellationToken::new();
|
||||
let mcp = litellm_gateway::build_mcp(
|
||||
&config,
|
||||
|
|
@ -243,7 +244,8 @@ async fn rejects_unsupported_or_invalid_settings_before_serving(
|
|||
#[case] expected: &str,
|
||||
) {
|
||||
let config = Config::from_yaml(&format!("general_settings: {{master_key: gateway-key}}\nmcp_servers:\n docs:\n url: http://127.0.0.1:1/mcp\n {settings}\n")).unwrap();
|
||||
let inference = litellm_gateway::build_inference(&config).unwrap();
|
||||
let inference =
|
||||
litellm_gateway::build_inference(&config, &litellm_layered::Env::default()).unwrap();
|
||||
let result = litellm_gateway::build_mcp(
|
||||
&config,
|
||||
Arc::new(NoSecrets),
|
||||
|
|
@ -261,7 +263,8 @@ async fn rejects_unsupported_or_invalid_settings_before_serving(
|
|||
#[tokio::test]
|
||||
async fn mcp_is_optional_and_requires_admission_credentials_when_enabled() {
|
||||
let empty = Config::from_yaml("{}").unwrap();
|
||||
let inference = litellm_gateway::build_inference(&empty).unwrap();
|
||||
let inference =
|
||||
litellm_gateway::build_inference(&empty, &litellm_layered::Env::default()).unwrap();
|
||||
assert!(
|
||||
litellm_gateway::build_mcp(
|
||||
&empty,
|
||||
|
|
@ -311,7 +314,9 @@ fn applies_configured_http_environment_before_building_clients() {
|
|||
missing.display(),
|
||||
))
|
||||
.unwrap();
|
||||
let error = litellm_gateway::build_inference(&config).err().unwrap();
|
||||
let error = litellm_gateway::build_inference(&config, &litellm_layered::Env::default())
|
||||
.err()
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
error,
|
||||
litellm_gateway::Error::Http(litellm_http::Error::Read { path, .. })
|
||||
|
|
@ -329,7 +334,8 @@ async fn refuses_unimplemented_global_mcp_policy(#[case] settings: &str) {
|
|||
"{settings}\nmcp_servers: {{docs: {{url: 'http://127.0.0.1:1/mcp'}}}}\n"
|
||||
))
|
||||
.unwrap();
|
||||
let inference = litellm_gateway::build_inference(&config).unwrap();
|
||||
let inference =
|
||||
litellm_gateway::build_inference(&config, &litellm_layered::Env::default()).unwrap();
|
||||
let error = litellm_gateway::build_mcp(
|
||||
&config,
|
||||
Arc::new(NoSecrets),
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ use rstest::{fixture, rstest};
|
|||
use serde_json::{Value, json};
|
||||
use tokio::{net::TcpListener, sync::oneshot, time::timeout};
|
||||
use tower::ServiceExt;
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method};
|
||||
|
||||
struct LogSink(mpsc::Sender<Value>);
|
||||
|
||||
|
|
@ -30,7 +31,11 @@ impl Sink for LogSink {
|
|||
|
||||
#[fixture]
|
||||
fn inference() -> Arc<Gateway> {
|
||||
litellm_gateway::build_inference(&Config::from_yaml("model_list: []").unwrap()).unwrap()
|
||||
litellm_gateway::build_inference(
|
||||
&Config::from_yaml("model_list: []").unwrap(),
|
||||
&litellm_layered::Env::default(),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
|
|
@ -113,6 +118,174 @@ async fn authenticates_before_serving_mounted_inference_routes(
|
|||
.unwrap();
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn ocr_gives_up_at_the_request_timeout_from_config_environment_variables() {
|
||||
let upstream = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.set_delay(Duration::from_secs(3))
|
||||
.set_body_json(json!({"pages": []})),
|
||||
)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let config = Config::from_yaml(&format!(
|
||||
"model_list:
|
||||
- model_name: public/ocr
|
||||
litellm_params:
|
||||
model: mistral/test-ocr
|
||||
api_base: {}
|
||||
api_key: test-key
|
||||
general_settings:
|
||||
master_key: gateway-key
|
||||
environment_variables:
|
||||
REQUEST_TIMEOUT: '0.2'
|
||||
",
|
||||
upstream.uri()
|
||||
))
|
||||
.unwrap();
|
||||
let inference =
|
||||
litellm_gateway::build_inference(&config, &litellm_layered::Env::default()).unwrap();
|
||||
let request = Request::post("/ocr")
|
||||
.header("authorization", "Bearer gateway-key")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "public/ocr",
|
||||
"document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"},
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.unwrap();
|
||||
|
||||
let response = timeout(
|
||||
Duration::from_secs(2),
|
||||
litellm_gateway::router(inference, &config, None, None).oneshot(request),
|
||||
)
|
||||
.await
|
||||
.expect("OCR must give up at REQUEST_TIMEOUT instead of the default")
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status().as_u16(), 408);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn messages_thinking_budgets_read_config_environment_variables() {
|
||||
let upstream = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-4-5",
|
||||
"content": [{"type": "text", "text": "hi"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1},
|
||||
})))
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let config = Config::from_yaml(&format!(
|
||||
"model_list:
|
||||
- model_name: public/claude
|
||||
litellm_params:
|
||||
model: anthropic/claude-sonnet-4-5
|
||||
api_key: sk-ant-test
|
||||
api_base: {}
|
||||
general_settings:
|
||||
master_key: gateway-key
|
||||
environment_variables:
|
||||
DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET: '2000'
|
||||
",
|
||||
upstream.uri()
|
||||
))
|
||||
.unwrap();
|
||||
let inference =
|
||||
litellm_gateway::build_inference(&config, &litellm_layered::Env::default()).unwrap();
|
||||
let request = Request::post("/v1/messages")
|
||||
.header("authorization", "Bearer gateway-key")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "public/claude",
|
||||
"max_tokens": 4096,
|
||||
"reasoning_effort": "low",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.unwrap();
|
||||
|
||||
let response = timeout(
|
||||
Duration::from_secs(10),
|
||||
litellm_gateway::router(inference, &config, None, None).oneshot(request),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status().as_u16(), 200);
|
||||
let sent: Value = upstream.received_requests().await.unwrap()[0]
|
||||
.body_json()
|
||||
.unwrap();
|
||||
assert_eq!(sent["thinking"]["budget_tokens"], 2000);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn messages_auth_reads_credentials_from_config_environment_variables() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let credentials = directory.path().join("service-account.json");
|
||||
std::fs::write(&credentials, "not a service account").unwrap();
|
||||
let config = Config::from_yaml(&format!(
|
||||
"model_list:
|
||||
- model_name: public/vertex
|
||||
litellm_params:
|
||||
model: vertex_ai/claude-sonnet-4-5
|
||||
vertex_project: test-project
|
||||
vertex_location: us-east5
|
||||
general_settings:
|
||||
master_key: gateway-key
|
||||
environment_variables:
|
||||
GOOGLE_APPLICATION_CREDENTIALS: '{}'
|
||||
",
|
||||
credentials.display()
|
||||
))
|
||||
.unwrap();
|
||||
let inference =
|
||||
litellm_gateway::build_inference(&config, &litellm_layered::Env::default()).unwrap();
|
||||
let request = Request::post("/v1/messages")
|
||||
.header("authorization", "Bearer gateway-key")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
json!({
|
||||
"model": "public/vertex",
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
})
|
||||
.to_string(),
|
||||
))
|
||||
.unwrap();
|
||||
|
||||
let response = timeout(
|
||||
Duration::from_secs(10),
|
||||
litellm_gateway::router(inference, &config, None, None).oneshot(request),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
let body = axum::body::to_bytes(response.into_body(), 64 * 1024)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
String::from_utf8_lossy(&body).contains("ApplicationCredentials"),
|
||||
"{}",
|
||||
String::from_utf8_lossy(&body)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn logs_request_outcome_without_credentials_or_query(inference: Arc<Gateway>) {
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ license.workspace = true
|
|||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
litellm-layered.workspace = true
|
||||
bytes.workspace = true
|
||||
futures-util.workspace = true
|
||||
litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] }
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ pub(super) async fn execute(
|
|||
model,
|
||||
call,
|
||||
secrets.as_ref(),
|
||||
route.env.as_ref(),
|
||||
context,
|
||||
)
|
||||
.boxed()
|
||||
|
|
@ -100,6 +101,7 @@ async fn provider_call(
|
|||
model: String,
|
||||
call: MessagesCall,
|
||||
secrets: &(dyn Lookup + Send + Sync),
|
||||
env: &(dyn Lookup + Send + Sync),
|
||||
context: &CallContext<'_, impl Interceptors<Error>>,
|
||||
) -> Result<ProviderCall, Error> {
|
||||
let MessagesCall {
|
||||
|
|
@ -143,7 +145,7 @@ async fn provider_call(
|
|||
|
||||
let body = config.transform_anthropic_messages_request(
|
||||
trimmed,
|
||||
&MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
|
||||
&MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params, env),
|
||||
)?;
|
||||
let environment = ValidatedEnvironment {
|
||||
headers: config.request_headers(
|
||||
|
|
@ -168,7 +170,10 @@ async fn provider_call(
|
|||
secret_fields: Vec::new(),
|
||||
api_key: api_key.map(SecretValue::new),
|
||||
};
|
||||
let authenticated = resolve_auth(auth, environment, &|key| std::env::var(key).ok()).await?;
|
||||
let authenticated = resolve_auth(auth, environment, &|key| {
|
||||
secrets.get(key).or_else(|| env.get(key))
|
||||
})
|
||||
.await?;
|
||||
let identity = ProviderIdentity {
|
||||
model: request_context.model.clone(),
|
||||
provider: request_context.custom_llm_provider.clone(),
|
||||
|
|
@ -436,6 +441,7 @@ mod tests {
|
|||
model,
|
||||
call,
|
||||
secrets,
|
||||
&|_: &str| None,
|
||||
&context,
|
||||
)
|
||||
.await
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ use litellm_auth::AuthServices;
|
|||
use litellm_host::interceptors::Interceptors;
|
||||
use litellm_inference::context::CallContext;
|
||||
use litellm_inference::provider::resolve_llm_provider;
|
||||
use litellm_layered::SharedLookup;
|
||||
use litellm_secrets::source::SecretSource;
|
||||
use std::sync::Arc;
|
||||
|
||||
|
|
@ -25,6 +26,7 @@ pub struct MessagesRoute {
|
|||
http: litellm_http::Client,
|
||||
auth: Arc<AuthServices>,
|
||||
secrets: Arc<dyn SecretSource>,
|
||||
env: SharedLookup,
|
||||
cache: Option<litellm_cache_response::ScopedCache>,
|
||||
}
|
||||
|
||||
|
|
@ -33,11 +35,13 @@ impl MessagesRoute {
|
|||
http: litellm_http::Client,
|
||||
auth: Arc<AuthServices>,
|
||||
secrets: Arc<dyn SecretSource>,
|
||||
env: SharedLookup,
|
||||
) -> Self {
|
||||
Self {
|
||||
http,
|
||||
auth,
|
||||
secrets,
|
||||
env,
|
||||
cache: None,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -69,7 +69,12 @@ impl Interceptors<Error> for LiveCall {
|
|||
#[rstest::fixture]
|
||||
pub(super) fn route() -> MessagesRoute {
|
||||
let resources = LiveResources::default();
|
||||
MessagesRoute::new(resources.http, resources.auth, resources.secrets)
|
||||
MessagesRoute::new(
|
||||
resources.http,
|
||||
resources.auth,
|
||||
resources.secrets,
|
||||
std::sync::Arc::new(litellm_layered::Env::from_process()),
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) async fn request(
|
||||
|
|
|
|||
|
|
@ -277,6 +277,7 @@ async fn the_facade_sends_through_the_injected_http_pool_configuration(call: Mes
|
|||
provider_http(&resources, &Resolution::from(&settings).config),
|
||||
resources.auth,
|
||||
no_secrets(),
|
||||
std::sync::Arc::new(litellm_layered::Env::of(&[])),
|
||||
)
|
||||
.execute(
|
||||
MessagesCall {
|
||||
|
|
@ -360,6 +361,7 @@ async fn route_uses_injected_dependencies_and_optional_cache(
|
|||
provider_http(&resources, &http_config()),
|
||||
resources.auth.clone(),
|
||||
Arc::new(RecordingSecrets::new([("ANTHROPIC_API_KEY", "route-key")])),
|
||||
std::sync::Arc::new(litellm_layered::Env::of(&[])),
|
||||
);
|
||||
let route = if caching {
|
||||
route.with_cache(ScopedCache::new(
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ pub fn messages_route(secrets: Arc<dyn SecretSource>) -> litellm_inference_messa
|
|||
provider_http(&resources, &http_config()),
|
||||
resources.auth,
|
||||
secrets,
|
||||
std::sync::Arc::new(litellm_layered::Env::of(&[])),
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -155,7 +155,7 @@ impl OcrConfigKind {
|
|||
LlmProviders::AzureAi => (None, Some("AZURE_AI_API_BASE")),
|
||||
_ => (None, None),
|
||||
};
|
||||
let secret = |name: &str| secrets.truthy(name);
|
||||
let secret = |name: &str| secrets.get(name).filter(|value| !value.is_empty());
|
||||
let dynamic_api_key = credentials.dynamic_api_key.or_else(|| {
|
||||
credentials.api_key.clone().or_else(|| {
|
||||
preferred_api_key_env
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use futures_util::TryStreamExt;
|
||||
|
|
|
|||
|
|
@ -32,11 +32,15 @@ impl Default for LiveResources {
|
|||
Self {
|
||||
http,
|
||||
auth: resources.auth,
|
||||
secrets: Arc::new(EnvironmentSecrets::python_compatible(secrets_http)),
|
||||
secrets: Arc::new(EnvironmentSecrets::python_compatible(
|
||||
secrets_http,
|
||||
Arc::new(Env::from_process()),
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::disallowed_methods)] // live tests read their own model names
|
||||
pub fn model(route: &str, provider: &str) -> String {
|
||||
let name = format!("LITELLM_LIVE_{route}_{provider}_MODEL").to_ascii_uppercase();
|
||||
std::env::var(&name)
|
||||
|
|
|
|||
|
|
@ -17,6 +17,8 @@ Each runtime writes its own `merge([..])` list per group, and a test per runtime
|
|||
- `from_env` and `from_litellm_params` are generated, and every runtime that has that source calls them
|
||||
- A source holding typed values (the config file, a live Python global) gets a hand-written constructor in the crate that owns the source, never in `layered` or in the group's crate
|
||||
- `layered` never depends on `litellm-config` or pyo3, and no library crate reads the process environment itself. It takes a `Lookup`: `Env` in the Rust gateway, `ProcessEnvironment` in `python-bridge`
|
||||
- `ProcessEnvironment` is private to `python-bridge`, so a library constructor takes the `Lookup` as a required argument and never falls back to the process through `new()`, `Default` or an optional builder step
|
||||
- An SDK that reads the process environment itself, like `aws-config`, gets every value that has an environment form from the `Lookup` first. Only sources with no environment form (profiles, ECS, IMDS) are left to the SDK
|
||||
|
||||
## Hand-written constructors list every field
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use std::{collections::BTreeMap, fmt};
|
||||
use std::{collections::BTreeMap, ffi::OsString, fmt};
|
||||
|
||||
use crate::Lookup;
|
||||
|
||||
|
|
@ -8,8 +8,12 @@ pub struct Env(BTreeMap<String, String>);
|
|||
impl Env {
|
||||
#[allow(clippy::disallowed_methods)]
|
||||
pub fn from_process() -> Self {
|
||||
Self::from_vars(std::env::vars_os())
|
||||
}
|
||||
|
||||
pub fn from_vars(vars: impl IntoIterator<Item = (OsString, OsString)>) -> Self {
|
||||
Self(
|
||||
std::env::vars_os()
|
||||
vars.into_iter()
|
||||
.filter_map(|(name, value)| {
|
||||
Some((name.into_string().ok()?, value.into_string().ok()?))
|
||||
})
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@ pub use litellm_core_utils::{Invalid, coercion::FromEnv, settings::Lookup};
|
|||
pub use litellm_layered_derive::Layered;
|
||||
pub use sources::Sources;
|
||||
|
||||
pub type SharedLookup = std::sync::Arc<dyn Lookup + Send + Sync>;
|
||||
|
||||
pub trait Partial: Default {
|
||||
fn or(self, lower: Self) -> Self;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ test-support = ["litellm-http/test-support"]
|
|||
[dependencies]
|
||||
litellm-llms-types.workspace = true
|
||||
litellm-core-utils.workspace = true
|
||||
litellm-layered.workspace = true
|
||||
litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] }
|
||||
litellm-auth-aws.workspace = true
|
||||
litellm-auth-azure.workspace = true
|
||||
|
|
|
|||
|
|
@ -371,10 +371,7 @@ pub fn map_openai_context_management_to_anthropic(
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::base_llm::messages::context::{
|
||||
MessagesModelCapabilities, ThinkingBudgets, ThinkingContext,
|
||||
};
|
||||
use std::process::Command;
|
||||
use crate::base_llm::messages::context::MessagesModelCapabilities;
|
||||
|
||||
use rstest::{fixture, rstest};
|
||||
|
||||
|
|
@ -388,7 +385,6 @@ mod tests {
|
|||
const OAUTH_BETA: &str = "oauth-2025-04-20";
|
||||
const BROWSER_ACCESS: (&str, &str) = ("anthropic-dangerous-direct-browser-access", "true");
|
||||
const LOW_BUDGET_ENV: &str = "DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET";
|
||||
const PROCESS_ENV_PROBE: &str = "LITELLM_MESSAGES_TRANSFORM_CONTEXT_PROBE";
|
||||
|
||||
fn merged(base: Value, fields: Value) -> Value {
|
||||
Value::Object(
|
||||
|
|
@ -443,7 +439,7 @@ mod tests {
|
|||
ANTHROPIC_MESSAGES_CONFIG
|
||||
.transform_anthropic_messages_request(
|
||||
request(fields),
|
||||
&MessagesTransformContext::with_lookup(capabilities, drop_params, &no_env),
|
||||
&MessagesTransformContext::new(capabilities, drop_params, &no_env),
|
||||
)
|
||||
.map(|transformed| serde_json::to_value(transformed).unwrap())
|
||||
}
|
||||
|
|
@ -768,7 +764,7 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn thinking_is_translated_with_the_context_budgets() {
|
||||
let context = MessagesTransformContext::with_lookup(
|
||||
let context = MessagesTransformContext::new(
|
||||
MessagesModelCapabilities {
|
||||
supports_reasoning: true,
|
||||
..Default::default()
|
||||
|
|
@ -791,54 +787,6 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_reads_thinking_budgets_from_the_process_environment() {
|
||||
if std::env::var_os(PROCESS_ENV_PROBE).is_some() {
|
||||
assert_eq!(
|
||||
MessagesTransformContext::new(sampling_removed(), true),
|
||||
MessagesTransformContext {
|
||||
thinking: ThinkingContext {
|
||||
capabilities: sampling_removed(),
|
||||
budgets: ThinkingBudgets {
|
||||
low: 2000,
|
||||
..ThinkingBudgets::default()
|
||||
},
|
||||
},
|
||||
drop_params: true,
|
||||
}
|
||||
);
|
||||
return;
|
||||
}
|
||||
let (_, test_path) = concat!(
|
||||
module_path!(),
|
||||
"::new_reads_thinking_budgets_from_the_process_environment"
|
||||
)
|
||||
.split_once("::")
|
||||
.unwrap();
|
||||
let other_tiers = ["MINIMAL", "MEDIUM", "HIGH", "XHIGH", "MAX"]
|
||||
.map(|tier| format!("DEFAULT_REASONING_EFFORT_{tier}_THINKING_BUDGET"));
|
||||
let output = other_tiers
|
||||
.iter()
|
||||
.fold(
|
||||
Command::new(std::env::current_exe().unwrap()),
|
||||
|mut command, name| {
|
||||
command.env_remove(name);
|
||||
command
|
||||
},
|
||||
)
|
||||
.args([test_path, "--exact"])
|
||||
.env(PROCESS_ENV_PROBE, "1")
|
||||
.env(LOW_BUDGET_ENV, "2000")
|
||||
.output()
|
||||
.unwrap();
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
assert!(
|
||||
output.status.success() && stdout.contains("1 passed"),
|
||||
"{stdout}{}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::public_endpoint(None, &[], "https://api.anthropic.com/v1/messages")]
|
||||
#[case::base_url_env(
|
||||
|
|
|
|||
|
|
@ -379,7 +379,7 @@ mod tests {
|
|||
"inference_geo": "us",
|
||||
"litellm_metadata": {"trace": "abc"}
|
||||
});
|
||||
let context = MessagesTransformContext::with_lookup(
|
||||
let context = MessagesTransformContext::new(
|
||||
MessagesModelCapabilities {
|
||||
supports_reasoning: true,
|
||||
supports_adaptive_thinking: true,
|
||||
|
|
@ -406,6 +406,7 @@ mod tests {
|
|||
..MessagesModelCapabilities::default()
|
||||
},
|
||||
false,
|
||||
&|_: &str| None,
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -507,7 +508,7 @@ mod tests {
|
|||
{"role": "assistant", "content": "hello"}
|
||||
]
|
||||
});
|
||||
let context = MessagesTransformContext::with_lookup(
|
||||
let context = MessagesTransformContext::new(
|
||||
MessagesModelCapabilities {
|
||||
supports_reasoning: true,
|
||||
supports_adaptive_thinking: true,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use litellm_core_utils::settings::{Lookup, ProcessEnvironment};
|
||||
use litellm_core_utils::settings::Lookup;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
|
|
@ -89,10 +89,11 @@ impl Default for ThinkingBudgets {
|
|||
}
|
||||
|
||||
impl ThinkingBudgets {
|
||||
pub fn from_lookup(env: &impl Lookup) -> Self {
|
||||
pub fn from_lookup(env: &(impl Lookup + ?Sized)) -> Self {
|
||||
let defaults = Self::default();
|
||||
let tier = |name: &str, default: u64| {
|
||||
env.parsed::<u64>(&format!("DEFAULT_REASONING_EFFORT_{name}_THINKING_BUDGET"))
|
||||
env.get(&format!("DEFAULT_REASONING_EFFORT_{name}_THINKING_BUDGET"))
|
||||
.and_then(|value| value.trim().parse().ok())
|
||||
.unwrap_or(default)
|
||||
};
|
||||
Self {
|
||||
|
|
@ -117,14 +118,10 @@ pub struct MessagesTransformContext {
|
|||
pub drop_params: bool,
|
||||
}
|
||||
impl MessagesTransformContext {
|
||||
pub fn new(capabilities: MessagesModelCapabilities, drop_params: bool) -> Self {
|
||||
Self::with_lookup(capabilities, drop_params, &ProcessEnvironment)
|
||||
}
|
||||
|
||||
pub fn with_lookup(
|
||||
pub fn new(
|
||||
capabilities: MessagesModelCapabilities,
|
||||
drop_params: bool,
|
||||
env: &impl Lookup,
|
||||
env: &(impl Lookup + ?Sized),
|
||||
) -> Self {
|
||||
Self {
|
||||
thinking: ThinkingContext {
|
||||
|
|
|
|||
|
|
@ -91,6 +91,7 @@ impl OcrClient {
|
|||
secrets: Arc::new(
|
||||
litellm_secrets::source::EnvironmentSecrets::python_compatible(
|
||||
provider_http.clone(),
|
||||
Arc::new(|_: &str| None),
|
||||
),
|
||||
),
|
||||
provider_http,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use litellm_core_utils::settings::Lookup;
|
||||
use litellm_core_utils::coercion;
|
||||
use litellm_layered::{Invalid, Layered};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct OcrSettings {
|
||||
|
|
@ -25,70 +26,88 @@ impl Default for OcrSettings {
|
|||
}
|
||||
}
|
||||
|
||||
impl OcrSettings {
|
||||
pub fn from_environment(env: &impl Lookup) -> Self {
|
||||
#[derive(Layered)]
|
||||
pub struct OcrKeys {
|
||||
#[layer(env = "REQUEST_TIMEOUT", read = timeout_seconds)]
|
||||
pub request_timeout: Duration,
|
||||
#[layer(env = "MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", read = download_megabytes)]
|
||||
pub max_download_bytes: u64,
|
||||
#[layer(env = "AZURE_OPERATION_POLLING_TIMEOUT", read = polling_seconds)]
|
||||
pub poll_timeout: Duration,
|
||||
#[layer(env = "AZURE_DOCUMENT_INTELLIGENCE_API_VERSION")]
|
||||
pub document_intelligence_api_version: String,
|
||||
#[layer(env = "AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI")]
|
||||
pub document_intelligence_dpi: i64,
|
||||
pub enable_azure_ad_token_refresh: bool,
|
||||
}
|
||||
|
||||
impl From<OcrKeysPartial> for OcrSettings {
|
||||
fn from(merged: OcrKeysPartial) -> Self {
|
||||
let defaults = Self::default();
|
||||
Self {
|
||||
request_timeout: env
|
||||
.parsed::<f64>("REQUEST_TIMEOUT")
|
||||
.and_then(|seconds| Duration::try_from_secs_f64(seconds).ok())
|
||||
.unwrap_or(defaults.request_timeout),
|
||||
max_download_bytes: env
|
||||
.parsed::<f64>("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB")
|
||||
.filter(|size| size.is_finite())
|
||||
.map_or(defaults.max_download_bytes, megabytes),
|
||||
poll_timeout: env
|
||||
.parsed::<i64>("AZURE_OPERATION_POLLING_TIMEOUT")
|
||||
.map_or(defaults.poll_timeout, |seconds| {
|
||||
Duration::from_secs(seconds.max(0).unsigned_abs())
|
||||
}),
|
||||
document_intelligence_api_version: env
|
||||
.get("AZURE_DOCUMENT_INTELLIGENCE_API_VERSION")
|
||||
request_timeout: merged.request_timeout.unwrap_or(defaults.request_timeout),
|
||||
max_download_bytes: merged
|
||||
.max_download_bytes
|
||||
.unwrap_or(defaults.max_download_bytes),
|
||||
poll_timeout: merged.poll_timeout.unwrap_or(defaults.poll_timeout),
|
||||
document_intelligence_api_version: merged
|
||||
.document_intelligence_api_version
|
||||
.unwrap_or(defaults.document_intelligence_api_version),
|
||||
document_intelligence_dpi: env
|
||||
.parsed("AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI")
|
||||
document_intelligence_dpi: merged
|
||||
.document_intelligence_dpi
|
||||
.unwrap_or(defaults.document_intelligence_dpi),
|
||||
..defaults
|
||||
enable_azure_ad_token_refresh: merged
|
||||
.enable_azure_ad_token_refresh
|
||||
.unwrap_or(defaults.enable_azure_ad_token_refresh),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn timeout_seconds(value: &str) -> Result<Option<Duration>, Invalid> {
|
||||
Ok(coercion::parsed::<f64>(value)?
|
||||
.and_then(|seconds| Duration::try_from_secs_f64(seconds).ok()))
|
||||
}
|
||||
|
||||
fn download_megabytes(value: &str) -> Result<Option<u64>, Invalid> {
|
||||
Ok(coercion::parsed::<f64>(value)?
|
||||
.filter(|size| size.is_finite())
|
||||
.map(megabytes))
|
||||
}
|
||||
|
||||
fn polling_seconds(value: &str) -> Result<Option<Duration>, Invalid> {
|
||||
Ok(coercion::parsed::<i64>(value)?
|
||||
.map(|seconds| Duration::from_secs(seconds.max(0).unsigned_abs())))
|
||||
}
|
||||
|
||||
fn megabytes(size: f64) -> u64 {
|
||||
(size * 1024.0 * 1024.0) as u64
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use litellm_layered::Env;
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn env_of(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
|
||||
move |name| {
|
||||
values
|
||||
.iter()
|
||||
.find(|(key, _)| *key == name)
|
||||
.map(|(_, value)| value.to_string())
|
||||
}
|
||||
fn from_env(values: &[(&str, &str)]) -> OcrSettings {
|
||||
OcrSettings::from(OcrKeysPartial::from_env(&Env::of(values)).unwrap())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_environment_keeps_the_python_defaults() {
|
||||
assert_eq!(
|
||||
OcrSettings::from_environment(&env_of(&[])),
|
||||
OcrSettings::default()
|
||||
);
|
||||
assert_eq!(from_env(&[]), OcrSettings::default());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_setting_follows_its_environment_variable() {
|
||||
let settings = OcrSettings::from_environment(&env_of(&[
|
||||
let settings = from_env(&[
|
||||
("REQUEST_TIMEOUT", "30.5"),
|
||||
("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", "0.5"),
|
||||
("AZURE_OPERATION_POLLING_TIMEOUT", " 600 "),
|
||||
("AZURE_DOCUMENT_INTELLIGENCE_API_VERSION", "2025-01-01"),
|
||||
("AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI", "72"),
|
||||
]));
|
||||
]);
|
||||
assert_eq!(
|
||||
settings,
|
||||
OcrSettings {
|
||||
|
|
@ -111,30 +130,25 @@ mod tests {
|
|||
#[case] value: &'static str,
|
||||
#[case] bytes: u64,
|
||||
) {
|
||||
let env =
|
||||
move |name: &str| (name == "MAX_IMAGE_URL_DOWNLOAD_SIZE_MB").then(|| value.to_string());
|
||||
assert_eq!(
|
||||
OcrSettings::from_environment(&env).max_download_bytes,
|
||||
from_env(&[("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", value)]).max_download_bytes,
|
||||
bytes
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_negative_polling_timeout_expires_immediately() {
|
||||
let env =
|
||||
|name: &str| (name == "AZURE_OPERATION_POLLING_TIMEOUT").then(|| "-5".to_string());
|
||||
assert_eq!(
|
||||
OcrSettings::from_environment(&env).poll_timeout,
|
||||
from_env(&[("AZURE_OPERATION_POLLING_TIMEOUT", "-5")]).poll_timeout,
|
||||
Duration::ZERO
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_api_version_is_sent_as_is_like_python_str_of_getenv() {
|
||||
let env =
|
||||
|name: &str| (name == "AZURE_DOCUMENT_INTELLIGENCE_API_VERSION").then(String::new);
|
||||
fn an_empty_api_version_shadows_the_default() {
|
||||
assert_eq!(
|
||||
OcrSettings::from_environment(&env).document_intelligence_api_version,
|
||||
from_env(&[("AZURE_DOCUMENT_INTELLIGENCE_API_VERSION", "")])
|
||||
.document_intelligence_api_version,
|
||||
""
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrDocument, OcrRespo
|
|||
use std::{collections::BTreeMap, future::Future, sync::Arc, time::Duration};
|
||||
|
||||
use litellm_auth::{InputSource, SecretValue, Sourced, TokenProviderHandle};
|
||||
use litellm_core_utils::{call_arguments::CallArguments, settings::ProcessEnvironment};
|
||||
use litellm_core_utils::call_arguments::CallArguments;
|
||||
use litellm_http::outbound::{OutboundRequest, RequestSigner};
|
||||
use litellm_secrets::source::Secrets;
|
||||
use serde::{
|
||||
|
|
@ -147,7 +147,7 @@ impl Default for OcrConnection {
|
|||
ResolvedOcrCredentials::default(),
|
||||
OcrTransportConfig::default(),
|
||||
OcrSettings::default(),
|
||||
Arc::new(ProcessEnvironment),
|
||||
Arc::new(|_: &str| None),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -455,7 +455,7 @@ mod tests {
|
|||
..OcrTransportConfig::default()
|
||||
},
|
||||
settings.clone(),
|
||||
Arc::new(ProcessEnvironment),
|
||||
Arc::new(|_: &str| None),
|
||||
)
|
||||
.timeout
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
|
||||
fn main() {
|
||||
if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos") {
|
||||
println!("cargo:rustc-cdylib-link-arg=-undefined");
|
||||
|
|
|
|||
|
|
@ -100,6 +100,7 @@ impl NativeCacheHandle {
|
|||
&RedisTopology::Standalone,
|
||||
Some(ttl),
|
||||
ResponseCacheCodec,
|
||||
&crate::environment::ProcessEnvironment,
|
||||
)
|
||||
})
|
||||
.map_err(cache_error)?
|
||||
|
|
|
|||
10
litellm-rust/crates/python-bridge/src/environment.rs
Normal file
10
litellm-rust/crates/python-bridge/src/environment.rs
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
use litellm_core_utils::settings::Lookup;
|
||||
|
||||
pub(crate) struct ProcessEnvironment;
|
||||
|
||||
impl Lookup for ProcessEnvironment {
|
||||
#[allow(clippy::disallowed_methods)] // the Python host's settings live in the process environment
|
||||
fn get(&self, name: &str) -> Option<String> {
|
||||
std::env::var(name).ok()
|
||||
}
|
||||
}
|
||||
|
|
@ -4,7 +4,7 @@ use std::{
|
|||
sync::{Arc, LazyLock, Mutex, PoisonError},
|
||||
};
|
||||
|
||||
use litellm_core_utils::settings::ProcessEnvironment;
|
||||
use crate::environment::ProcessEnvironment;
|
||||
use litellm_http::{
|
||||
Client, ClientVariant, HttpClientConfig, HttpClientPool, HttpKeysPartial, HttpSettings,
|
||||
Resolution, SslVerify, TlsSource, Unsupported,
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ mod callable;
|
|||
mod coercion;
|
||||
mod credentials;
|
||||
mod diagnostics;
|
||||
mod environment;
|
||||
mod errors;
|
||||
mod execution;
|
||||
mod http;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
|
||||
use std::{process::Command, task::Poll};
|
||||
|
||||
use litellm_host::{
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyR
|
|||
.map_err(crate::http::client_error)?,
|
||||
crate::http::resources().auth.clone(),
|
||||
crate::secrets::source(py)?,
|
||||
std::sync::Arc::new(crate::environment::ProcessEnvironment),
|
||||
);
|
||||
Ok(litellm_host::call::hosted_call(
|
||||
request,
|
||||
|
|
|
|||
|
|
@ -3,13 +3,14 @@ mod errors;
|
|||
mod host;
|
||||
mod project;
|
||||
|
||||
use crate::environment::ProcessEnvironment;
|
||||
use host::OcrPythonHost;
|
||||
use litellm_callbacks_legacy_python::LoggingOperation;
|
||||
use litellm_core_utils::settings::ProcessEnvironment;
|
||||
use litellm_host_python::to_py;
|
||||
use litellm_inference_ocr::provider_config;
|
||||
use litellm_llms::base_llm::ocr::settings::OcrSettings;
|
||||
use pyo3::prelude::*;
|
||||
use litellm_layered::merge;
|
||||
use litellm_llms::base_llm::ocr::settings::{OcrKeysPartial, OcrSettings};
|
||||
use pyo3::{exceptions::PyValueError, prelude::*};
|
||||
|
||||
use super::NativeCall;
|
||||
|
||||
|
|
@ -55,9 +56,21 @@ fn ocr_settings(py: Python<'_>) -> PyResult<OcrSettings> {
|
|||
}
|
||||
|
||||
fn project_provider_defaults(snapshot: &Snapshot<'_>) -> PyResult<OcrSettings> {
|
||||
Ok(OcrSettings {
|
||||
enable_azure_ad_token_refresh: snapshot.read(&ENABLE_AZURE_AD_TOKEN_REFRESH)?,
|
||||
..OcrSettings::from_environment(&ProcessEnvironment)
|
||||
Ok(OcrSettings::from(merge([
|
||||
OcrKeysPartial::from_env(&ProcessEnvironment)
|
||||
.map_err(|error| PyValueError::new_err(error.to_string()))?,
|
||||
globals(snapshot)?,
|
||||
])))
|
||||
}
|
||||
|
||||
fn globals(snapshot: &Snapshot<'_>) -> PyResult<OcrKeysPartial> {
|
||||
Ok(OcrKeysPartial {
|
||||
request_timeout: None,
|
||||
max_download_bytes: None,
|
||||
poll_timeout: None,
|
||||
document_intelligence_api_version: None,
|
||||
document_intelligence_dpi: None,
|
||||
enable_azure_ad_token_refresh: Some(snapshot.read(&ENABLE_AZURE_AD_TOKEN_REFRESH)?),
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ pub(crate) fn source(py: Python<'_>) -> PyResult<Arc<dyn SecretSource>> {
|
|||
let Some(snapshot) = PythonSettings::SecretManager.read_or_unset(py)? else {
|
||||
return Ok(Arc::new(EnvironmentSecrets::python_compatible(
|
||||
crate::http::host_client(py, litellm_http::ClientVariant::Provider)?,
|
||||
Arc::new(crate::environment::ProcessEnvironment),
|
||||
)));
|
||||
};
|
||||
if snapshot.read(&NATIVE)? {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use crate::environment::ProcessEnvironment;
|
||||
use futures_util::future::BoxFuture;
|
||||
use litellm_core_utils::settings::ProcessEnvironment;
|
||||
use litellm_host_python::PythonContext;
|
||||
use litellm_http::Client;
|
||||
use litellm_secrets::source::SecretSource;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
use std::{collections::BTreeMap, sync::Arc};
|
||||
|
||||
use litellm_core_utils::settings::{Lookup, ProcessEnvironment};
|
||||
use litellm_core_utils::settings::Lookup;
|
||||
|
||||
use crate::environment::ProcessEnvironment;
|
||||
use litellm_host_python::{from_py, json_object_field, to_py};
|
||||
use litellm_secrets::{
|
||||
KeyManagementSettings, KeyManagementSystem, Secret, SecretManager, load_native_manager,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
|
||||
//! Replays `generated/values.json`, which CPython wrote with `scripts/generate_fixtures.py`.
|
||||
|
||||
use std::{collections::BTreeMap, fs::File, io::Write};
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
|
||||
use std::sync::Arc;
|
||||
|
||||
use litellm_core_utils::settings::ProcessEnvironment;
|
||||
use litellm_secrets_azure::AzureKeyVault;
|
||||
use litellm_secrets_types::Secret;
|
||||
use rstest::rstest;
|
||||
|
|
@ -9,7 +9,7 @@ use rstest::rstest;
|
|||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn reads_a_real_secret() {
|
||||
let environment = Arc::new(ProcessEnvironment);
|
||||
let environment = Arc::new(|name: &str| std::env::var(name).ok());
|
||||
let manager = AzureKeyVault::new(litellm_http::Client::plain_for_test(), environment).unwrap();
|
||||
let name = std::env::var("AZURE_KEY_VAULT_LIVE_SECRET_NAME").unwrap();
|
||||
let secret = manager.get_secret(&name).await.unwrap().unwrap();
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
|
|
@ -282,7 +283,7 @@ async fn rotation_applies_timeout_to_each_request(token_values: Vec<(&str, &str)
|
|||
#[ignore]
|
||||
async fn live_vault_round_trip() {
|
||||
let environment: Arc<dyn Lookup + Send + Sync> =
|
||||
Arc::new(litellm_core_utils::settings::ProcessEnvironment);
|
||||
Arc::new(|name: &str| std::env::var(name).ok());
|
||||
let manager: HashicorpVault = HashicorpVault::new(environment, true).unwrap();
|
||||
let name: String = std::env::var("LITELLM_VAULT_LIVE_SECRET_NAME").unwrap();
|
||||
let value: SecretValue = SecretValue::new("native-live-value");
|
||||
|
|
|
|||
|
|
@ -40,10 +40,10 @@ impl SecretSource for SecretResolver {
|
|||
pub struct EnvironmentSecrets(SecretResolver);
|
||||
|
||||
impl EnvironmentSecrets {
|
||||
pub fn python_compatible(client: litellm_http::Client) -> Self {
|
||||
pub fn python_compatible(client: litellm_http::Client, env: Secrets) -> Self {
|
||||
Self(SecretResolver::new_python_compatible(
|
||||
Arc::new(crate::SecretManagerState::default()),
|
||||
Arc::new(litellm_core_utils::settings::ProcessEnvironment),
|
||||
env,
|
||||
crate::OidcResolver::new(client),
|
||||
))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,16 +11,17 @@ mod tests {
|
|||
#[tokio::test]
|
||||
async fn python_environment_values_are_absent_like_get_secret_str(
|
||||
#[case] name: &'static str,
|
||||
#[case] value: &str,
|
||||
#[case] value: &'static str,
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
unsafe { std::env::set_var(name, value) };
|
||||
let secret = EnvironmentSecrets::python_compatible(litellm_http::Client::plain_for_test())
|
||||
.resolve(&[name])
|
||||
.await
|
||||
.unwrap()
|
||||
.get(name);
|
||||
unsafe { std::env::remove_var(name) };
|
||||
let secret = EnvironmentSecrets::python_compatible(
|
||||
litellm_http::Client::plain_for_test(),
|
||||
std::sync::Arc::new(move |key: &str| (key == name).then(|| value.to_owned())),
|
||||
)
|
||||
.resolve(&[name])
|
||||
.await
|
||||
.unwrap()
|
||||
.get(name);
|
||||
assert_eq!(secret.as_deref(), expected);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
|||
use crate::Error;
|
||||
use litellm_storage_clickhouse::Connection;
|
||||
|
||||
#[allow(clippy::disallowed_methods)] // the ClickHouse sink is not built from an injected Lookup yet
|
||||
fn max_insert_bytes() -> Result<usize, Error> {
|
||||
let name = "CLICKHOUSE_TRACE_MAX_INSERT_BYTES";
|
||||
match std::env::var(name) {
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
|
||||
use std::{collections::BTreeMap, time::Duration};
|
||||
|
||||
use litellm_http::Client;
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ use litellm_http::Client;
|
|||
|
||||
use crate::{Connection, Error, valid_identifier};
|
||||
|
||||
#[allow(clippy::disallowed_methods)] // the ClickHouse sink is not built from an injected Lookup yet
|
||||
fn insert_timeout() -> Result<Duration, Error> {
|
||||
let name = "CLICKHOUSE_INSERT_TIMEOUT_SECONDS";
|
||||
match std::env::var(name) {
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use litellm_http::Client;
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ impl HttpFetch {
|
|||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::disallowed_methods)] // test tooling reads its own token
|
||||
pub fn from_env() -> Self {
|
||||
Self::new(std::env::var("GITHUB_TOKEN").ok())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
#![allow(clippy::disallowed_methods)] // test and build tooling read their own settings from the process environment
|
||||
//! Drives the real agents through a real gateway. Run with `cargo test -p litellm-testkit --test live -- --ignored`
|
||||
//! after exporting `TESTKIT_GATEWAY_URL`, `TESTKIT_GATEWAY_KEY`, one `TESTKIT_MODEL_<WIRE>` per wire
|
||||
//! (`MESSAGES`, `RESPONSES`, `CHAT_COMPLETIONS`) and one `TESTKIT_<AGENT>_VERSION` per agent
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue