refactor(auth-types): derive AWS and Vertex credential sources and remove ParamSpec
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled

AWS and Vertex params are declared with the Layered derive, which replaces ParamSpec, and vertex_project and vertex_location become typed config settings

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yujong Lee 2026-10-26 10:07:59 -07:00
parent b61449e35d
commit e3ad07aa68
33 changed files with 584 additions and 467 deletions

View file

@ -3366,6 +3366,7 @@ dependencies = [
"aws-types",
"litellm-auth-types",
"litellm-http",
"litellm-layered",
"moka",
"reqwest 0.12.28",
"rstest",
@ -3404,6 +3405,7 @@ dependencies = [
"google-cloud-auth",
"http 1.4.2",
"litellm-auth-types",
"litellm-layered",
"moka",
"rstest",
"serde",
@ -3418,6 +3420,7 @@ dependencies = [
name = "litellm-auth-types"
version = "0.1.0"
dependencies = [
"litellm-layered",
"rstest",
"serde",
"serde_json",

View file

@ -6,6 +6,7 @@ license.workspace = true
repository.workspace = true
[dependencies]
litellm-layered.workspace = true
litellm-auth-types.workspace = true
litellm-http.workspace = true

View file

@ -13,15 +13,14 @@ use aws_sigv4::http_request::{
use aws_sigv4::sign::v4;
use aws_smithy_runtime_api::client::identity::Identity;
use litellm_auth_types::AwsParams;
use litellm_auth_types::{AwsKeysPartial, AwsParams};
use litellm_layered::merge;
use super::Error;
use super::constants::{
AWS_ACCESS_KEY_ID, AWS_EXTERNAL_ID, AWS_PROFILE, AWS_PROFILE_NAME, AWS_REGION_NAME,
AWS_ROLE_ARN, AWS_ROLE_NAME, AWS_ROLE_SESSION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_NAME,
AWS_SESSION_TOKEN, AWS_SIGNED_HEADER_NAMES, AWS_STS_ENDPOINT, AWS_WEB_IDENTITY_TOKEN,
AWS_WEB_IDENTITY_TOKEN_FILE, DEFAULT_BEDROCK_REGION, DEFAULT_SESSION_NAME_PREFIX,
SIGV4_COMPUTED_HEADER_NAMES,
AWS_ACCESS_KEY_ID, AWS_PROFILE, AWS_ROLE_ARN, AWS_ROLE_SESSION_NAME, AWS_SECRET_ACCESS_KEY,
AWS_SESSION_TOKEN, AWS_SIGNED_HEADER_NAMES, AWS_WEB_IDENTITY_TOKEN_FILE,
DEFAULT_BEDROCK_REGION, DEFAULT_SESSION_NAME_PREFIX, SIGV4_COMPUTED_HEADER_NAMES,
};
const STATIC_CREDENTIALS_TTL: Duration = Duration::from_secs(3600 - 60);
@ -75,21 +74,39 @@ pub struct AwsAuthConfig {
impl AwsAuthConfig {
fn with_environment(self, env_lookup: &(dyn Fn(&str) -> Option<String> + Sync)) -> Self {
Self::from(merge([self.keys(), env_keys(env_lookup)]))
}
fn keys(self) -> AwsKeysPartial {
AwsKeysPartial {
access_key_id: self.access_key_id,
secret_access_key: self.secret_access_key,
session_token: self.session_token,
region: self.region_name,
session_name: self.session_name,
profile_name: self.profile_name,
role_name: self.role_name,
web_identity_token: self.web_identity_token,
sts_endpoint: self.sts_endpoint,
external_id: self.external_id,
bedrock_runtime_endpoint: None,
}
}
}
impl From<AwsKeysPartial> for AwsAuthConfig {
fn from(keys: AwsKeysPartial) -> Self {
Self {
access_key_id: self.access_key_id.or_else(|| env_lookup(AWS_ACCESS_KEY_ID)),
secret_access_key: self
.secret_access_key
.or_else(|| env_lookup(AWS_SECRET_ACCESS_KEY)),
session_token: self.session_token.or_else(|| env_lookup(AWS_SESSION_TOKEN)),
region_name: self.region_name.or_else(|| env_lookup(AWS_REGION_NAME)),
session_name: self.session_name.or_else(|| env_lookup(AWS_SESSION_NAME)),
profile_name: self.profile_name.or_else(|| env_lookup(AWS_PROFILE_NAME)),
role_name: self.role_name.or_else(|| env_lookup(AWS_ROLE_NAME)),
web_identity_token: self
.web_identity_token
.or_else(|| env_lookup(AWS_WEB_IDENTITY_TOKEN)),
sts_endpoint: self.sts_endpoint.or_else(|| env_lookup(AWS_STS_ENDPOINT)),
external_id: self.external_id.or_else(|| env_lookup(AWS_EXTERNAL_ID)),
access_key_id: keys.access_key_id,
secret_access_key: keys.secret_access_key,
session_token: keys.session_token,
region_name: keys.region,
session_name: keys.session_name,
profile_name: keys.profile_name,
role_name: keys.role_name,
web_identity_token: keys.web_identity_token,
sts_endpoint: keys.sts_endpoint,
external_id: keys.external_id,
}
}
}
@ -612,9 +629,10 @@ pub fn resolve_aws_region(
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Option<String> {
params
.resolve(&AwsParams::REGION, &|_| None)
.keys()
.region
.or_else(|| model_region.map(str::to_string))
.or_else(|| AwsParams::REGION.resolve(&|_| None, env_lookup))
.or_else(|| env_keys(env_lookup).region)
}
pub fn resolve_bedrock_region(
@ -630,18 +648,11 @@ pub fn aws_auth_config(
params: &AwsParams,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> AwsAuthConfig {
AwsAuthConfig {
access_key_id: params.resolve(&AwsParams::ACCESS_KEY_ID, env_lookup),
secret_access_key: params.resolve(&AwsParams::SECRET_ACCESS_KEY, env_lookup),
session_token: params.resolve(&AwsParams::SESSION_TOKEN, env_lookup),
region_name: params.resolve(&AwsParams::REGION, env_lookup),
session_name: params.resolve(&AwsParams::SESSION_NAME, env_lookup),
profile_name: params.resolve(&AwsParams::PROFILE_NAME, env_lookup),
role_name: params.resolve(&AwsParams::ROLE_NAME, env_lookup),
web_identity_token: params.resolve(&AwsParams::WEB_IDENTITY_TOKEN, env_lookup),
sts_endpoint: params.resolve(&AwsParams::STS_ENDPOINT, env_lookup),
external_id: params.resolve(&AwsParams::EXTERNAL_ID, env_lookup),
}
AwsAuthConfig::from(merge([params.keys(), env_keys(env_lookup)]))
}
fn env_keys(env_lookup: &dyn Fn(&str) -> Option<String>) -> AwsKeysPartial {
AwsKeysPartial::from_env(&env_lookup).unwrap_or_default()
}
/// Where the credentials that sign a request come from, decided when the request is
@ -702,7 +713,10 @@ pub fn host_supplied_credentials(params: &AwsParams) -> Option<Credentials> {
#[cfg(test)]
mod tests {
use super::*;
use crate::constants::{AWS_REGION, BEDROCK_SERVICE};
use crate::constants::{
AWS_PROFILE_NAME, AWS_REGION, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, BEDROCK_SERVICE,
};
use litellm_auth_types::AwsKeys;
fn no_env(_: &str) -> Option<String> {
None
@ -724,7 +738,7 @@ mod tests {
seen.lock()
.unwrap()
.iter()
.all(|name| AwsParams::secret_names().contains(&name.as_str()))
.all(|name| AwsKeys::ENV_NAMES.contains(&name.as_str()))
);
}

View file

@ -1,22 +1,22 @@
use litellm_auth_types::AwsParams;
use litellm_auth_types::AwsKeys;
pub const AWS_ACCESS_KEY_ID: &str = AwsParams::ACCESS_KEY_ID.env[0];
pub const AWS_SECRET_ACCESS_KEY: &str = AwsParams::SECRET_ACCESS_KEY.env[0];
pub const AWS_SESSION_TOKEN: &str = AwsParams::SESSION_TOKEN.env[0];
pub const AWS_REGION_NAME: &str = AwsParams::REGION.env[0];
pub const AWS_REGION: &str = AwsParams::REGION.env[1];
pub const AWS_ACCESS_KEY_ID: &str = AwsKeys::ACCESS_KEY_ID.env[0];
pub const AWS_SECRET_ACCESS_KEY: &str = AwsKeys::SECRET_ACCESS_KEY.env[0];
pub const AWS_SESSION_TOKEN: &str = AwsKeys::SESSION_TOKEN.env[0];
pub const AWS_REGION_NAME: &str = AwsKeys::REGION.env[0];
pub const AWS_REGION: &str = AwsKeys::REGION.env[1];
pub const AWS_DEFAULT_REGION: &str = "AWS_DEFAULT_REGION";
pub const AWS_BEDROCK_RUNTIME_ENDPOINT: &str = AwsParams::BEDROCK_RUNTIME_ENDPOINT.env[0];
pub const AWS_SESSION_NAME: &str = AwsParams::SESSION_NAME.env[0];
pub const AWS_PROFILE_NAME: &str = AwsParams::PROFILE_NAME.env[0];
pub const AWS_ROLE_NAME: &str = AwsParams::ROLE_NAME.env[0];
pub const AWS_WEB_IDENTITY_TOKEN: &str = AwsParams::WEB_IDENTITY_TOKEN.env[0];
pub const AWS_BEDROCK_RUNTIME_ENDPOINT: &str = AwsKeys::BEDROCK_RUNTIME_ENDPOINT.env[0];
pub const AWS_SESSION_NAME: &str = AwsKeys::SESSION_NAME.env[0];
pub const AWS_PROFILE_NAME: &str = AwsKeys::PROFILE_NAME.env[0];
pub const AWS_ROLE_NAME: &str = AwsKeys::ROLE_NAME.env[0];
pub const AWS_WEB_IDENTITY_TOKEN: &str = AwsKeys::WEB_IDENTITY_TOKEN.env[0];
pub const AWS_ROLE_ARN: &str = "AWS_ROLE_ARN";
pub const AWS_ROLE_SESSION_NAME: &str = "AWS_ROLE_SESSION_NAME";
pub const AWS_PROFILE: &str = "AWS_PROFILE";
pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = "AWS_WEB_IDENTITY_TOKEN_FILE";
pub const AWS_STS_ENDPOINT: &str = AwsParams::STS_ENDPOINT.env[0];
pub const AWS_EXTERNAL_ID: &str = AwsParams::EXTERNAL_ID.env[0];
pub const AWS_STS_ENDPOINT: &str = AwsKeys::STS_ENDPOINT.env[0];
pub const AWS_EXTERNAL_ID: &str = AwsKeys::EXTERNAL_ID.env[0];
pub const AWS_BEARER_TOKEN_BEDROCK: &str = "AWS_BEARER_TOKEN_BEDROCK";
/// Headers SigV4 covers, beyond the `x-amz-` / `x-amzn-` prefixes. Mirrors
/// Python's `_filter_headers_for_aws_signature` allowlist.

View file

@ -9,6 +9,7 @@ repository.workspace = true
google-sdk = ["dep:google-cloud-auth", "dep:http"]
[dependencies]
litellm-layered.workspace = true
litellm-auth-types.workspace = true
base64.workspace = true

View file

@ -7,7 +7,8 @@ use base64::{
use litellm_auth_types::{Error, ErrorDetail, InputSource, SecretValue, Sourced};
use serde_json::{Map, Value};
use litellm_auth_types::VertexParams;
use litellm_auth_types::{VertexKeys, VertexKeysPartial, VertexParams};
use litellm_layered::merge;
use crate::constants::GOOGLE_APPLICATION_CREDENTIALS_ENV;
@ -36,9 +37,9 @@ impl VertexConfig {
sources: &BTreeMap<String, InputSource>,
) -> Result<Self, Error> {
Ok(Self::new(
optional_credentials(params, sources, VertexParams::CREDENTIALS.wire)?,
optional_string(params, VertexParams::PROJECT.wire)?,
optional_string(params, VertexParams::LOCATION.wire)?,
optional_credentials(params, sources, VertexKeys::CREDENTIALS.litellm_params)?,
optional_string(params, VertexKeys::PROJECT.litellm_params)?,
optional_string(params, VertexKeys::LOCATION.litellm_params)?,
))
}
@ -61,16 +62,28 @@ impl VertexConfig {
pub fn location(&self) -> Option<&str> {
self.location.as_deref()
}
fn keys(&self) -> VertexKeysPartial {
VertexKeysPartial {
credentials: self
.credentials
.as_ref()
.map(|credentials| credentials.value().expose().to_string()),
project: self.project_id.clone(),
location: self.location.clone(),
}
}
fn resolved(&self, env_lookup: &dyn Fn(&str) -> Option<String>) -> VertexKeysPartial {
merge([self.keys(), env_keys(env_lookup)])
}
}
pub fn get_vertex_ai_project(
config: &VertexConfig,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Option<String> {
config
.project_id()
.map(str::to_string)
.or_else(|| VertexParams::PROJECT.resolve(&|_| None, env_lookup))
config.resolved(env_lookup).project
}
/// The project a request is billed to when nothing names it: the `project_id` inside the
@ -104,10 +117,11 @@ pub fn get_vertex_ai_location(
config: &VertexConfig,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Option<String> {
config
.location()
.map(str::to_string)
.or_else(|| VertexParams::LOCATION.resolve(&|_| None, env_lookup))
config.resolved(env_lookup).location
}
fn env_keys(env_lookup: &dyn Fn(&str) -> Option<String>) -> VertexKeysPartial {
VertexKeysPartial::from_env(&env_lookup).unwrap_or_default()
}
const INLINE_JSON_LEADING_NOISE: [char; 7] = [' ', '\t', '\r', '\n', '\u{feff}', '"', '\''];
@ -156,7 +170,7 @@ pub(crate) fn credential_source(
}
};
}
if let Some(configured) = VertexParams::CREDENTIALS.resolve(&|_| None, env_lookup) {
if let Some(configured) = config.resolved(env_lookup).credentials {
return CredentialSource::Trusted(SecretValue::new(configured));
}
non_empty_env(env_lookup, GOOGLE_APPLICATION_CREDENTIALS_ENV)

View file

@ -1,14 +1,14 @@
use std::sync::LazyLock;
use litellm_auth_types::VertexParams;
use litellm_auth_types::VertexKeys;
pub const CLOUD_PLATFORM_SCOPE: &str = "https://www.googleapis.com/auth/cloud-platform";
pub const GOOGLE_OAUTH_TOKEN_ENDPOINT: &str = "https://oauth2.googleapis.com/token";
pub const GOOGLE_APPLICATION_CREDENTIALS_ENV: &str = "GOOGLE_APPLICATION_CREDENTIALS";
pub const VERTEX_AI_API_KEY_ENV: &str = "VERTEX_AI_API_KEY";
pub const VERTEXAI_API_KEY_ENV: &str = "VERTEXAI_API_KEY";
pub const VERTEXAI_CREDENTIALS_ENV: &str = VertexParams::CREDENTIALS.env[0];
pub const VERTEX_LOCATION_ENV: &str = VertexParams::LOCATION.env[1];
pub const VERTEXAI_CREDENTIALS_ENV: &str = VertexKeys::CREDENTIALS.env[0];
pub const VERTEX_LOCATION_ENV: &str = VertexKeys::LOCATION.env[1];
/// Every environment name the Vertex params and the token acquisition read, for hosts
/// that resolve secrets up front.
@ -20,11 +20,7 @@ pub fn secret_names() -> &'static [&'static str] {
GOOGLE_APPLICATION_CREDENTIALS_ENV,
]
.into_iter()
.chain(
VertexParams::SPECS
.iter()
.flat_map(|spec| spec.env.iter().copied()),
)
.chain(VertexKeys::ENV_NAMES.iter().copied())
.fold(Vec::new(), |mut names, name| {
if !names.contains(&name) {
names.push(name);

View file

@ -6,6 +6,7 @@ license.workspace = true
repository.workspace = true
[dependencies]
litellm-layered.workspace = true
serde.workspace = true
serde_json.workspace = true
subtle.workspace = true

View file

@ -19,10 +19,10 @@ pub enum Error {
provider: &'static str,
environment_variable: &'static str,
},
#[error("Missing {provider} {} - {}", .spec.setting, .spec.guidance())]
#[error("Missing {provider} {} - {}", .spec.label, .spec.guidance())]
MissingParam {
provider: &'static str,
spec: &'static crate::ParamSpec,
spec: &'static litellm_layered::Sources,
},
#[error("Missing {provider} API Base - {guidance}")]
MissingApiBase {
@ -108,3 +108,9 @@ impl PartialEq for ErrorSource {
}
impl Eq for ErrorSource {}
impl Error {
pub fn missing(provider: &'static str, spec: &'static litellm_layered::Sources) -> Self {
Self::MissingParam { provider, spec }
}
}

View file

@ -53,7 +53,8 @@ pub use credential::{
};
pub use error::{Error, ErrorDetail, ErrorSource};
pub use http::CredentialPlacement;
pub use params::{AwsParams, ParamSpec, VertexParams};
pub use litellm_layered::Sources;
pub use params::{AwsKeys, AwsKeysPartial, AwsParams, VertexKeys, VertexKeysPartial, VertexParams};
pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy};
pub use secret::SecretValue;
pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle};

View file

@ -1,22 +1,74 @@
use std::sync::LazyLock;
use litellm_layered::Layered;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use veil::Redact;
use super::ParamSpec;
const fn spec(
setting: &'static str,
wire: &'static [&'static str],
env: &'static [&'static str],
) -> ParamSpec {
ParamSpec {
setting,
wire,
module_global: None,
env,
}
#[derive(Layered)]
pub struct AwsKeys {
#[layer(
skip_blank,
litellm_param = "aws_access_key_id",
env = "AWS_ACCESS_KEY_ID"
)]
pub access_key_id: String,
#[layer(
skip_blank,
litellm_param = "aws_secret_access_key",
env = "AWS_SECRET_ACCESS_KEY",
secret
)]
pub secret_access_key: String,
#[layer(
skip_blank,
litellm_param = "aws_session_token",
env = "AWS_SESSION_TOKEN",
secret
)]
pub session_token: String,
#[layer(skip_blank, litellm_param = "aws_region_name", env = ["AWS_REGION_NAME", "AWS_REGION"])]
pub region: String,
#[layer(
skip_blank,
litellm_param = "aws_session_name",
env = "AWS_SESSION_NAME"
)]
pub session_name: String,
#[layer(
skip_blank,
litellm_param = "aws_profile_name",
env = "AWS_PROFILE_NAME"
)]
pub profile_name: String,
#[layer(skip_blank, litellm_param = "aws_role_name", env = "AWS_ROLE_NAME")]
pub role_name: String,
#[layer(
skip_blank,
litellm_param = "aws_web_identity_token",
env = "AWS_WEB_IDENTITY_TOKEN",
secret
)]
pub web_identity_token: String,
#[layer(
skip_blank,
litellm_param = "aws_sts_endpoint",
env = "AWS_STS_ENDPOINT",
label = "STS endpoint"
)]
pub sts_endpoint: String,
#[layer(
skip_blank,
litellm_param = "aws_external_id",
env = "AWS_EXTERNAL_ID",
secret
)]
pub external_id: String,
#[layer(
skip_blank,
litellm_param = "aws_bedrock_runtime_endpoint",
env = "AWS_BEDROCK_RUNTIME_ENDPOINT",
label = "Bedrock runtime endpoint"
)]
pub bedrock_runtime_endpoint: String,
}
/// The `aws_*` fields of Python's `GenericLiteLLMParams`.
@ -51,102 +103,8 @@ pub struct AwsParams {
}
impl AwsParams {
pub const ACCESS_KEY_ID: ParamSpec = spec(
"access key id",
&["aws_access_key_id"],
&["AWS_ACCESS_KEY_ID"],
);
pub const SECRET_ACCESS_KEY: ParamSpec = spec(
"secret access key",
&["aws_secret_access_key"],
&["AWS_SECRET_ACCESS_KEY"],
);
pub const SESSION_TOKEN: ParamSpec = spec(
"session token",
&["aws_session_token"],
&["AWS_SESSION_TOKEN"],
);
pub const REGION: ParamSpec = spec(
"region",
&["aws_region_name"],
&["AWS_REGION_NAME", "AWS_REGION"],
);
pub const SESSION_NAME: ParamSpec =
spec("session name", &["aws_session_name"], &["AWS_SESSION_NAME"]);
pub const PROFILE_NAME: ParamSpec =
spec("profile name", &["aws_profile_name"], &["AWS_PROFILE_NAME"]);
pub const ROLE_NAME: ParamSpec = spec("role name", &["aws_role_name"], &["AWS_ROLE_NAME"]);
pub const WEB_IDENTITY_TOKEN: ParamSpec = spec(
"web identity token",
&["aws_web_identity_token"],
&["AWS_WEB_IDENTITY_TOKEN"],
);
pub const STS_ENDPOINT: ParamSpec =
spec("STS endpoint", &["aws_sts_endpoint"], &["AWS_STS_ENDPOINT"]);
pub const EXTERNAL_ID: ParamSpec =
spec("external id", &["aws_external_id"], &["AWS_EXTERNAL_ID"]);
pub const BEDROCK_RUNTIME_ENDPOINT: ParamSpec = spec(
"Bedrock runtime endpoint",
&["aws_bedrock_runtime_endpoint"],
&["AWS_BEDROCK_RUNTIME_ENDPOINT"],
);
/// Every `aws_*` param Python reads, in declaration order, with the environment names
/// each falls back to.
pub const SPECS: [ParamSpec; 11] = [
Self::ACCESS_KEY_ID,
Self::SECRET_ACCESS_KEY,
Self::SESSION_TOKEN,
Self::REGION,
Self::SESSION_NAME,
Self::PROFILE_NAME,
Self::ROLE_NAME,
Self::WEB_IDENTITY_TOKEN,
Self::STS_ENDPOINT,
Self::EXTERNAL_ID,
Self::BEDROCK_RUNTIME_ENDPOINT,
];
/// The wire names a host projects out of a caller's kwargs, derived from [`Self::SPECS`].
pub fn fields() -> impl Iterator<Item = &'static str> {
Self::SPECS
.iter()
.flat_map(|spec| spec.wire.iter().copied())
}
/// The environment names the specs read, for hosts that resolve secrets up front.
pub fn secret_names() -> &'static [&'static str] {
static NAMES: LazyLock<Vec<&'static str>> = LazyLock::new(|| {
AwsParams::SPECS
.iter()
.flat_map(|spec| spec.env.iter().copied())
.fold(Vec::new(), |mut names, name| {
if !names.contains(&name) {
names.push(name);
}
names
})
});
&NAMES
}
/// The value under a wire name, read through serde so the names can never drift from
/// the struct.
pub fn get(&self, wire: &str) -> Option<String> {
serde_json::to_value(self)
.ok()?
.get(wire)?
.as_str()
.map(str::to_string)
}
/// The spec's value from these params, then the environment.
pub fn resolve(
&self,
spec: &ParamSpec,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Option<String> {
spec.resolve(&|name| self.get(name), env_lookup)
pub fn keys(&self) -> AwsKeysPartial {
AwsKeysPartial::from_litellm_params(&super::as_litellm_params(self))
}
/// Reads the string-valued `aws_*` keys of an untyped params map, ignoring anything else.

View file

@ -1,48 +1,15 @@
mod aws;
mod vertex;
pub use aws::AwsParams;
pub use vertex::VertexParams;
pub use aws::{AwsKeys, AwsKeysPartial, AwsParams};
pub use vertex::{VertexKeys, VertexKeysPartial, VertexParams};
/// One connection param of Python's `CredentialLiteLLMParams` and every place Python reads
/// it from, in order: the wire names on a call or a deployment, the `litellm.<name>` module
/// global a host folds in, then the environment names.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct ParamSpec {
pub setting: &'static str,
pub wire: &'static [&'static str],
pub module_global: Option<&'static str>,
pub env: &'static [&'static str],
}
use serde::Serialize;
use serde_json::{Map, Value};
impl ParamSpec {
/// The first non-blank value: each wire name through `params`, then each environment
/// name through `env`.
pub fn resolve(
&self,
params: &dyn Fn(&str) -> Option<String>,
env: &dyn Fn(&str) -> Option<String>,
) -> Option<String> {
self.wire
.iter()
.map(|name| params(name))
.chain(self.env.iter().map(|name| env(name)))
.find_map(|value| value.filter(|value| !value.trim().is_empty()))
}
pub fn missing(&'static self, provider: &'static str) -> crate::Error {
crate::Error::MissingParam {
provider,
spec: self,
}
}
pub(crate) fn guidance(&self) -> String {
let wire = self.wire.join(" or ");
let env = self.env.join(" or ");
match self.module_global {
Some(global) => format!("pass {wire}, set litellm.{global}, or set {env}"),
None => format!("pass {wire} or set {env}"),
}
fn as_litellm_params(params: &impl Serialize) -> Map<String, Value> {
match serde_json::to_value(params) {
Ok(Value::Object(map)) => map,
_ => Map::new(),
}
}

View file

@ -1,8 +1,30 @@
use litellm_layered::Layered;
use serde::{Deserialize, Deserializer, Serialize, de::Unexpected};
use serde_json::Value;
use veil::Redact;
use super::ParamSpec;
#[derive(Layered)]
pub struct VertexKeys {
#[layer(skip_blank,
litellm_param = ["vertex_credentials", "vertex_ai_credentials"],
deserialize_with = optional_credential_text,
env = "VERTEXAI_CREDENTIALS",
secret
)]
pub credentials: String,
#[layer(skip_blank,
litellm_param = ["vertex_project", "vertex_ai_project"],
litellm_setting = "vertex_project",
env = "VERTEXAI_PROJECT",
)]
pub project: String,
#[layer(skip_blank,
litellm_param = ["vertex_location", "vertex_ai_location"],
litellm_setting = "vertex_location",
env = ["VERTEXAI_LOCATION", "VERTEX_LOCATION"],
)]
pub location: String,
}
/// The Vertex AI fields of Python's `GenericLiteLLMParams`, both the current names and the
/// `vertex_ai_*` spellings `VertexBase.safe_get_vertex_ai_*` still read.
@ -25,65 +47,20 @@ pub struct VertexParams {
}
impl VertexParams {
pub const CREDENTIALS: ParamSpec = ParamSpec {
setting: "credentials",
wire: &["vertex_credentials", "vertex_ai_credentials"],
module_global: None,
env: &["VERTEXAI_CREDENTIALS"],
};
pub const PROJECT: ParamSpec = ParamSpec {
setting: "project",
wire: &["vertex_project", "vertex_ai_project"],
module_global: Some("vertex_project"),
env: &["VERTEXAI_PROJECT"],
};
pub const LOCATION: ParamSpec = ParamSpec {
setting: "location",
wire: &["vertex_location", "vertex_ai_location"],
module_global: Some("vertex_location"),
env: &["VERTEXAI_LOCATION", "VERTEX_LOCATION"],
};
/// Every Vertex AI param Python reads, with both spellings, the module global
/// `VertexBase.get_vertex_ai_*` consults, and the environment names it falls back to.
pub const SPECS: [ParamSpec; 3] = [Self::CREDENTIALS, Self::PROJECT, Self::LOCATION];
/// The wire names a host projects out of a caller's kwargs, derived from [`Self::SPECS`].
pub fn fields() -> impl Iterator<Item = &'static str> {
Self::SPECS
.iter()
.flat_map(|spec| spec.wire.iter().copied())
}
/// The value under a wire name, read through serde so the names can never drift from
/// the struct.
pub fn get(&self, wire: &str) -> Option<String> {
serde_json::to_value(self)
.ok()?
.get(wire)?
.as_str()
.map(str::to_string)
}
/// The spec's value from these params, then the environment.
pub fn resolve(
&self,
spec: &ParamSpec,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Option<String> {
spec.resolve(&|name| self.get(name), env_lookup)
pub fn keys(&self) -> VertexKeysPartial {
VertexKeysPartial::from_litellm_params(&super::as_litellm_params(self))
}
pub fn credentials(&self) -> Option<String> {
self.resolve(&Self::CREDENTIALS, &|_| None)
self.keys().credentials
}
pub fn project(&self) -> Option<String> {
self.resolve(&Self::PROJECT, &|_| None)
self.keys().project
}
pub fn location(&self) -> Option<String> {
self.resolve(&Self::LOCATION, &|_| None)
self.keys().location
}
}

View file

@ -1,26 +1,27 @@
use litellm_auth_types::AwsParams;
use std::collections::BTreeSet;
use litellm_auth_types::{AwsKeys, AwsParams};
use rstest::rstest;
use serde_json::{Map, Value};
#[rstest]
fn fields_name_every_param_once_and_in_declaration_order() {
let filled: Map<String, Value> = AwsParams::fields()
fn fields_name_every_param_once() {
let filled: Map<String, Value> = AwsKeys::LITELLM_PARAMS
.iter()
.map(|name| (name.to_string(), Value::from(format!("value-of-{name}"))))
.collect();
let typed = AwsParams::from_optional_params(&filled);
let serialized = serde_json::to_value(&typed).unwrap();
assert_eq!(
serialized
.as_object()
.unwrap()
.keys()
.map(String::as_str)
.collect::<Vec<_>>(),
AwsParams::fields().collect::<Vec<_>>()
);
assert_eq!(serialized, Value::Object(filled.clone()));
assert_eq!(
serde_json::from_value::<AwsParams>(Value::Object(filled)).unwrap(),
typed
);
assert_eq!(
AwsKeys::LITELLM_PARAMS
.iter()
.collect::<BTreeSet<_>>()
.len(),
AwsKeys::LITELLM_PARAMS.len()
);
}

View file

@ -1,20 +1,16 @@
use litellm_auth_types::{AwsParams, ParamSpec};
use litellm_auth_types::{AwsKeys, Error, Sources, VertexKeys, VertexKeysPartial, VertexParams};
use litellm_layered::{Env, merge};
use rstest::rstest;
const LOCATION: ParamSpec = ParamSpec {
setting: "location",
wire: &["vertex_location", "vertex_ai_location"],
module_global: Some("vertex_location"),
env: &["VERTEXAI_LOCATION", "VERTEX_LOCATION"],
};
fn lookup(entries: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
move |name| {
fn vertex_params(entries: &[(&str, &str)]) -> VertexParams {
serde_json::from_value(
entries
.iter()
.find(|(key, _)| *key == name)
.map(|(_, value)| value.to_string())
}
.map(|(name, value)| (name.to_string(), serde_json::Value::from(*value)))
.collect::<serde_json::Map<_, _>>()
.into(),
)
.unwrap()
}
#[rstest]
@ -38,35 +34,32 @@ fn lookup(entries: &'static [(&'static str, &'static str)]) -> impl Fn(&str) ->
&[("VERTEX_LOCATION", "second"), ("VERTEXAI_LOCATION", "first")],
Some("first")
)]
#[case::values_are_kept_untrimmed(&[("vertex_location", " eu ")], &[], Some(" eu "))]
#[case::nothing_set(&[], &[], None)]
fn resolution_walks_wire_names_then_environment_names(
#[case] params: &'static [(&'static str, &'static str)],
#[case] env: &'static [(&'static str, &'static str)],
fn params_are_stacked_above_the_environment(
#[case] params: &[(&str, &str)],
#[case] env: &[(&str, &str)],
#[case] expected: Option<&str>,
) {
assert_eq!(
LOCATION.resolve(&lookup(params), &lookup(env)).as_deref(),
expected
);
let merged = merge([
vertex_params(params).keys(),
VertexKeysPartial::from_env(&Env::of(env)).unwrap(),
]);
assert_eq!(merged.location.as_deref(), expected);
}
#[rstest]
#[case::without_a_module_global(
&AwsParams::REGION,
#[case::without_a_litellm_setting(
AwsKeys::REGION,
"Missing AWS region - pass aws_region_name or set AWS_REGION_NAME or AWS_REGION"
)]
#[case::with_a_module_global(
&LOCATION,
#[case::with_a_litellm_setting(
VertexKeys::LOCATION,
"Missing AWS location - pass vertex_location or vertex_ai_location, set litellm.vertex_location, or set VERTEXAI_LOCATION or VERTEX_LOCATION"
)]
fn a_missing_param_names_every_place_the_spec_reads(
#[case] spec: &'static ParamSpec,
fn a_missing_param_names_every_source_the_group_reads(
#[case] sources: &'static Sources,
#[case] expected: &str,
) {
let message = spec.missing("AWS").to_string();
assert_eq!(message, expected);
assert!(spec.wire.iter().all(|name| message.contains(name)));
assert!(spec.env.iter().all(|name| message.contains(name)));
assert_eq!(message.contains("litellm."), spec.module_global.is_some());
assert_eq!(Error::missing("AWS", sources).to_string(), expected);
}

View file

@ -1,6 +1,7 @@
use std::collections::BTreeSet;
use litellm_auth_types::VertexParams;
use litellm_auth_types::{VertexKeys, VertexKeysPartial, VertexParams};
use litellm_layered::{Env, merge};
use rstest::rstest;
use serde_json::{Map, Value, json};
@ -61,8 +62,9 @@ fn a_param_of_the_wrong_type_is_rejected(#[case] params: Value) {
#[rstest]
fn fields_name_every_param_once() {
let filled: Value = VertexParams::fields()
.map(|name| (name.to_string(), Value::from(name)))
let filled: Value = VertexKeys::LITELLM_PARAMS
.iter()
.map(|name| (name.to_string(), Value::from(*name)))
.collect::<Map<_, _>>()
.into();
let params: VertexParams = serde_json::from_value(filled).unwrap();
@ -76,8 +78,11 @@ fn fields_name_every_param_once() {
};
assert_eq!(params, expected);
assert_eq!(
VertexParams::fields().collect::<BTreeSet<_>>().len(),
VertexParams::fields().count()
VertexKeys::LITELLM_PARAMS
.iter()
.collect::<BTreeSet<_>>()
.len(),
VertexKeys::LITELLM_PARAMS.len()
);
}
@ -101,7 +106,7 @@ fn debug_output_does_not_expose_credentials() {
#[case::legacy_spelling(None, Some("legacy"), &[("VERTEXAI_PROJECT", "env")], Some("legacy"))]
#[case::blank_params_fall_to_the_environment(Some(" "), None, &[("VERTEXAI_PROJECT", "env")], Some("env"))]
#[case::nothing(None, None, &[], None)]
fn a_spec_resolves_from_the_params_then_the_environment(
fn a_setting_resolves_from_the_params_then_the_environment(
#[case] vertex_project: Option<&str>,
#[case] vertex_ai_project: Option<&str>,
#[case] environment: &[(&str, &str)],
@ -112,15 +117,10 @@ fn a_spec_resolves_from_the_params_then_the_environment(
vertex_ai_project: vertex_ai_project.map(str::to_string),
..VertexParams::default()
};
let env = |name: &str| {
environment
.iter()
.find(|(key, _)| *key == name)
.map(|(_, value)| value.to_string())
};
let merged = merge([
params.keys(),
VertexKeysPartial::from_env(&Env::of(environment)).unwrap(),
]);
assert_eq!(
params.resolve(&VertexParams::PROJECT, &env).as_deref(),
expected
);
assert_eq!(merged.project.as_deref(), expected);
}

View file

@ -216,6 +216,8 @@ pub struct LiteLlmSettings {
pub aiohttp_trust_env: Option<bool>,
pub disable_aiohttp_trust_env: Option<bool>,
pub disable_aiohttp_transport: Option<bool>,
pub vertex_project: Option<String>,
pub vertex_location: Option<String>,
pub drop_params: Option<Spelled<bool>>,
pub request_timeout: Option<Spelled<f64>>,
pub num_retries: Option<u64>,

View file

@ -2,6 +2,7 @@ mod auth;
mod error;
mod http;
mod secrets;
mod vertex;
pub use error::Error;
use std::{sync::Arc, time::Instant};
@ -50,7 +51,10 @@ pub fn build_inference(config: &Config, process: &Env) -> Result<Arc<Gateway>, E
ocr,
env,
secrets,
ModelRouter::from_model_list(&config.model_list),
ModelRouter::from_model_list(&vertex::with_written_settings(
&config.model_list,
&config.litellm_settings,
)),
)?))
}

View file

@ -0,0 +1,89 @@
use litellm_auth_types::{VertexKeysPartial, VertexParams};
use litellm_config::{LiteLlmSettings, LitellmParams, Model};
use litellm_layered::merge;
pub(crate) fn with_written_settings(models: &[Model], settings: &LiteLlmSettings) -> Vec<Model> {
let file = written(settings);
models
.iter()
.map(|model| {
let keys = merge([model.litellm_params.vertex.keys(), file.clone()]);
Model {
litellm_params: LitellmParams {
vertex: VertexParams {
vertex_project: keys.project,
vertex_location: keys.location,
..model.litellm_params.vertex.clone()
},
..model.litellm_params.clone()
},
..model.clone()
}
})
.collect()
}
fn written(settings: &LiteLlmSettings) -> VertexKeysPartial {
VertexKeysPartial {
credentials: None,
project: settings.vertex_project.clone(),
location: settings.vertex_location.clone(),
}
}
#[cfg(test)]
mod tests {
use litellm_auth_types::VertexKeys;
use litellm_config::Config;
use litellm_http::HttpKeys;
use rstest::rstest;
use serde_json::{Map, Value};
use super::*;
fn models(yaml: &str) -> Vec<Model> {
let config = Config::from_yaml(yaml).unwrap();
with_written_settings(&config.model_list, &config.litellm_settings)
}
#[rstest]
#[case::file_fills_an_unset_deployment(
"model_list:\n - model_name: m\n litellm_params:\n model: vertex_ai/claude\nlitellm_settings:\n vertex_project: file-project\n",
Some("file-project")
)]
#[case::the_deployment_wins(
"model_list:\n - model_name: m\n litellm_params:\n model: vertex_ai/claude\n vertex_ai_project: deployment\nlitellm_settings:\n vertex_project: file-project\n",
Some("deployment")
)]
#[case::a_blank_deployment_value_takes_the_file(
"model_list:\n - model_name: m\n litellm_params:\n model: vertex_ai/claude\n vertex_project: ' '\nlitellm_settings:\n vertex_project: file-project\n",
Some("file-project")
)]
#[case::neither(
"model_list:\n - model_name: m\n litellm_params:\n model: vertex_ai/claude\n",
None
)]
fn deployment_params_are_stacked_above_the_config_file(
#[case] yaml: &str,
#[case] expected: Option<&str>,
) {
assert_eq!(
models(yaml)[0].litellm_params.vertex.project().as_deref(),
expected
);
}
#[rstest]
fn every_litellm_setting_a_group_declares_is_a_typed_config_field() {
let names = HttpKeys::LITELLM_SETTINGS
.iter()
.chain(VertexKeys::LITELLM_SETTINGS)
.map(|(name, _)| ((*name).to_owned(), Value::Null))
.collect::<Map<_, _>>();
let settings: LiteLlmSettings = serde_json::from_value(Value::Object(names)).unwrap();
assert_eq!(
settings.additional_fields.keys().collect::<Vec<_>>(),
Vec::<&String>::new()
);
}
}

View file

@ -233,8 +233,19 @@ environment_variables:
}
#[rstest]
#[case::project_on_the_deployment(
" vertex_project: test-project\n vertex_location: us-east5\n",
""
)]
#[case::project_only_in_litellm_settings(
"",
"litellm_settings:\n vertex_project: test-project\n vertex_location: us-east5\n"
)]
#[tokio::test]
async fn messages_auth_reads_credentials_from_config_environment_variables() {
async fn messages_auth_reads_credentials_from_config_environment_variables(
#[case] deployment: &str,
#[case] litellm_settings: &str,
) {
let directory = tempfile::tempdir().unwrap();
let credentials = directory.path().join("service-account.json");
std::fs::write(&credentials, "not a service account").unwrap();
@ -243,13 +254,11 @@ async fn messages_auth_reads_credentials_from_config_environment_variables() {
- model_name: public/vertex
litellm_params:
model: vertex_ai/claude-sonnet-4-5
vertex_project: test-project
vertex_location: us-east5
general_settings:
{deployment}general_settings:
master_key: gateway-key
environment_variables:
GOOGLE_APPLICATION_CREDENTIALS: '{}'
",
{litellm_settings}",
credentials.display()
))
.unwrap();

View file

@ -41,7 +41,7 @@ impl BaseOcrConfig for TextractAnalyzeDocumentConfig {
type Environment = TextractEnvironment;
fn secret_names(&self) -> Vec<&'static str> {
litellm_auth::AwsParams::secret_names().to_vec()
litellm_auth::AwsKeys::ENV_NAMES.to_vec()
}
fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] {

View file

@ -239,8 +239,12 @@ pub(super) async fn environment(
) -> Result<TextractEnvironment, Error> {
let env_lookup = |name: &str| request.connection.secret(name);
let params = AwsParams::from_optional_params(&request.optional_params);
let region = resolve_aws_region(None, &params, &env_lookup)
.ok_or_else(|| Error::Auth(AwsParams::REGION.missing("AWS")))?;
let region = resolve_aws_region(None, &params, &env_lookup).ok_or_else(|| {
Error::Auth(litellm_auth::Error::missing(
"AWS",
litellm_auth::AwsKeys::REGION,
))
})?;
let signer = SigV4Signer::resolve(
auth,
region.clone(),

View file

@ -30,7 +30,7 @@ impl BaseOcrConfig for TextractDetectTextConfig {
type Environment = TextractEnvironment;
fn secret_names(&self) -> Vec<&'static str> {
litellm_auth::AwsParams::secret_names().to_vec()
litellm_auth::AwsKeys::ENV_NAMES.to_vec()
}
fn get_health_check_document(&self) -> OcrDocument {

View file

@ -88,7 +88,7 @@ fn optional_string<'a>(params: &'a Map<String, Value>, key: &str) -> Option<&'a
impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
fn secret_names(&self) -> Vec<&'static str> {
litellm_auth::AwsParams::secret_names().to_vec()
litellm_auth::AwsKeys::ENV_NAMES.to_vec()
}
fn get_supported_openai_params(&self) -> &'static [&'static str] {

View file

@ -94,7 +94,7 @@ pub const BEDROCK_CHAT_COMPLETIONS_CONFIG: AmazonConverseConfig = AmazonConverse
impl BaseConfig for AmazonConverseConfig {
fn secret_names(&self) -> Vec<&'static str> {
litellm_auth::AwsParams::secret_names()
litellm_auth::AwsKeys::ENV_NAMES
.iter()
.copied()
.chain([AWS_BEARER_TOKEN_BEDROCK])

View file

@ -1,9 +1,10 @@
use std::sync::LazyLock;
use litellm_auth::{AwsParams, CredentialPlacement, ParamSpec, SecretValue};
use litellm_auth::{CredentialPlacement, SecretValue};
use litellm_auth_aws::{AwsCredentialSource, resolve_aws_region};
use litellm_core_utils::settings::resolve_non_empty;
use litellm_http::request::{header_value, with_header, without_headers};
use litellm_layered::Layered;
use litellm_llms_types::{
formats::messages::{MessagesOptionalParams, MessagesRequest},
providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
@ -26,20 +27,26 @@ use crate::{
const SERVICE: &str = "aws-external-anthropic";
const BASE_ENVS: &[&str] = &["ANTHROPIC_AWS_BASE_URL", "ANTHROPIC_AWS_API_BASE"];
const WORKSPACE_ENVS: &[&str] = &["ANTHROPIC_AWS_WORKSPACE_ID", "ANTHROPIC_WORKSPACE_ID"];
const WORKSPACE_FIELDS: &[&str] = &[
"workspace_id",
"aws_workspace_id",
"anthropic-workspace-id",
"anthropic_workspace_id",
];
const WORKSPACE_ENVS: &[&str] = WorkspaceKeys::ENV_NAMES;
const WORKSPACE_FIELDS: &[&str] = WorkspaceKeys::LITELLM_PARAMS;
const WORKSPACE_HEADER: &str = "anthropic-workspace-id";
static WORKSPACE: ParamSpec = ParamSpec {
setting: "workspace_id",
wire: WORKSPACE_FIELDS,
module_global: None,
env: WORKSPACE_ENVS,
};
#[derive(Layered)]
#[expect(dead_code, reason = "only WorkspaceKeysPartial is built")]
struct WorkspaceKeys {
#[layer(
skip_blank,
litellm_param = [
"workspace_id",
"aws_workspace_id",
"anthropic-workspace-id",
"anthropic_workspace_id",
],
env = ["ANTHROPIC_AWS_WORKSPACE_ID", "ANTHROPIC_WORKSPACE_ID"],
label = "workspace_id"
)]
workspace_id: String,
}
const UNSUPPORTED_OVERRIDE: &str = "claude_platform_unsupported_params";
pub struct BedrockClaudePlatformMessagesConfig;
@ -72,8 +79,12 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
.or_else(|| workspace_value(params.extra.get(*field)))
})
});
let workspace = resolve_non_empty(workspace, env, WORKSPACE_ENVS)
.ok_or_else(|| Error::Auth(WORKSPACE.missing("Claude Platform on AWS")))?;
let workspace = resolve_non_empty(workspace, env, WORKSPACE_ENVS).ok_or_else(|| {
Error::Auth(litellm_auth::Error::missing(
"Claude Platform on AWS",
WorkspaceKeys::WORKSPACE_ID,
))
})?;
let headers = with_header(headers, WORKSPACE_HEADER, workspace);
let key = resolve_non_empty(api_key, env, &["ANTHROPIC_AWS_API_KEY"]);
let headers = without_headers(headers, &["authorization"]);
@ -150,7 +161,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
.chain(BASE_ENVS.iter().copied())
.chain(WORKSPACE_ENVS.iter().copied())
.chain(["AWS_DEFAULT_REGION"])
.chain(AwsParams::secret_names().iter().copied())
.chain(litellm_auth::AwsKeys::ENV_NAMES.iter().copied())
.collect()
});
&NAMES
@ -199,7 +210,12 @@ fn platform_region(
let region = host_region
.or_else(|| resolve_aws_region(None, &params.aws, env))
.or_else(|| resolve_non_empty(None, env, &["AWS_DEFAULT_REGION"]))
.ok_or_else(|| Error::Auth(AwsParams::REGION.missing("Claude Platform on AWS")))?;
.ok_or_else(|| {
Error::Auth(litellm_auth::Error::missing(
"Claude Platform on AWS",
litellm_auth::AwsKeys::REGION,
))
})?;
validated_region(region)
}
@ -250,6 +266,7 @@ fn invalid_request(error: serde_json::Error) -> Error {
#[cfg(test)]
mod tests {
use super::*;
use litellm_auth::AwsParams;
use litellm_llms_types::providers::anthropic::AnthropicBeta;
use rstest::rstest;
use serde_json::json;
@ -267,7 +284,10 @@ mod tests {
&|_| None
)
.unwrap_err(),
Error::Auth(WORKSPACE.missing("Claude Platform on AWS"))
Error::Auth(litellm_auth::Error::missing(
"Claude Platform on AWS",
WorkspaceKeys::WORKSPACE_ID,
))
);
}

View file

@ -1,6 +1,6 @@
use std::sync::LazyLock;
use litellm_auth::{AwsParams, CredentialPlacement, SecretValue};
use litellm_auth::{CredentialPlacement, SecretValue};
use litellm_auth_aws::AwsCredentialSource;
use litellm_auth_aws::bedrock_model_id_and_region;
use litellm_core_utils::settings::resolve_non_empty;
@ -164,7 +164,7 @@ impl BaseMessagesConfig for BedrockMantleAnthropicMessagesConfig {
.into_iter()
.chain(REGION_ENVS.iter().copied())
.chain(
AwsParams::secret_names()
litellm_auth::AwsKeys::ENV_NAMES
.iter()
.copied()
.filter(|name| !REGION_ENVS.contains(name)),
@ -263,6 +263,7 @@ fn mantle_region(
#[cfg(test)]
mod tests {
use super::*;
use litellm_auth::AwsParams;
use litellm_http::request::header_value;
use rstest::rstest;
use serde_json::json;

View file

@ -1,4 +1,3 @@
use litellm_auth::VertexParams;
use litellm_auth_gcp::{
VertexConfig, get_vertex_ai_location, get_vertex_ai_project,
get_vertex_ai_project_from_credentials, secret_names,
@ -64,7 +63,12 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
let config = VertexConfig::from_params(&litellm_params.vertex);
let project = get_vertex_ai_project(&config, env_lookup)
.or_else(|| get_vertex_ai_project_from_credentials(&config, env_lookup))
.ok_or_else(|| Error::Auth(VertexParams::PROJECT.missing("Vertex AI")))?;
.ok_or_else(|| {
Error::Auth(litellm_auth::Error::missing(
"Vertex AI",
litellm_auth::VertexKeys::PROJECT,
))
})?;
let location = get_vertex_ai_location(&config, env_lookup)
.unwrap_or_else(|| DEFAULT_LOCATION.to_string());
complete_vertex_anthropic_url(api_base, &project, &location, model, stream)
@ -227,6 +231,7 @@ fn vertex_feature_betas(request: &MessagesRequest) -> BetaSet {
#[cfg(test)]
mod tests {
use litellm_auth::VertexParams;
use rstest::rstest;
use serde_json::json;
@ -422,7 +427,7 @@ mod tests {
error,
Error::Auth(litellm_auth::Error::MissingParam {
provider: "Vertex AI",
spec: &VertexParams::PROJECT,
spec: litellm_auth::VertexKeys::PROJECT,
})
));
assert_eq!(

View file

@ -0,0 +1,74 @@
use litellm_auth::Sources;
use pyo3::prelude::*;
use serde_json::{Map, Value};
use crate::{
coercion::FieldSpec,
python_settings::{PythonSettings, Snapshot},
};
/// The settings among `sources` that have a `litellm.*` global and that `params` leaves unset,
/// counting a blank value as unset the way Python's `or` chain does.
pub(crate) fn unset<'s>(
params: &Map<String, Value>,
sources: impl IntoIterator<Item = &'s Sources>,
) -> Vec<&'s Sources> {
sources
.into_iter()
.filter(|sources| sources.litellm_setting.is_some())
.filter(|sources| {
!sources
.litellm_params
.iter()
.any(|name| params.get(*name).is_some_and(is_set))
})
.collect()
}
fn is_set(value: &Value) -> bool {
match value {
Value::Null => false,
Value::String(text) => !text.trim().is_empty(),
_ => true,
}
}
/// Reads the globals of `unset` through the `ProviderDefaults` accessor and places each truthy
/// one under its setting's first param name, replacing a blank value the call passed.
pub(crate) fn fold(
py: Python<'_>,
params: Map<String, Value>,
unset: &[&Sources],
) -> PyResult<Map<String, Value>> {
if unset.is_empty() {
return Ok(params);
}
let Some(defaults) = PythonSettings::ProviderDefaults.read_or_unset(py)? else {
return Ok(params);
};
fold_from(params, unset, &defaults)
}
pub(crate) fn fold_from(
params: Map<String, Value>,
unset: &[&Sources],
defaults: &Snapshot<'_>,
) -> PyResult<Map<String, Value>> {
let globals = unset
.iter()
.filter_map(|sources| {
let setting = sources.litellm_setting?;
let name = sources.litellm_params.first()?;
let spec = FieldSpec::new(setting, |field| field.falsy_optional_string());
Some(
defaults
.read(&spec)
.map(|value| value.map(|value| (name.to_string(), Value::String(value)))),
)
})
.collect::<Result<Vec<_>, _>>()?;
Ok(params
.into_iter()
.chain(globals.into_iter().flatten())
.collect())
}

View file

@ -2,6 +2,7 @@ use crate::cache::{CacheCall, Cached, PythonCache, Selection};
use litellm_host_python::{PythonHostCalls, PythonOwned};
use bytes::Bytes;
use litellm_auth::Sources;
use litellm_host_python::{InvokeError, PythonBinding, from_py, to_py};
use litellm_http::transport::Error as TransportError;
use litellm_inference_messages::{
@ -22,8 +23,7 @@ use serde_json::{Map, Value};
use crate::{
errors::{RustUpstreamError, route_error_to_pyerr},
marshal::{optional_timeout, project_optional_fields, public_response, python_timeout_seconds},
python_settings::missing_module,
routes::RequestView,
routes::{RequestView, credential_globals},
};
const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.messages";
@ -41,31 +41,16 @@ fn merge_headers(
(!merged.is_empty()).then_some(merged)
}
/// Reads `litellm.<name>`, the module global a param spec names, the way
/// `VertexBase.get_vertex_ai_project` reads `litellm.vertex_project`.
fn module_global<'py>(py: Python<'py>, name: &str) -> PyResult<Option<Bound<'py, PyAny>>> {
let module = match py.import("litellm") {
Ok(module) => module,
Err(error) if missing_module(py, &error, "litellm")? => return Ok(None),
Err(error) => return Err(error),
};
let value = module.getattr(name)?;
Ok((!value.is_none()).then_some(value))
}
/// The caller's litellm params, read from the kwargs by the names the typed params declare,
/// so a key the configs do not read is never converted; a spec whose spellings the call all
/// leaves out falls back to the module global it names, which stays the host's concern.
/// so a key the configs do not read is never converted. A setting the call leaves unset or
/// blank takes its `litellm.*` global, as Python's `or` chain does.
fn project_litellm_params<'py>(
argument: impl Fn(&str) -> PyResult<Option<Bound<'py, PyAny>>>,
global: impl Fn(&str) -> PyResult<Option<Bound<'py, PyAny>>>,
fold: impl Fn(Map<String, Value>, &[&Sources]) -> PyResult<Map<String, Value>>,
) -> PyResult<Result<LitellmParams, Error>> {
let fields = project_optional_fields(LitellmParams::fields(), &argument)?;
let globals = LitellmParams::specs()
.filter(|spec| !spec.wire.iter().any(|name| fields.contains_key(*name)))
.filter_map(|spec| spec.module_global);
let folded = project_optional_fields(globals, &global)?;
Ok(litellm_params(fields.into_iter().chain(folded).collect()))
let unset = credential_globals::unset(&fields, LitellmParams::litellm_settings());
Ok(litellm_params(fold(fields, &unset)?))
}
fn native_error(py: Python<'_>, error: Error) -> PyResult<PyErr> {
@ -136,7 +121,9 @@ impl MessagesPythonHost {
let api_base = string("api_base")?;
let extra_headers = self.merged_headers(py, arguments)?;
let provider_specific_header = self.provider_specific_header(py, arguments)?;
let litellm_params = project_litellm_params(argument, |name| module_global(py, name))?;
let litellm_params = project_litellm_params(argument, |fields, unset| {
credential_globals::fold(py, fields, unset)
})?;
Ok(messages_body(body).and_then(|body| {
Ok(MessagesCall {
body,
@ -336,6 +323,7 @@ mod tests {
use serde_json::json;
use super::*;
use crate::python_settings::PythonSettings;
fn map(value: Value) -> Map<String, Value> {
serde_json::from_value(value).unwrap()
@ -404,7 +392,12 @@ mod tests {
json!({"vertex_project": "from-global"}),
Ok(json!({"model": "m", "vertex_project": "from-global"})),
)]
#[case::a_global_of_the_wrong_type_is_a_request_error(
#[case::a_blank_value_in_the_call_falls_back(
json!({"model": "m", "vertex_project": "", "vertex_ai_project": " "}),
json!({"vertex_project": "from-global"}),
Ok(json!({"model": "m", "vertex_project": "from-global", "vertex_ai_project": " "})),
)]
#[case::a_global_of_the_wrong_type_is_an_error(
json!({"model": "m"}),
json!({"vertex_location": 5}),
Err(()),
@ -447,19 +440,27 @@ mod tests {
.unwrap()
};
let kwargs = dict(&kwargs);
let globals = dict(&globals);
let defaults = dict(&json!({"vertex_project": null, "vertex_location": null}));
defaults.update(dict(&globals).as_mapping()).unwrap();
let namespace = py
.import("types")
.unwrap()
.getattr("SimpleNamespace")
.unwrap()
.call((), Some(&defaults))
.unwrap();
let snapshot = PythonSettings::ProviderDefaults.snapshot(namespace);
let bound = PyDict::new(py);
let projected = project_litellm_params(
|name| present(&kwargs, &bound, name),
|name| present(&globals, &bound, name),
)
.unwrap();
|fields, unset| credential_globals::fold_from(fields, unset, &snapshot),
);
match (projected, expected) {
(Ok(projected), Ok(fields)) => assert_eq!(
(Ok(Ok(projected)), Ok(fields)) => assert_eq!(
projected,
litellm_params(serde_json::from_value(fields).unwrap()).unwrap()
),
(Err(error), Err(())) => assert!(matches!(error, Error::InvalidRequest(_))),
(Err(_) | Ok(Err(_)), Err(())) => {}
(projected, expected) => panic!("got {projected:?}, expected {expected:?}"),
}
});

View file

@ -1,6 +1,7 @@
pub(crate) mod audio_transcription;
pub(crate) mod chat_completions;
pub(crate) mod clickhouse_spend;
mod credential_globals;
pub(crate) mod embeddings;
mod inference;
pub(crate) mod messages;

View file

@ -1,5 +1,5 @@
use litellm_auth::SecretValue;
use litellm_auth::VertexParams;
use litellm_auth::{Sources, VertexKeys};
use litellm_host_python::{from_py, present};
use litellm_inference_ocr::{
types::{LiteLLMOcrRequest, OcrDocumentInput},
@ -11,10 +11,9 @@ use serde_json::{Map, Value};
use super::{document::FileDocumentInput, errors::to_pyerr as ocr_error_to_pyerr};
use crate::{
coercion::FieldSpec,
credentials::{self, CallerTokenProvider},
marshal::{project_optional_fields, python_timeout_seconds, request_input_sources},
python_settings::{PythonSettings, Snapshot},
routes::credential_globals,
};
/// What the host keeps after projection: the caller's token callable that answers the
@ -126,16 +125,9 @@ pub(super) fn project_request(
let names = specs.iter().map(|spec| spec.name).collect::<Vec<_>>();
let optional_params =
project_optional_fields(names.iter().copied(), |name| present(kwargs, bound, name))?;
let globals = module_globals_to_read(&optional_params, &names);
let defaults = if globals.is_empty() {
None
} else {
PythonSettings::ProviderDefaults.read_or_unset(bound.py())?
};
let optional_params = match defaults {
Some(defaults) => fold_module_globals(bound.py(), optional_params, &globals, &defaults)?,
None => optional_params,
};
let unset = unset_vertex_settings(&optional_params, &names);
let optional_params = credential_globals::fold(bound.py(), optional_params, &unset)
.map_err(|error| settings_error(bound.py(), error))?;
let input_sources = request_input_sources(
kwargs,
names
@ -169,48 +161,21 @@ pub(super) fn project_request(
))
}
/// Python's `VertexBase.get_vertex_ai_project` and `get_vertex_ai_location` fall back to a
/// `litellm.<name>` module global when a call names neither spelling: the globals of the
/// specs this route consumes whose wire names the call left out.
fn module_globals_to_read(
/// The Vertex settings this route consumes that the call leaves unset, whose `litellm.*`
/// globals `VertexBase.get_vertex_ai_project` and `get_vertex_ai_location` fall back to.
fn unset_vertex_settings(
optional_params: &Map<String, Value>,
consumed: &[&str],
) -> Vec<&'static str> {
VertexParams::SPECS
.iter()
.filter(|spec| spec.wire.iter().all(|name| consumed.contains(name)))
.filter(|spec| {
!spec
.wire
) -> Vec<&'static Sources> {
credential_globals::unset(
optional_params,
VertexKeys::SOURCES.iter().filter(|sources| {
sources
.litellm_params
.iter()
.any(|name| optional_params.contains_key(*name))
})
.filter_map(|spec| spec.module_global)
.collect()
}
/// Folds each module global in under the name its spec declares, skipping falsy values the
/// way Python's `or` chain does.
fn fold_module_globals(
py: Python<'_>,
optional_params: Map<String, Value>,
globals: &[&'static str],
defaults: &Snapshot<'_>,
) -> PyResult<Map<String, Value>> {
let read = globals
.iter()
.map(|name| {
let spec = FieldSpec::new(name, |field| field.falsy_optional_string());
Ok(defaults
.read(&spec)
.map_err(|error| settings_error(py, error.into()))?
.map(|value| (name.to_string(), Value::String(value))))
})
.collect::<PyResult<Vec<_>>>()?;
Ok(optional_params
.into_iter()
.chain(read.into_iter().flatten())
.collect())
.all(|name| consumed.contains(name))
}),
)
}
/// A misconfigured litellm setting is the operator's error, not the request's or the
@ -228,6 +193,7 @@ mod tests {
use pyo3::exceptions::PyValueError;
use super::*;
use crate::python_settings::PythonSettings;
#[rstest::rstest]
#[case::global_fills_a_missing_project(
@ -263,11 +229,10 @@ mod tests {
)
.unwrap();
let snapshot = PythonSettings::ProviderDefaults.snapshot(namespace);
let consumed = VertexParams::fields().collect::<Vec<_>>();
let params = params.as_object().cloned().unwrap();
let globals = module_globals_to_read(&params, &consumed);
let unset = unset_vertex_settings(&params, VertexKeys::LITELLM_PARAMS);
let folded = fold_module_globals(py, params, &globals, &snapshot).unwrap();
let folded = credential_globals::fold_from(params, &unset, &snapshot).unwrap();
assert_eq!(Value::Object(folded), expected);
});
@ -286,8 +251,11 @@ mod tests {
.unwrap();
let snapshot = PythonSettings::ProviderDefaults.snapshot(namespace);
let error =
fold_module_globals(py, Map::new(), &["vertex_project"], &snapshot).unwrap_err();
let error = settings_error(
py,
credential_globals::fold_from(Map::new(), &[VertexKeys::PROJECT], &snapshot)
.unwrap_err(),
);
assert!(error.is_instance_of::<PyValueError>(py));
assert!(
@ -323,7 +291,10 @@ mod tests {
#[case] expected: &[&str],
) {
assert_eq!(
module_globals_to_read(params.as_object().unwrap(), consumed),
unset_vertex_settings(params.as_object().unwrap(), consumed)
.iter()
.filter_map(|sources| sources.litellm_setting)
.collect::<Vec<_>>(),
expected
);
}

View file

@ -1,6 +1,6 @@
use std::{collections::BTreeMap, fmt, ops::Deref};
use litellm_auth_types::{AwsParams, ParamSpec, SecretValue, VertexParams};
use litellm_auth_types::{AwsKeys, AwsParams, SecretValue, VertexKeys, VertexParams};
use serde::Deserialize;
use serde_json::Value;
use serde_with::{OneOrMany, formats::PreferMany, serde_as};
@ -54,10 +54,13 @@ pub struct LitellmParams {
}
impl LitellmParams {
/// Every connection param spec the credential families declare, for hosts that fold a
/// spec's module global in when a call names none of its spellings.
pub fn specs() -> impl Iterator<Item = &'static ParamSpec> {
AwsParams::SPECS.iter().chain(VertexParams::SPECS.iter())
/// Every credential setting that has a `litellm.*` global, for hosts that fold the global
/// in when a call leaves the setting unset.
pub fn litellm_settings() -> impl Iterator<Item = &'static litellm_auth_types::Sources> {
AwsKeys::SOURCES
.iter()
.chain(VertexKeys::SOURCES.iter())
.filter(|sources| sources.litellm_setting.is_some())
}
/// The wire names a host projects out of a caller's kwargs: the model, the credentials
@ -66,8 +69,8 @@ impl LitellmParams {
pub fn fields() -> impl Iterator<Item = &'static str> {
["model", "api_key", "api_base", "api_version"]
.into_iter()
.chain(AwsParams::fields())
.chain(VertexParams::fields())
.chain(AwsKeys::LITELLM_PARAMS.iter().copied())
.chain(VertexKeys::LITELLM_PARAMS.iter().copied())
}
}