diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 84415820d89..df448508871 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3366,6 +3366,7 @@ dependencies = [ "aws-types", "litellm-auth-types", "litellm-http", + "litellm-layered", "moka", "reqwest 0.12.28", "rstest", @@ -3404,6 +3405,7 @@ dependencies = [ "google-cloud-auth", "http 1.4.2", "litellm-auth-types", + "litellm-layered", "moka", "rstest", "serde", @@ -3418,6 +3420,7 @@ dependencies = [ name = "litellm-auth-types" version = "0.1.0" dependencies = [ + "litellm-layered", "rstest", "serde", "serde_json", diff --git a/litellm-rust/crates/auth-aws/Cargo.toml b/litellm-rust/crates/auth-aws/Cargo.toml index 014f9341d81..03c26e49770 100644 --- a/litellm-rust/crates/auth-aws/Cargo.toml +++ b/litellm-rust/crates/auth-aws/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-layered.workspace = true litellm-auth-types.workspace = true litellm-http.workspace = true diff --git a/litellm-rust/crates/auth-aws/src/aws.rs b/litellm-rust/crates/auth-aws/src/aws.rs index 34199cec5fa..5dd77b6c5b6 100644 --- a/litellm-rust/crates/auth-aws/src/aws.rs +++ b/litellm-rust/crates/auth-aws/src/aws.rs @@ -13,15 +13,14 @@ use aws_sigv4::http_request::{ use aws_sigv4::sign::v4; use aws_smithy_runtime_api::client::identity::Identity; -use litellm_auth_types::AwsParams; +use litellm_auth_types::{AwsKeysPartial, AwsParams}; +use litellm_layered::merge; use super::Error; use super::constants::{ - AWS_ACCESS_KEY_ID, AWS_EXTERNAL_ID, AWS_PROFILE, AWS_PROFILE_NAME, AWS_REGION_NAME, - AWS_ROLE_ARN, AWS_ROLE_NAME, AWS_ROLE_SESSION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_NAME, - AWS_SESSION_TOKEN, AWS_SIGNED_HEADER_NAMES, AWS_STS_ENDPOINT, AWS_WEB_IDENTITY_TOKEN, - AWS_WEB_IDENTITY_TOKEN_FILE, DEFAULT_BEDROCK_REGION, DEFAULT_SESSION_NAME_PREFIX, - SIGV4_COMPUTED_HEADER_NAMES, + AWS_ACCESS_KEY_ID, AWS_PROFILE, AWS_ROLE_ARN, AWS_ROLE_SESSION_NAME, AWS_SECRET_ACCESS_KEY, + AWS_SESSION_TOKEN, AWS_SIGNED_HEADER_NAMES, AWS_WEB_IDENTITY_TOKEN_FILE, + DEFAULT_BEDROCK_REGION, DEFAULT_SESSION_NAME_PREFIX, SIGV4_COMPUTED_HEADER_NAMES, }; const STATIC_CREDENTIALS_TTL: Duration = Duration::from_secs(3600 - 60); @@ -75,21 +74,39 @@ pub struct AwsAuthConfig { impl AwsAuthConfig { fn with_environment(self, env_lookup: &(dyn Fn(&str) -> Option + Sync)) -> Self { + Self::from(merge([self.keys(), env_keys(env_lookup)])) + } + + fn keys(self) -> AwsKeysPartial { + AwsKeysPartial { + access_key_id: self.access_key_id, + secret_access_key: self.secret_access_key, + session_token: self.session_token, + region: self.region_name, + session_name: self.session_name, + profile_name: self.profile_name, + role_name: self.role_name, + web_identity_token: self.web_identity_token, + sts_endpoint: self.sts_endpoint, + external_id: self.external_id, + bedrock_runtime_endpoint: None, + } + } +} + +impl From for AwsAuthConfig { + fn from(keys: AwsKeysPartial) -> Self { Self { - access_key_id: self.access_key_id.or_else(|| env_lookup(AWS_ACCESS_KEY_ID)), - secret_access_key: self - .secret_access_key - .or_else(|| env_lookup(AWS_SECRET_ACCESS_KEY)), - session_token: self.session_token.or_else(|| env_lookup(AWS_SESSION_TOKEN)), - region_name: self.region_name.or_else(|| env_lookup(AWS_REGION_NAME)), - session_name: self.session_name.or_else(|| env_lookup(AWS_SESSION_NAME)), - profile_name: self.profile_name.or_else(|| env_lookup(AWS_PROFILE_NAME)), - role_name: self.role_name.or_else(|| env_lookup(AWS_ROLE_NAME)), - web_identity_token: self - .web_identity_token - .or_else(|| env_lookup(AWS_WEB_IDENTITY_TOKEN)), - sts_endpoint: self.sts_endpoint.or_else(|| env_lookup(AWS_STS_ENDPOINT)), - external_id: self.external_id.or_else(|| env_lookup(AWS_EXTERNAL_ID)), + access_key_id: keys.access_key_id, + secret_access_key: keys.secret_access_key, + session_token: keys.session_token, + region_name: keys.region, + session_name: keys.session_name, + profile_name: keys.profile_name, + role_name: keys.role_name, + web_identity_token: keys.web_identity_token, + sts_endpoint: keys.sts_endpoint, + external_id: keys.external_id, } } } @@ -612,9 +629,10 @@ pub fn resolve_aws_region( env_lookup: &dyn Fn(&str) -> Option, ) -> Option { params - .resolve(&AwsParams::REGION, &|_| None) + .keys() + .region .or_else(|| model_region.map(str::to_string)) - .or_else(|| AwsParams::REGION.resolve(&|_| None, env_lookup)) + .or_else(|| env_keys(env_lookup).region) } pub fn resolve_bedrock_region( @@ -630,18 +648,11 @@ pub fn aws_auth_config( params: &AwsParams, env_lookup: &dyn Fn(&str) -> Option, ) -> AwsAuthConfig { - AwsAuthConfig { - access_key_id: params.resolve(&AwsParams::ACCESS_KEY_ID, env_lookup), - secret_access_key: params.resolve(&AwsParams::SECRET_ACCESS_KEY, env_lookup), - session_token: params.resolve(&AwsParams::SESSION_TOKEN, env_lookup), - region_name: params.resolve(&AwsParams::REGION, env_lookup), - session_name: params.resolve(&AwsParams::SESSION_NAME, env_lookup), - profile_name: params.resolve(&AwsParams::PROFILE_NAME, env_lookup), - role_name: params.resolve(&AwsParams::ROLE_NAME, env_lookup), - web_identity_token: params.resolve(&AwsParams::WEB_IDENTITY_TOKEN, env_lookup), - sts_endpoint: params.resolve(&AwsParams::STS_ENDPOINT, env_lookup), - external_id: params.resolve(&AwsParams::EXTERNAL_ID, env_lookup), - } + AwsAuthConfig::from(merge([params.keys(), env_keys(env_lookup)])) +} + +fn env_keys(env_lookup: &dyn Fn(&str) -> Option) -> AwsKeysPartial { + AwsKeysPartial::from_env(&env_lookup).unwrap_or_default() } /// Where the credentials that sign a request come from, decided when the request is @@ -702,7 +713,10 @@ pub fn host_supplied_credentials(params: &AwsParams) -> Option { #[cfg(test)] mod tests { use super::*; - use crate::constants::{AWS_REGION, BEDROCK_SERVICE}; + use crate::constants::{ + AWS_PROFILE_NAME, AWS_REGION, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, BEDROCK_SERVICE, + }; + use litellm_auth_types::AwsKeys; fn no_env(_: &str) -> Option { None @@ -724,7 +738,7 @@ mod tests { seen.lock() .unwrap() .iter() - .all(|name| AwsParams::secret_names().contains(&name.as_str())) + .all(|name| AwsKeys::ENV_NAMES.contains(&name.as_str())) ); } diff --git a/litellm-rust/crates/auth-aws/src/constants.rs b/litellm-rust/crates/auth-aws/src/constants.rs index 860b4ab7724..8859d147c5b 100644 --- a/litellm-rust/crates/auth-aws/src/constants.rs +++ b/litellm-rust/crates/auth-aws/src/constants.rs @@ -1,22 +1,22 @@ -use litellm_auth_types::AwsParams; +use litellm_auth_types::AwsKeys; -pub const AWS_ACCESS_KEY_ID: &str = AwsParams::ACCESS_KEY_ID.env[0]; -pub const AWS_SECRET_ACCESS_KEY: &str = AwsParams::SECRET_ACCESS_KEY.env[0]; -pub const AWS_SESSION_TOKEN: &str = AwsParams::SESSION_TOKEN.env[0]; -pub const AWS_REGION_NAME: &str = AwsParams::REGION.env[0]; -pub const AWS_REGION: &str = AwsParams::REGION.env[1]; +pub const AWS_ACCESS_KEY_ID: &str = AwsKeys::ACCESS_KEY_ID.env[0]; +pub const AWS_SECRET_ACCESS_KEY: &str = AwsKeys::SECRET_ACCESS_KEY.env[0]; +pub const AWS_SESSION_TOKEN: &str = AwsKeys::SESSION_TOKEN.env[0]; +pub const AWS_REGION_NAME: &str = AwsKeys::REGION.env[0]; +pub const AWS_REGION: &str = AwsKeys::REGION.env[1]; pub const AWS_DEFAULT_REGION: &str = "AWS_DEFAULT_REGION"; -pub const AWS_BEDROCK_RUNTIME_ENDPOINT: &str = AwsParams::BEDROCK_RUNTIME_ENDPOINT.env[0]; -pub const AWS_SESSION_NAME: &str = AwsParams::SESSION_NAME.env[0]; -pub const AWS_PROFILE_NAME: &str = AwsParams::PROFILE_NAME.env[0]; -pub const AWS_ROLE_NAME: &str = AwsParams::ROLE_NAME.env[0]; -pub const AWS_WEB_IDENTITY_TOKEN: &str = AwsParams::WEB_IDENTITY_TOKEN.env[0]; +pub const AWS_BEDROCK_RUNTIME_ENDPOINT: &str = AwsKeys::BEDROCK_RUNTIME_ENDPOINT.env[0]; +pub const AWS_SESSION_NAME: &str = AwsKeys::SESSION_NAME.env[0]; +pub const AWS_PROFILE_NAME: &str = AwsKeys::PROFILE_NAME.env[0]; +pub const AWS_ROLE_NAME: &str = AwsKeys::ROLE_NAME.env[0]; +pub const AWS_WEB_IDENTITY_TOKEN: &str = AwsKeys::WEB_IDENTITY_TOKEN.env[0]; pub const AWS_ROLE_ARN: &str = "AWS_ROLE_ARN"; pub const AWS_ROLE_SESSION_NAME: &str = "AWS_ROLE_SESSION_NAME"; pub const AWS_PROFILE: &str = "AWS_PROFILE"; pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = "AWS_WEB_IDENTITY_TOKEN_FILE"; -pub const AWS_STS_ENDPOINT: &str = AwsParams::STS_ENDPOINT.env[0]; -pub const AWS_EXTERNAL_ID: &str = AwsParams::EXTERNAL_ID.env[0]; +pub const AWS_STS_ENDPOINT: &str = AwsKeys::STS_ENDPOINT.env[0]; +pub const AWS_EXTERNAL_ID: &str = AwsKeys::EXTERNAL_ID.env[0]; pub const AWS_BEARER_TOKEN_BEDROCK: &str = "AWS_BEARER_TOKEN_BEDROCK"; /// Headers SigV4 covers, beyond the `x-amz-` / `x-amzn-` prefixes. Mirrors /// Python's `_filter_headers_for_aws_signature` allowlist. diff --git a/litellm-rust/crates/auth-gcp/Cargo.toml b/litellm-rust/crates/auth-gcp/Cargo.toml index bb4c022580a..a48e236e0db 100644 --- a/litellm-rust/crates/auth-gcp/Cargo.toml +++ b/litellm-rust/crates/auth-gcp/Cargo.toml @@ -9,6 +9,7 @@ repository.workspace = true google-sdk = ["dep:google-cloud-auth", "dep:http"] [dependencies] +litellm-layered.workspace = true litellm-auth-types.workspace = true base64.workspace = true diff --git a/litellm-rust/crates/auth-gcp/src/config.rs b/litellm-rust/crates/auth-gcp/src/config.rs index 5444897aaaf..0ab35e0fcf7 100644 --- a/litellm-rust/crates/auth-gcp/src/config.rs +++ b/litellm-rust/crates/auth-gcp/src/config.rs @@ -7,7 +7,8 @@ use base64::{ use litellm_auth_types::{Error, ErrorDetail, InputSource, SecretValue, Sourced}; use serde_json::{Map, Value}; -use litellm_auth_types::VertexParams; +use litellm_auth_types::{VertexKeys, VertexKeysPartial, VertexParams}; +use litellm_layered::merge; use crate::constants::GOOGLE_APPLICATION_CREDENTIALS_ENV; @@ -36,9 +37,9 @@ impl VertexConfig { sources: &BTreeMap, ) -> Result { Ok(Self::new( - optional_credentials(params, sources, VertexParams::CREDENTIALS.wire)?, - optional_string(params, VertexParams::PROJECT.wire)?, - optional_string(params, VertexParams::LOCATION.wire)?, + optional_credentials(params, sources, VertexKeys::CREDENTIALS.litellm_params)?, + optional_string(params, VertexKeys::PROJECT.litellm_params)?, + optional_string(params, VertexKeys::LOCATION.litellm_params)?, )) } @@ -61,16 +62,28 @@ impl VertexConfig { pub fn location(&self) -> Option<&str> { self.location.as_deref() } + + fn keys(&self) -> VertexKeysPartial { + VertexKeysPartial { + credentials: self + .credentials + .as_ref() + .map(|credentials| credentials.value().expose().to_string()), + project: self.project_id.clone(), + location: self.location.clone(), + } + } + + fn resolved(&self, env_lookup: &dyn Fn(&str) -> Option) -> VertexKeysPartial { + merge([self.keys(), env_keys(env_lookup)]) + } } pub fn get_vertex_ai_project( config: &VertexConfig, env_lookup: &dyn Fn(&str) -> Option, ) -> Option { - config - .project_id() - .map(str::to_string) - .or_else(|| VertexParams::PROJECT.resolve(&|_| None, env_lookup)) + config.resolved(env_lookup).project } /// The project a request is billed to when nothing names it: the `project_id` inside the @@ -104,10 +117,11 @@ pub fn get_vertex_ai_location( config: &VertexConfig, env_lookup: &dyn Fn(&str) -> Option, ) -> Option { - config - .location() - .map(str::to_string) - .or_else(|| VertexParams::LOCATION.resolve(&|_| None, env_lookup)) + config.resolved(env_lookup).location +} + +fn env_keys(env_lookup: &dyn Fn(&str) -> Option) -> VertexKeysPartial { + VertexKeysPartial::from_env(&env_lookup).unwrap_or_default() } const INLINE_JSON_LEADING_NOISE: [char; 7] = [' ', '\t', '\r', '\n', '\u{feff}', '"', '\'']; @@ -156,7 +170,7 @@ pub(crate) fn credential_source( } }; } - if let Some(configured) = VertexParams::CREDENTIALS.resolve(&|_| None, env_lookup) { + if let Some(configured) = config.resolved(env_lookup).credentials { return CredentialSource::Trusted(SecretValue::new(configured)); } non_empty_env(env_lookup, GOOGLE_APPLICATION_CREDENTIALS_ENV) diff --git a/litellm-rust/crates/auth-gcp/src/constants.rs b/litellm-rust/crates/auth-gcp/src/constants.rs index e76b8dfa8ff..7f2f9502dc8 100644 --- a/litellm-rust/crates/auth-gcp/src/constants.rs +++ b/litellm-rust/crates/auth-gcp/src/constants.rs @@ -1,14 +1,14 @@ use std::sync::LazyLock; -use litellm_auth_types::VertexParams; +use litellm_auth_types::VertexKeys; pub const CLOUD_PLATFORM_SCOPE: &str = "https://www.googleapis.com/auth/cloud-platform"; pub const GOOGLE_OAUTH_TOKEN_ENDPOINT: &str = "https://oauth2.googleapis.com/token"; pub const GOOGLE_APPLICATION_CREDENTIALS_ENV: &str = "GOOGLE_APPLICATION_CREDENTIALS"; pub const VERTEX_AI_API_KEY_ENV: &str = "VERTEX_AI_API_KEY"; pub const VERTEXAI_API_KEY_ENV: &str = "VERTEXAI_API_KEY"; -pub const VERTEXAI_CREDENTIALS_ENV: &str = VertexParams::CREDENTIALS.env[0]; -pub const VERTEX_LOCATION_ENV: &str = VertexParams::LOCATION.env[1]; +pub const VERTEXAI_CREDENTIALS_ENV: &str = VertexKeys::CREDENTIALS.env[0]; +pub const VERTEX_LOCATION_ENV: &str = VertexKeys::LOCATION.env[1]; /// Every environment name the Vertex params and the token acquisition read, for hosts /// that resolve secrets up front. @@ -20,11 +20,7 @@ pub fn secret_names() -> &'static [&'static str] { GOOGLE_APPLICATION_CREDENTIALS_ENV, ] .into_iter() - .chain( - VertexParams::SPECS - .iter() - .flat_map(|spec| spec.env.iter().copied()), - ) + .chain(VertexKeys::ENV_NAMES.iter().copied()) .fold(Vec::new(), |mut names, name| { if !names.contains(&name) { names.push(name); diff --git a/litellm-rust/crates/auth-types/Cargo.toml b/litellm-rust/crates/auth-types/Cargo.toml index b54c04f0229..4b3cdcb19d0 100644 --- a/litellm-rust/crates/auth-types/Cargo.toml +++ b/litellm-rust/crates/auth-types/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-layered.workspace = true serde.workspace = true serde_json.workspace = true subtle.workspace = true diff --git a/litellm-rust/crates/auth-types/src/error.rs b/litellm-rust/crates/auth-types/src/error.rs index ff17dcf3810..64fb0846d1b 100644 --- a/litellm-rust/crates/auth-types/src/error.rs +++ b/litellm-rust/crates/auth-types/src/error.rs @@ -19,10 +19,10 @@ pub enum Error { provider: &'static str, environment_variable: &'static str, }, - #[error("Missing {provider} {} - {}", .spec.setting, .spec.guidance())] + #[error("Missing {provider} {} - {}", .spec.label, .spec.guidance())] MissingParam { provider: &'static str, - spec: &'static crate::ParamSpec, + spec: &'static litellm_layered::Sources, }, #[error("Missing {provider} API Base - {guidance}")] MissingApiBase { @@ -108,3 +108,9 @@ impl PartialEq for ErrorSource { } impl Eq for ErrorSource {} + +impl Error { + pub fn missing(provider: &'static str, spec: &'static litellm_layered::Sources) -> Self { + Self::MissingParam { provider, spec } + } +} diff --git a/litellm-rust/crates/auth-types/src/lib.rs b/litellm-rust/crates/auth-types/src/lib.rs index c875bda7979..db7bb18c0f2 100644 --- a/litellm-rust/crates/auth-types/src/lib.rs +++ b/litellm-rust/crates/auth-types/src/lib.rs @@ -53,7 +53,8 @@ pub use credential::{ }; pub use error::{Error, ErrorDetail, ErrorSource}; pub use http::CredentialPlacement; -pub use params::{AwsParams, ParamSpec, VertexParams}; +pub use litellm_layered::Sources; +pub use params::{AwsKeys, AwsKeysPartial, AwsParams, VertexKeys, VertexKeysPartial, VertexParams}; pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy}; pub use secret::SecretValue; pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle}; diff --git a/litellm-rust/crates/auth-types/src/params/aws.rs b/litellm-rust/crates/auth-types/src/params/aws.rs index 25fe52b2ce1..d497c65a62f 100644 --- a/litellm-rust/crates/auth-types/src/params/aws.rs +++ b/litellm-rust/crates/auth-types/src/params/aws.rs @@ -1,22 +1,74 @@ -use std::sync::LazyLock; - +use litellm_layered::Layered; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use veil::Redact; -use super::ParamSpec; - -const fn spec( - setting: &'static str, - wire: &'static [&'static str], - env: &'static [&'static str], -) -> ParamSpec { - ParamSpec { - setting, - wire, - module_global: None, - env, - } +#[derive(Layered)] +pub struct AwsKeys { + #[layer( + skip_blank, + litellm_param = "aws_access_key_id", + env = "AWS_ACCESS_KEY_ID" + )] + pub access_key_id: String, + #[layer( + skip_blank, + litellm_param = "aws_secret_access_key", + env = "AWS_SECRET_ACCESS_KEY", + secret + )] + pub secret_access_key: String, + #[layer( + skip_blank, + litellm_param = "aws_session_token", + env = "AWS_SESSION_TOKEN", + secret + )] + pub session_token: String, + #[layer(skip_blank, litellm_param = "aws_region_name", env = ["AWS_REGION_NAME", "AWS_REGION"])] + pub region: String, + #[layer( + skip_blank, + litellm_param = "aws_session_name", + env = "AWS_SESSION_NAME" + )] + pub session_name: String, + #[layer( + skip_blank, + litellm_param = "aws_profile_name", + env = "AWS_PROFILE_NAME" + )] + pub profile_name: String, + #[layer(skip_blank, litellm_param = "aws_role_name", env = "AWS_ROLE_NAME")] + pub role_name: String, + #[layer( + skip_blank, + litellm_param = "aws_web_identity_token", + env = "AWS_WEB_IDENTITY_TOKEN", + secret + )] + pub web_identity_token: String, + #[layer( + skip_blank, + litellm_param = "aws_sts_endpoint", + env = "AWS_STS_ENDPOINT", + label = "STS endpoint" + )] + pub sts_endpoint: String, + #[layer( + skip_blank, + litellm_param = "aws_external_id", + env = "AWS_EXTERNAL_ID", + secret + )] + pub external_id: String, + #[layer( + skip_blank, + litellm_param = "aws_bedrock_runtime_endpoint", + env = "AWS_BEDROCK_RUNTIME_ENDPOINT", + label = "Bedrock runtime endpoint" + )] + pub bedrock_runtime_endpoint: String, } /// The `aws_*` fields of Python's `GenericLiteLLMParams`. @@ -51,102 +103,8 @@ pub struct AwsParams { } impl AwsParams { - pub const ACCESS_KEY_ID: ParamSpec = spec( - "access key id", - &["aws_access_key_id"], - &["AWS_ACCESS_KEY_ID"], - ); - pub const SECRET_ACCESS_KEY: ParamSpec = spec( - "secret access key", - &["aws_secret_access_key"], - &["AWS_SECRET_ACCESS_KEY"], - ); - pub const SESSION_TOKEN: ParamSpec = spec( - "session token", - &["aws_session_token"], - &["AWS_SESSION_TOKEN"], - ); - pub const REGION: ParamSpec = spec( - "region", - &["aws_region_name"], - &["AWS_REGION_NAME", "AWS_REGION"], - ); - pub const SESSION_NAME: ParamSpec = - spec("session name", &["aws_session_name"], &["AWS_SESSION_NAME"]); - pub const PROFILE_NAME: ParamSpec = - spec("profile name", &["aws_profile_name"], &["AWS_PROFILE_NAME"]); - pub const ROLE_NAME: ParamSpec = spec("role name", &["aws_role_name"], &["AWS_ROLE_NAME"]); - pub const WEB_IDENTITY_TOKEN: ParamSpec = spec( - "web identity token", - &["aws_web_identity_token"], - &["AWS_WEB_IDENTITY_TOKEN"], - ); - pub const STS_ENDPOINT: ParamSpec = - spec("STS endpoint", &["aws_sts_endpoint"], &["AWS_STS_ENDPOINT"]); - pub const EXTERNAL_ID: ParamSpec = - spec("external id", &["aws_external_id"], &["AWS_EXTERNAL_ID"]); - pub const BEDROCK_RUNTIME_ENDPOINT: ParamSpec = spec( - "Bedrock runtime endpoint", - &["aws_bedrock_runtime_endpoint"], - &["AWS_BEDROCK_RUNTIME_ENDPOINT"], - ); - - /// Every `aws_*` param Python reads, in declaration order, with the environment names - /// each falls back to. - pub const SPECS: [ParamSpec; 11] = [ - Self::ACCESS_KEY_ID, - Self::SECRET_ACCESS_KEY, - Self::SESSION_TOKEN, - Self::REGION, - Self::SESSION_NAME, - Self::PROFILE_NAME, - Self::ROLE_NAME, - Self::WEB_IDENTITY_TOKEN, - Self::STS_ENDPOINT, - Self::EXTERNAL_ID, - Self::BEDROCK_RUNTIME_ENDPOINT, - ]; - - /// The wire names a host projects out of a caller's kwargs, derived from [`Self::SPECS`]. - pub fn fields() -> impl Iterator { - Self::SPECS - .iter() - .flat_map(|spec| spec.wire.iter().copied()) - } - - /// The environment names the specs read, for hosts that resolve secrets up front. - pub fn secret_names() -> &'static [&'static str] { - static NAMES: LazyLock> = LazyLock::new(|| { - AwsParams::SPECS - .iter() - .flat_map(|spec| spec.env.iter().copied()) - .fold(Vec::new(), |mut names, name| { - if !names.contains(&name) { - names.push(name); - } - names - }) - }); - &NAMES - } - - /// The value under a wire name, read through serde so the names can never drift from - /// the struct. - pub fn get(&self, wire: &str) -> Option { - serde_json::to_value(self) - .ok()? - .get(wire)? - .as_str() - .map(str::to_string) - } - - /// The spec's value from these params, then the environment. - pub fn resolve( - &self, - spec: &ParamSpec, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Option { - spec.resolve(&|name| self.get(name), env_lookup) + pub fn keys(&self) -> AwsKeysPartial { + AwsKeysPartial::from_litellm_params(&super::as_litellm_params(self)) } /// Reads the string-valued `aws_*` keys of an untyped params map, ignoring anything else. diff --git a/litellm-rust/crates/auth-types/src/params/mod.rs b/litellm-rust/crates/auth-types/src/params/mod.rs index 88bc64cc2fc..839eb30da5e 100644 --- a/litellm-rust/crates/auth-types/src/params/mod.rs +++ b/litellm-rust/crates/auth-types/src/params/mod.rs @@ -1,48 +1,15 @@ mod aws; mod vertex; -pub use aws::AwsParams; -pub use vertex::VertexParams; +pub use aws::{AwsKeys, AwsKeysPartial, AwsParams}; +pub use vertex::{VertexKeys, VertexKeysPartial, VertexParams}; -/// One connection param of Python's `CredentialLiteLLMParams` and every place Python reads -/// it from, in order: the wire names on a call or a deployment, the `litellm.` module -/// global a host folds in, then the environment names. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct ParamSpec { - pub setting: &'static str, - pub wire: &'static [&'static str], - pub module_global: Option<&'static str>, - pub env: &'static [&'static str], -} +use serde::Serialize; +use serde_json::{Map, Value}; -impl ParamSpec { - /// The first non-blank value: each wire name through `params`, then each environment - /// name through `env`. - pub fn resolve( - &self, - params: &dyn Fn(&str) -> Option, - env: &dyn Fn(&str) -> Option, - ) -> Option { - self.wire - .iter() - .map(|name| params(name)) - .chain(self.env.iter().map(|name| env(name))) - .find_map(|value| value.filter(|value| !value.trim().is_empty())) - } - - pub fn missing(&'static self, provider: &'static str) -> crate::Error { - crate::Error::MissingParam { - provider, - spec: self, - } - } - - pub(crate) fn guidance(&self) -> String { - let wire = self.wire.join(" or "); - let env = self.env.join(" or "); - match self.module_global { - Some(global) => format!("pass {wire}, set litellm.{global}, or set {env}"), - None => format!("pass {wire} or set {env}"), - } +fn as_litellm_params(params: &impl Serialize) -> Map { + match serde_json::to_value(params) { + Ok(Value::Object(map)) => map, + _ => Map::new(), } } diff --git a/litellm-rust/crates/auth-types/src/params/vertex.rs b/litellm-rust/crates/auth-types/src/params/vertex.rs index 06ec3008dfb..7fbb7069263 100644 --- a/litellm-rust/crates/auth-types/src/params/vertex.rs +++ b/litellm-rust/crates/auth-types/src/params/vertex.rs @@ -1,8 +1,30 @@ +use litellm_layered::Layered; use serde::{Deserialize, Deserializer, Serialize, de::Unexpected}; use serde_json::Value; use veil::Redact; -use super::ParamSpec; +#[derive(Layered)] +pub struct VertexKeys { + #[layer(skip_blank, + litellm_param = ["vertex_credentials", "vertex_ai_credentials"], + deserialize_with = optional_credential_text, + env = "VERTEXAI_CREDENTIALS", + secret + )] + pub credentials: String, + #[layer(skip_blank, + litellm_param = ["vertex_project", "vertex_ai_project"], + litellm_setting = "vertex_project", + env = "VERTEXAI_PROJECT", + )] + pub project: String, + #[layer(skip_blank, + litellm_param = ["vertex_location", "vertex_ai_location"], + litellm_setting = "vertex_location", + env = ["VERTEXAI_LOCATION", "VERTEX_LOCATION"], + )] + pub location: String, +} /// The Vertex AI fields of Python's `GenericLiteLLMParams`, both the current names and the /// `vertex_ai_*` spellings `VertexBase.safe_get_vertex_ai_*` still read. @@ -25,65 +47,20 @@ pub struct VertexParams { } impl VertexParams { - pub const CREDENTIALS: ParamSpec = ParamSpec { - setting: "credentials", - wire: &["vertex_credentials", "vertex_ai_credentials"], - module_global: None, - env: &["VERTEXAI_CREDENTIALS"], - }; - pub const PROJECT: ParamSpec = ParamSpec { - setting: "project", - wire: &["vertex_project", "vertex_ai_project"], - module_global: Some("vertex_project"), - env: &["VERTEXAI_PROJECT"], - }; - pub const LOCATION: ParamSpec = ParamSpec { - setting: "location", - wire: &["vertex_location", "vertex_ai_location"], - module_global: Some("vertex_location"), - env: &["VERTEXAI_LOCATION", "VERTEX_LOCATION"], - }; - - /// Every Vertex AI param Python reads, with both spellings, the module global - /// `VertexBase.get_vertex_ai_*` consults, and the environment names it falls back to. - pub const SPECS: [ParamSpec; 3] = [Self::CREDENTIALS, Self::PROJECT, Self::LOCATION]; - - /// The wire names a host projects out of a caller's kwargs, derived from [`Self::SPECS`]. - pub fn fields() -> impl Iterator { - Self::SPECS - .iter() - .flat_map(|spec| spec.wire.iter().copied()) - } - - /// The value under a wire name, read through serde so the names can never drift from - /// the struct. - pub fn get(&self, wire: &str) -> Option { - serde_json::to_value(self) - .ok()? - .get(wire)? - .as_str() - .map(str::to_string) - } - - /// The spec's value from these params, then the environment. - pub fn resolve( - &self, - spec: &ParamSpec, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Option { - spec.resolve(&|name| self.get(name), env_lookup) + pub fn keys(&self) -> VertexKeysPartial { + VertexKeysPartial::from_litellm_params(&super::as_litellm_params(self)) } pub fn credentials(&self) -> Option { - self.resolve(&Self::CREDENTIALS, &|_| None) + self.keys().credentials } pub fn project(&self) -> Option { - self.resolve(&Self::PROJECT, &|_| None) + self.keys().project } pub fn location(&self) -> Option { - self.resolve(&Self::LOCATION, &|_| None) + self.keys().location } } diff --git a/litellm-rust/crates/auth-types/tests/aws_params.rs b/litellm-rust/crates/auth-types/tests/aws_params.rs index ed9a4f78ec2..0ac57c39bcc 100644 --- a/litellm-rust/crates/auth-types/tests/aws_params.rs +++ b/litellm-rust/crates/auth-types/tests/aws_params.rs @@ -1,26 +1,27 @@ -use litellm_auth_types::AwsParams; +use std::collections::BTreeSet; + +use litellm_auth_types::{AwsKeys, AwsParams}; use rstest::rstest; use serde_json::{Map, Value}; #[rstest] -fn fields_name_every_param_once_and_in_declaration_order() { - let filled: Map = AwsParams::fields() +fn fields_name_every_param_once() { + let filled: Map = AwsKeys::LITELLM_PARAMS + .iter() .map(|name| (name.to_string(), Value::from(format!("value-of-{name}")))) .collect(); let typed = AwsParams::from_optional_params(&filled); let serialized = serde_json::to_value(&typed).unwrap(); - assert_eq!( - serialized - .as_object() - .unwrap() - .keys() - .map(String::as_str) - .collect::>(), - AwsParams::fields().collect::>() - ); assert_eq!(serialized, Value::Object(filled.clone())); assert_eq!( serde_json::from_value::(Value::Object(filled)).unwrap(), typed ); + assert_eq!( + AwsKeys::LITELLM_PARAMS + .iter() + .collect::>() + .len(), + AwsKeys::LITELLM_PARAMS.len() + ); } diff --git a/litellm-rust/crates/auth-types/tests/params.rs b/litellm-rust/crates/auth-types/tests/params.rs index 9d488b170e1..6b035d7803d 100644 --- a/litellm-rust/crates/auth-types/tests/params.rs +++ b/litellm-rust/crates/auth-types/tests/params.rs @@ -1,20 +1,16 @@ -use litellm_auth_types::{AwsParams, ParamSpec}; +use litellm_auth_types::{AwsKeys, Error, Sources, VertexKeys, VertexKeysPartial, VertexParams}; +use litellm_layered::{Env, merge}; use rstest::rstest; -const LOCATION: ParamSpec = ParamSpec { - setting: "location", - wire: &["vertex_location", "vertex_ai_location"], - module_global: Some("vertex_location"), - env: &["VERTEXAI_LOCATION", "VERTEX_LOCATION"], -}; - -fn lookup(entries: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option { - move |name| { +fn vertex_params(entries: &[(&str, &str)]) -> VertexParams { + serde_json::from_value( entries .iter() - .find(|(key, _)| *key == name) - .map(|(_, value)| value.to_string()) - } + .map(|(name, value)| (name.to_string(), serde_json::Value::from(*value))) + .collect::>() + .into(), + ) + .unwrap() } #[rstest] @@ -38,35 +34,32 @@ fn lookup(entries: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> &[("VERTEX_LOCATION", "second"), ("VERTEXAI_LOCATION", "first")], Some("first") )] +#[case::values_are_kept_untrimmed(&[("vertex_location", " eu ")], &[], Some(" eu "))] #[case::nothing_set(&[], &[], None)] -fn resolution_walks_wire_names_then_environment_names( - #[case] params: &'static [(&'static str, &'static str)], - #[case] env: &'static [(&'static str, &'static str)], +fn params_are_stacked_above_the_environment( + #[case] params: &[(&str, &str)], + #[case] env: &[(&str, &str)], #[case] expected: Option<&str>, ) { - assert_eq!( - LOCATION.resolve(&lookup(params), &lookup(env)).as_deref(), - expected - ); + let merged = merge([ + vertex_params(params).keys(), + VertexKeysPartial::from_env(&Env::of(env)).unwrap(), + ]); + assert_eq!(merged.location.as_deref(), expected); } #[rstest] -#[case::without_a_module_global( - &AwsParams::REGION, +#[case::without_a_litellm_setting( + AwsKeys::REGION, "Missing AWS region - pass aws_region_name or set AWS_REGION_NAME or AWS_REGION" )] -#[case::with_a_module_global( - &LOCATION, +#[case::with_a_litellm_setting( + VertexKeys::LOCATION, "Missing AWS location - pass vertex_location or vertex_ai_location, set litellm.vertex_location, or set VERTEXAI_LOCATION or VERTEX_LOCATION" )] -fn a_missing_param_names_every_place_the_spec_reads( - #[case] spec: &'static ParamSpec, +fn a_missing_param_names_every_source_the_group_reads( + #[case] sources: &'static Sources, #[case] expected: &str, ) { - let message = spec.missing("AWS").to_string(); - - assert_eq!(message, expected); - assert!(spec.wire.iter().all(|name| message.contains(name))); - assert!(spec.env.iter().all(|name| message.contains(name))); - assert_eq!(message.contains("litellm."), spec.module_global.is_some()); + assert_eq!(Error::missing("AWS", sources).to_string(), expected); } diff --git a/litellm-rust/crates/auth-types/tests/vertex_params.rs b/litellm-rust/crates/auth-types/tests/vertex_params.rs index cf0b7f824d1..10c266d06bb 100644 --- a/litellm-rust/crates/auth-types/tests/vertex_params.rs +++ b/litellm-rust/crates/auth-types/tests/vertex_params.rs @@ -1,6 +1,7 @@ use std::collections::BTreeSet; -use litellm_auth_types::VertexParams; +use litellm_auth_types::{VertexKeys, VertexKeysPartial, VertexParams}; +use litellm_layered::{Env, merge}; use rstest::rstest; use serde_json::{Map, Value, json}; @@ -61,8 +62,9 @@ fn a_param_of_the_wrong_type_is_rejected(#[case] params: Value) { #[rstest] fn fields_name_every_param_once() { - let filled: Value = VertexParams::fields() - .map(|name| (name.to_string(), Value::from(name))) + let filled: Value = VertexKeys::LITELLM_PARAMS + .iter() + .map(|name| (name.to_string(), Value::from(*name))) .collect::>() .into(); let params: VertexParams = serde_json::from_value(filled).unwrap(); @@ -76,8 +78,11 @@ fn fields_name_every_param_once() { }; assert_eq!(params, expected); assert_eq!( - VertexParams::fields().collect::>().len(), - VertexParams::fields().count() + VertexKeys::LITELLM_PARAMS + .iter() + .collect::>() + .len(), + VertexKeys::LITELLM_PARAMS.len() ); } @@ -101,7 +106,7 @@ fn debug_output_does_not_expose_credentials() { #[case::legacy_spelling(None, Some("legacy"), &[("VERTEXAI_PROJECT", "env")], Some("legacy"))] #[case::blank_params_fall_to_the_environment(Some(" "), None, &[("VERTEXAI_PROJECT", "env")], Some("env"))] #[case::nothing(None, None, &[], None)] -fn a_spec_resolves_from_the_params_then_the_environment( +fn a_setting_resolves_from_the_params_then_the_environment( #[case] vertex_project: Option<&str>, #[case] vertex_ai_project: Option<&str>, #[case] environment: &[(&str, &str)], @@ -112,15 +117,10 @@ fn a_spec_resolves_from_the_params_then_the_environment( vertex_ai_project: vertex_ai_project.map(str::to_string), ..VertexParams::default() }; - let env = |name: &str| { - environment - .iter() - .find(|(key, _)| *key == name) - .map(|(_, value)| value.to_string()) - }; + let merged = merge([ + params.keys(), + VertexKeysPartial::from_env(&Env::of(environment)).unwrap(), + ]); - assert_eq!( - params.resolve(&VertexParams::PROJECT, &env).as_deref(), - expected - ); + assert_eq!(merged.project.as_deref(), expected); } diff --git a/litellm-rust/crates/config/src/settings.rs b/litellm-rust/crates/config/src/settings.rs index 1094de1638e..c4db497f653 100644 --- a/litellm-rust/crates/config/src/settings.rs +++ b/litellm-rust/crates/config/src/settings.rs @@ -216,6 +216,8 @@ pub struct LiteLlmSettings { pub aiohttp_trust_env: Option, pub disable_aiohttp_trust_env: Option, pub disable_aiohttp_transport: Option, + pub vertex_project: Option, + pub vertex_location: Option, pub drop_params: Option>, pub request_timeout: Option>, pub num_retries: Option, diff --git a/litellm-rust/crates/gateway/src/lib.rs b/litellm-rust/crates/gateway/src/lib.rs index a9617cf2630..a4014946e29 100644 --- a/litellm-rust/crates/gateway/src/lib.rs +++ b/litellm-rust/crates/gateway/src/lib.rs @@ -2,6 +2,7 @@ mod auth; mod error; mod http; mod secrets; +mod vertex; pub use error::Error; use std::{sync::Arc, time::Instant}; @@ -50,7 +51,10 @@ pub fn build_inference(config: &Config, process: &Env) -> Result, E ocr, env, secrets, - ModelRouter::from_model_list(&config.model_list), + ModelRouter::from_model_list(&vertex::with_written_settings( + &config.model_list, + &config.litellm_settings, + )), )?)) } diff --git a/litellm-rust/crates/gateway/src/vertex.rs b/litellm-rust/crates/gateway/src/vertex.rs new file mode 100644 index 00000000000..5e191ec298b --- /dev/null +++ b/litellm-rust/crates/gateway/src/vertex.rs @@ -0,0 +1,89 @@ +use litellm_auth_types::{VertexKeysPartial, VertexParams}; +use litellm_config::{LiteLlmSettings, LitellmParams, Model}; +use litellm_layered::merge; + +pub(crate) fn with_written_settings(models: &[Model], settings: &LiteLlmSettings) -> Vec { + let file = written(settings); + models + .iter() + .map(|model| { + let keys = merge([model.litellm_params.vertex.keys(), file.clone()]); + Model { + litellm_params: LitellmParams { + vertex: VertexParams { + vertex_project: keys.project, + vertex_location: keys.location, + ..model.litellm_params.vertex.clone() + }, + ..model.litellm_params.clone() + }, + ..model.clone() + } + }) + .collect() +} + +fn written(settings: &LiteLlmSettings) -> VertexKeysPartial { + VertexKeysPartial { + credentials: None, + project: settings.vertex_project.clone(), + location: settings.vertex_location.clone(), + } +} + +#[cfg(test)] +mod tests { + use litellm_auth_types::VertexKeys; + use litellm_config::Config; + use litellm_http::HttpKeys; + use rstest::rstest; + use serde_json::{Map, Value}; + + use super::*; + + fn models(yaml: &str) -> Vec { + let config = Config::from_yaml(yaml).unwrap(); + with_written_settings(&config.model_list, &config.litellm_settings) + } + + #[rstest] + #[case::file_fills_an_unset_deployment( + "model_list:\n - model_name: m\n litellm_params:\n model: vertex_ai/claude\nlitellm_settings:\n vertex_project: file-project\n", + Some("file-project") + )] + #[case::the_deployment_wins( + "model_list:\n - model_name: m\n litellm_params:\n model: vertex_ai/claude\n vertex_ai_project: deployment\nlitellm_settings:\n vertex_project: file-project\n", + Some("deployment") + )] + #[case::a_blank_deployment_value_takes_the_file( + "model_list:\n - model_name: m\n litellm_params:\n model: vertex_ai/claude\n vertex_project: ' '\nlitellm_settings:\n vertex_project: file-project\n", + Some("file-project") + )] + #[case::neither( + "model_list:\n - model_name: m\n litellm_params:\n model: vertex_ai/claude\n", + None + )] + fn deployment_params_are_stacked_above_the_config_file( + #[case] yaml: &str, + #[case] expected: Option<&str>, + ) { + assert_eq!( + models(yaml)[0].litellm_params.vertex.project().as_deref(), + expected + ); + } + + #[rstest] + fn every_litellm_setting_a_group_declares_is_a_typed_config_field() { + let names = HttpKeys::LITELLM_SETTINGS + .iter() + .chain(VertexKeys::LITELLM_SETTINGS) + .map(|(name, _)| ((*name).to_owned(), Value::Null)) + .collect::>(); + let settings: LiteLlmSettings = serde_json::from_value(Value::Object(names)).unwrap(); + assert_eq!( + settings.additional_fields.keys().collect::>(), + Vec::<&String>::new() + ); + } +} diff --git a/litellm-rust/crates/gateway/tests/server.rs b/litellm-rust/crates/gateway/tests/server.rs index f153e2cae67..8099640e535 100644 --- a/litellm-rust/crates/gateway/tests/server.rs +++ b/litellm-rust/crates/gateway/tests/server.rs @@ -233,8 +233,19 @@ environment_variables: } #[rstest] +#[case::project_on_the_deployment( + " vertex_project: test-project\n vertex_location: us-east5\n", + "" +)] +#[case::project_only_in_litellm_settings( + "", + "litellm_settings:\n vertex_project: test-project\n vertex_location: us-east5\n" +)] #[tokio::test] -async fn messages_auth_reads_credentials_from_config_environment_variables() { +async fn messages_auth_reads_credentials_from_config_environment_variables( + #[case] deployment: &str, + #[case] litellm_settings: &str, +) { let directory = tempfile::tempdir().unwrap(); let credentials = directory.path().join("service-account.json"); std::fs::write(&credentials, "not a service account").unwrap(); @@ -243,13 +254,11 @@ async fn messages_auth_reads_credentials_from_config_environment_variables() { - model_name: public/vertex litellm_params: model: vertex_ai/claude-sonnet-4-5 - vertex_project: test-project - vertex_location: us-east5 -general_settings: +{deployment}general_settings: master_key: gateway-key environment_variables: GOOGLE_APPLICATION_CREDENTIALS: '{}' -", +{litellm_settings}", credentials.display() )) .unwrap(); diff --git a/litellm-rust/crates/llms/src/aws_textract/ocr/analyze_transformation.rs b/litellm-rust/crates/llms/src/aws_textract/ocr/analyze_transformation.rs index e813d4ac8cc..f7032da2aa6 100644 --- a/litellm-rust/crates/llms/src/aws_textract/ocr/analyze_transformation.rs +++ b/litellm-rust/crates/llms/src/aws_textract/ocr/analyze_transformation.rs @@ -41,7 +41,7 @@ impl BaseOcrConfig for TextractAnalyzeDocumentConfig { type Environment = TextractEnvironment; fn secret_names(&self) -> Vec<&'static str> { - litellm_auth::AwsParams::secret_names().to_vec() + litellm_auth::AwsKeys::ENV_NAMES.to_vec() } fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] { diff --git a/litellm-rust/crates/llms/src/aws_textract/ocr/common_utils.rs b/litellm-rust/crates/llms/src/aws_textract/ocr/common_utils.rs index bc199f40938..89c8fcd20b2 100644 --- a/litellm-rust/crates/llms/src/aws_textract/ocr/common_utils.rs +++ b/litellm-rust/crates/llms/src/aws_textract/ocr/common_utils.rs @@ -239,8 +239,12 @@ pub(super) async fn environment( ) -> Result { let env_lookup = |name: &str| request.connection.secret(name); let params = AwsParams::from_optional_params(&request.optional_params); - let region = resolve_aws_region(None, ¶ms, &env_lookup) - .ok_or_else(|| Error::Auth(AwsParams::REGION.missing("AWS")))?; + let region = resolve_aws_region(None, ¶ms, &env_lookup).ok_or_else(|| { + Error::Auth(litellm_auth::Error::missing( + "AWS", + litellm_auth::AwsKeys::REGION, + )) + })?; let signer = SigV4Signer::resolve( auth, region.clone(), diff --git a/litellm-rust/crates/llms/src/aws_textract/ocr/transformation.rs b/litellm-rust/crates/llms/src/aws_textract/ocr/transformation.rs index 4b27a191734..b18cbb38e64 100644 --- a/litellm-rust/crates/llms/src/aws_textract/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/aws_textract/ocr/transformation.rs @@ -30,7 +30,7 @@ impl BaseOcrConfig for TextractDetectTextConfig { type Environment = TextractEnvironment; fn secret_names(&self) -> Vec<&'static str> { - litellm_auth::AwsParams::secret_names().to_vec() + litellm_auth::AwsKeys::ENV_NAMES.to_vec() } fn get_health_check_document(&self) -> OcrDocument { diff --git a/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs b/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs index 61dd264336e..ebb487b49b1 100644 --- a/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs +++ b/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs @@ -88,7 +88,7 @@ fn optional_string<'a>(params: &'a Map, key: &str) -> Option<&'a impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { fn secret_names(&self) -> Vec<&'static str> { - litellm_auth::AwsParams::secret_names().to_vec() + litellm_auth::AwsKeys::ENV_NAMES.to_vec() } fn get_supported_openai_params(&self) -> &'static [&'static str] { diff --git a/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs b/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs index caa345a7024..1077ae10a95 100644 --- a/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs @@ -94,7 +94,7 @@ pub const BEDROCK_CHAT_COMPLETIONS_CONFIG: AmazonConverseConfig = AmazonConverse impl BaseConfig for AmazonConverseConfig { fn secret_names(&self) -> Vec<&'static str> { - litellm_auth::AwsParams::secret_names() + litellm_auth::AwsKeys::ENV_NAMES .iter() .copied() .chain([AWS_BEARER_TOKEN_BEDROCK]) diff --git a/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs b/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs index 72958fd6cb9..19f1a06e7d2 100644 --- a/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs @@ -1,9 +1,10 @@ use std::sync::LazyLock; -use litellm_auth::{AwsParams, CredentialPlacement, ParamSpec, SecretValue}; +use litellm_auth::{CredentialPlacement, SecretValue}; use litellm_auth_aws::{AwsCredentialSource, resolve_aws_region}; use litellm_core_utils::settings::resolve_non_empty; use litellm_http::request::{header_value, with_header, without_headers}; +use litellm_layered::Layered; use litellm_llms_types::{ formats::messages::{MessagesOptionalParams, MessagesRequest}, providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH}, @@ -26,20 +27,26 @@ use crate::{ const SERVICE: &str = "aws-external-anthropic"; const BASE_ENVS: &[&str] = &["ANTHROPIC_AWS_BASE_URL", "ANTHROPIC_AWS_API_BASE"]; -const WORKSPACE_ENVS: &[&str] = &["ANTHROPIC_AWS_WORKSPACE_ID", "ANTHROPIC_WORKSPACE_ID"]; -const WORKSPACE_FIELDS: &[&str] = &[ - "workspace_id", - "aws_workspace_id", - "anthropic-workspace-id", - "anthropic_workspace_id", -]; +const WORKSPACE_ENVS: &[&str] = WorkspaceKeys::ENV_NAMES; +const WORKSPACE_FIELDS: &[&str] = WorkspaceKeys::LITELLM_PARAMS; const WORKSPACE_HEADER: &str = "anthropic-workspace-id"; -static WORKSPACE: ParamSpec = ParamSpec { - setting: "workspace_id", - wire: WORKSPACE_FIELDS, - module_global: None, - env: WORKSPACE_ENVS, -}; + +#[derive(Layered)] +#[expect(dead_code, reason = "only WorkspaceKeysPartial is built")] +struct WorkspaceKeys { + #[layer( + skip_blank, + litellm_param = [ + "workspace_id", + "aws_workspace_id", + "anthropic-workspace-id", + "anthropic_workspace_id", + ], + env = ["ANTHROPIC_AWS_WORKSPACE_ID", "ANTHROPIC_WORKSPACE_ID"], + label = "workspace_id" + )] + workspace_id: String, +} const UNSUPPORTED_OVERRIDE: &str = "claude_platform_unsupported_params"; pub struct BedrockClaudePlatformMessagesConfig; @@ -72,8 +79,12 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig { .or_else(|| workspace_value(params.extra.get(*field))) }) }); - let workspace = resolve_non_empty(workspace, env, WORKSPACE_ENVS) - .ok_or_else(|| Error::Auth(WORKSPACE.missing("Claude Platform on AWS")))?; + let workspace = resolve_non_empty(workspace, env, WORKSPACE_ENVS).ok_or_else(|| { + Error::Auth(litellm_auth::Error::missing( + "Claude Platform on AWS", + WorkspaceKeys::WORKSPACE_ID, + )) + })?; let headers = with_header(headers, WORKSPACE_HEADER, workspace); let key = resolve_non_empty(api_key, env, &["ANTHROPIC_AWS_API_KEY"]); let headers = without_headers(headers, &["authorization"]); @@ -150,7 +161,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig { .chain(BASE_ENVS.iter().copied()) .chain(WORKSPACE_ENVS.iter().copied()) .chain(["AWS_DEFAULT_REGION"]) - .chain(AwsParams::secret_names().iter().copied()) + .chain(litellm_auth::AwsKeys::ENV_NAMES.iter().copied()) .collect() }); &NAMES @@ -199,7 +210,12 @@ fn platform_region( let region = host_region .or_else(|| resolve_aws_region(None, ¶ms.aws, env)) .or_else(|| resolve_non_empty(None, env, &["AWS_DEFAULT_REGION"])) - .ok_or_else(|| Error::Auth(AwsParams::REGION.missing("Claude Platform on AWS")))?; + .ok_or_else(|| { + Error::Auth(litellm_auth::Error::missing( + "Claude Platform on AWS", + litellm_auth::AwsKeys::REGION, + )) + })?; validated_region(region) } @@ -250,6 +266,7 @@ fn invalid_request(error: serde_json::Error) -> Error { #[cfg(test)] mod tests { use super::*; + use litellm_auth::AwsParams; use litellm_llms_types::providers::anthropic::AnthropicBeta; use rstest::rstest; use serde_json::json; @@ -267,7 +284,10 @@ mod tests { &|_| None ) .unwrap_err(), - Error::Auth(WORKSPACE.missing("Claude Platform on AWS")) + Error::Auth(litellm_auth::Error::missing( + "Claude Platform on AWS", + WorkspaceKeys::WORKSPACE_ID, + )) ); } diff --git a/litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs b/litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs index ae13704f4a0..46ba0823ede 100644 --- a/litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs @@ -1,6 +1,6 @@ use std::sync::LazyLock; -use litellm_auth::{AwsParams, CredentialPlacement, SecretValue}; +use litellm_auth::{CredentialPlacement, SecretValue}; use litellm_auth_aws::AwsCredentialSource; use litellm_auth_aws::bedrock_model_id_and_region; use litellm_core_utils::settings::resolve_non_empty; @@ -164,7 +164,7 @@ impl BaseMessagesConfig for BedrockMantleAnthropicMessagesConfig { .into_iter() .chain(REGION_ENVS.iter().copied()) .chain( - AwsParams::secret_names() + litellm_auth::AwsKeys::ENV_NAMES .iter() .copied() .filter(|name| !REGION_ENVS.contains(name)), @@ -263,6 +263,7 @@ fn mantle_region( #[cfg(test)] mod tests { use super::*; + use litellm_auth::AwsParams; use litellm_http::request::header_value; use rstest::rstest; use serde_json::json; diff --git a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs index 2e77dbc1ca8..e28b09dc8c7 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs @@ -1,4 +1,3 @@ -use litellm_auth::VertexParams; use litellm_auth_gcp::{ VertexConfig, get_vertex_ai_location, get_vertex_ai_project, get_vertex_ai_project_from_credentials, secret_names, @@ -64,7 +63,12 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig { let config = VertexConfig::from_params(&litellm_params.vertex); let project = get_vertex_ai_project(&config, env_lookup) .or_else(|| get_vertex_ai_project_from_credentials(&config, env_lookup)) - .ok_or_else(|| Error::Auth(VertexParams::PROJECT.missing("Vertex AI")))?; + .ok_or_else(|| { + Error::Auth(litellm_auth::Error::missing( + "Vertex AI", + litellm_auth::VertexKeys::PROJECT, + )) + })?; let location = get_vertex_ai_location(&config, env_lookup) .unwrap_or_else(|| DEFAULT_LOCATION.to_string()); complete_vertex_anthropic_url(api_base, &project, &location, model, stream) @@ -227,6 +231,7 @@ fn vertex_feature_betas(request: &MessagesRequest) -> BetaSet { #[cfg(test)] mod tests { + use litellm_auth::VertexParams; use rstest::rstest; use serde_json::json; @@ -422,7 +427,7 @@ mod tests { error, Error::Auth(litellm_auth::Error::MissingParam { provider: "Vertex AI", - spec: &VertexParams::PROJECT, + spec: litellm_auth::VertexKeys::PROJECT, }) )); assert_eq!( diff --git a/litellm-rust/crates/python-bridge/src/routes/credential_globals.rs b/litellm-rust/crates/python-bridge/src/routes/credential_globals.rs new file mode 100644 index 00000000000..8bab68584b8 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/credential_globals.rs @@ -0,0 +1,74 @@ +use litellm_auth::Sources; +use pyo3::prelude::*; +use serde_json::{Map, Value}; + +use crate::{ + coercion::FieldSpec, + python_settings::{PythonSettings, Snapshot}, +}; + +/// The settings among `sources` that have a `litellm.*` global and that `params` leaves unset, +/// counting a blank value as unset the way Python's `or` chain does. +pub(crate) fn unset<'s>( + params: &Map, + sources: impl IntoIterator, +) -> Vec<&'s Sources> { + sources + .into_iter() + .filter(|sources| sources.litellm_setting.is_some()) + .filter(|sources| { + !sources + .litellm_params + .iter() + .any(|name| params.get(*name).is_some_and(is_set)) + }) + .collect() +} + +fn is_set(value: &Value) -> bool { + match value { + Value::Null => false, + Value::String(text) => !text.trim().is_empty(), + _ => true, + } +} + +/// Reads the globals of `unset` through the `ProviderDefaults` accessor and places each truthy +/// one under its setting's first param name, replacing a blank value the call passed. +pub(crate) fn fold( + py: Python<'_>, + params: Map, + unset: &[&Sources], +) -> PyResult> { + if unset.is_empty() { + return Ok(params); + } + let Some(defaults) = PythonSettings::ProviderDefaults.read_or_unset(py)? else { + return Ok(params); + }; + fold_from(params, unset, &defaults) +} + +pub(crate) fn fold_from( + params: Map, + unset: &[&Sources], + defaults: &Snapshot<'_>, +) -> PyResult> { + let globals = unset + .iter() + .filter_map(|sources| { + let setting = sources.litellm_setting?; + let name = sources.litellm_params.first()?; + let spec = FieldSpec::new(setting, |field| field.falsy_optional_string()); + Some( + defaults + .read(&spec) + .map(|value| value.map(|value| (name.to_string(), Value::String(value)))), + ) + }) + .collect::, _>>()?; + Ok(params + .into_iter() + .chain(globals.into_iter().flatten()) + .collect()) +} diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs index 5e445fcaa7d..01c9b534f1d 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs @@ -2,6 +2,7 @@ use crate::cache::{CacheCall, Cached, PythonCache, Selection}; use litellm_host_python::{PythonHostCalls, PythonOwned}; use bytes::Bytes; +use litellm_auth::Sources; use litellm_host_python::{InvokeError, PythonBinding, from_py, to_py}; use litellm_http::transport::Error as TransportError; use litellm_inference_messages::{ @@ -22,8 +23,7 @@ use serde_json::{Map, Value}; use crate::{ errors::{RustUpstreamError, route_error_to_pyerr}, marshal::{optional_timeout, project_optional_fields, public_response, python_timeout_seconds}, - python_settings::missing_module, - routes::RequestView, + routes::{RequestView, credential_globals}, }; const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.messages"; @@ -41,31 +41,16 @@ fn merge_headers( (!merged.is_empty()).then_some(merged) } -/// Reads `litellm.`, the module global a param spec names, the way -/// `VertexBase.get_vertex_ai_project` reads `litellm.vertex_project`. -fn module_global<'py>(py: Python<'py>, name: &str) -> PyResult>> { - let module = match py.import("litellm") { - Ok(module) => module, - Err(error) if missing_module(py, &error, "litellm")? => return Ok(None), - Err(error) => return Err(error), - }; - let value = module.getattr(name)?; - Ok((!value.is_none()).then_some(value)) -} - /// The caller's litellm params, read from the kwargs by the names the typed params declare, -/// so a key the configs do not read is never converted; a spec whose spellings the call all -/// leaves out falls back to the module global it names, which stays the host's concern. +/// so a key the configs do not read is never converted. A setting the call leaves unset or +/// blank takes its `litellm.*` global, as Python's `or` chain does. fn project_litellm_params<'py>( argument: impl Fn(&str) -> PyResult>>, - global: impl Fn(&str) -> PyResult>>, + fold: impl Fn(Map, &[&Sources]) -> PyResult>, ) -> PyResult> { let fields = project_optional_fields(LitellmParams::fields(), &argument)?; - let globals = LitellmParams::specs() - .filter(|spec| !spec.wire.iter().any(|name| fields.contains_key(*name))) - .filter_map(|spec| spec.module_global); - let folded = project_optional_fields(globals, &global)?; - Ok(litellm_params(fields.into_iter().chain(folded).collect())) + let unset = credential_globals::unset(&fields, LitellmParams::litellm_settings()); + Ok(litellm_params(fold(fields, &unset)?)) } fn native_error(py: Python<'_>, error: Error) -> PyResult { @@ -136,7 +121,9 @@ impl MessagesPythonHost { let api_base = string("api_base")?; let extra_headers = self.merged_headers(py, arguments)?; let provider_specific_header = self.provider_specific_header(py, arguments)?; - let litellm_params = project_litellm_params(argument, |name| module_global(py, name))?; + let litellm_params = project_litellm_params(argument, |fields, unset| { + credential_globals::fold(py, fields, unset) + })?; Ok(messages_body(body).and_then(|body| { Ok(MessagesCall { body, @@ -336,6 +323,7 @@ mod tests { use serde_json::json; use super::*; + use crate::python_settings::PythonSettings; fn map(value: Value) -> Map { serde_json::from_value(value).unwrap() @@ -404,7 +392,12 @@ mod tests { json!({"vertex_project": "from-global"}), Ok(json!({"model": "m", "vertex_project": "from-global"})), )] - #[case::a_global_of_the_wrong_type_is_a_request_error( + #[case::a_blank_value_in_the_call_falls_back( + json!({"model": "m", "vertex_project": "", "vertex_ai_project": " "}), + json!({"vertex_project": "from-global"}), + Ok(json!({"model": "m", "vertex_project": "from-global", "vertex_ai_project": " "})), + )] + #[case::a_global_of_the_wrong_type_is_an_error( json!({"model": "m"}), json!({"vertex_location": 5}), Err(()), @@ -447,19 +440,27 @@ mod tests { .unwrap() }; let kwargs = dict(&kwargs); - let globals = dict(&globals); + let defaults = dict(&json!({"vertex_project": null, "vertex_location": null})); + defaults.update(dict(&globals).as_mapping()).unwrap(); + let namespace = py + .import("types") + .unwrap() + .getattr("SimpleNamespace") + .unwrap() + .call((), Some(&defaults)) + .unwrap(); + let snapshot = PythonSettings::ProviderDefaults.snapshot(namespace); let bound = PyDict::new(py); let projected = project_litellm_params( |name| present(&kwargs, &bound, name), - |name| present(&globals, &bound, name), - ) - .unwrap(); + |fields, unset| credential_globals::fold_from(fields, unset, &snapshot), + ); match (projected, expected) { - (Ok(projected), Ok(fields)) => assert_eq!( + (Ok(Ok(projected)), Ok(fields)) => assert_eq!( projected, litellm_params(serde_json::from_value(fields).unwrap()).unwrap() ), - (Err(error), Err(())) => assert!(matches!(error, Error::InvalidRequest(_))), + (Err(_) | Ok(Err(_)), Err(())) => {} (projected, expected) => panic!("got {projected:?}, expected {expected:?}"), } }); diff --git a/litellm-rust/crates/python-bridge/src/routes/mod.rs b/litellm-rust/crates/python-bridge/src/routes/mod.rs index f51c755c33e..e68af64df0b 100644 --- a/litellm-rust/crates/python-bridge/src/routes/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod audio_transcription; pub(crate) mod chat_completions; pub(crate) mod clickhouse_spend; +mod credential_globals; pub(crate) mod embeddings; mod inference; pub(crate) mod messages; diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs index fe2f83bff46..953e0bf0a67 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs @@ -1,5 +1,5 @@ use litellm_auth::SecretValue; -use litellm_auth::VertexParams; +use litellm_auth::{Sources, VertexKeys}; use litellm_host_python::{from_py, present}; use litellm_inference_ocr::{ types::{LiteLLMOcrRequest, OcrDocumentInput}, @@ -11,10 +11,9 @@ use serde_json::{Map, Value}; use super::{document::FileDocumentInput, errors::to_pyerr as ocr_error_to_pyerr}; use crate::{ - coercion::FieldSpec, credentials::{self, CallerTokenProvider}, marshal::{project_optional_fields, python_timeout_seconds, request_input_sources}, - python_settings::{PythonSettings, Snapshot}, + routes::credential_globals, }; /// What the host keeps after projection: the caller's token callable that answers the @@ -126,16 +125,9 @@ pub(super) fn project_request( let names = specs.iter().map(|spec| spec.name).collect::>(); let optional_params = project_optional_fields(names.iter().copied(), |name| present(kwargs, bound, name))?; - let globals = module_globals_to_read(&optional_params, &names); - let defaults = if globals.is_empty() { - None - } else { - PythonSettings::ProviderDefaults.read_or_unset(bound.py())? - }; - let optional_params = match defaults { - Some(defaults) => fold_module_globals(bound.py(), optional_params, &globals, &defaults)?, - None => optional_params, - }; + let unset = unset_vertex_settings(&optional_params, &names); + let optional_params = credential_globals::fold(bound.py(), optional_params, &unset) + .map_err(|error| settings_error(bound.py(), error))?; let input_sources = request_input_sources( kwargs, names @@ -169,48 +161,21 @@ pub(super) fn project_request( )) } -/// Python's `VertexBase.get_vertex_ai_project` and `get_vertex_ai_location` fall back to a -/// `litellm.` module global when a call names neither spelling: the globals of the -/// specs this route consumes whose wire names the call left out. -fn module_globals_to_read( +/// The Vertex settings this route consumes that the call leaves unset, whose `litellm.*` +/// globals `VertexBase.get_vertex_ai_project` and `get_vertex_ai_location` fall back to. +fn unset_vertex_settings( optional_params: &Map, consumed: &[&str], -) -> Vec<&'static str> { - VertexParams::SPECS - .iter() - .filter(|spec| spec.wire.iter().all(|name| consumed.contains(name))) - .filter(|spec| { - !spec - .wire +) -> Vec<&'static Sources> { + credential_globals::unset( + optional_params, + VertexKeys::SOURCES.iter().filter(|sources| { + sources + .litellm_params .iter() - .any(|name| optional_params.contains_key(*name)) - }) - .filter_map(|spec| spec.module_global) - .collect() -} - -/// Folds each module global in under the name its spec declares, skipping falsy values the -/// way Python's `or` chain does. -fn fold_module_globals( - py: Python<'_>, - optional_params: Map, - globals: &[&'static str], - defaults: &Snapshot<'_>, -) -> PyResult> { - let read = globals - .iter() - .map(|name| { - let spec = FieldSpec::new(name, |field| field.falsy_optional_string()); - Ok(defaults - .read(&spec) - .map_err(|error| settings_error(py, error.into()))? - .map(|value| (name.to_string(), Value::String(value)))) - }) - .collect::>>()?; - Ok(optional_params - .into_iter() - .chain(read.into_iter().flatten()) - .collect()) + .all(|name| consumed.contains(name)) + }), + ) } /// A misconfigured litellm setting is the operator's error, not the request's or the @@ -228,6 +193,7 @@ mod tests { use pyo3::exceptions::PyValueError; use super::*; + use crate::python_settings::PythonSettings; #[rstest::rstest] #[case::global_fills_a_missing_project( @@ -263,11 +229,10 @@ mod tests { ) .unwrap(); let snapshot = PythonSettings::ProviderDefaults.snapshot(namespace); - let consumed = VertexParams::fields().collect::>(); let params = params.as_object().cloned().unwrap(); - let globals = module_globals_to_read(¶ms, &consumed); + let unset = unset_vertex_settings(¶ms, VertexKeys::LITELLM_PARAMS); - let folded = fold_module_globals(py, params, &globals, &snapshot).unwrap(); + let folded = credential_globals::fold_from(params, &unset, &snapshot).unwrap(); assert_eq!(Value::Object(folded), expected); }); @@ -286,8 +251,11 @@ mod tests { .unwrap(); let snapshot = PythonSettings::ProviderDefaults.snapshot(namespace); - let error = - fold_module_globals(py, Map::new(), &["vertex_project"], &snapshot).unwrap_err(); + let error = settings_error( + py, + credential_globals::fold_from(Map::new(), &[VertexKeys::PROJECT], &snapshot) + .unwrap_err(), + ); assert!(error.is_instance_of::(py)); assert!( @@ -323,7 +291,10 @@ mod tests { #[case] expected: &[&str], ) { assert_eq!( - module_globals_to_read(params.as_object().unwrap(), consumed), + unset_vertex_settings(params.as_object().unwrap(), consumed) + .iter() + .filter_map(|sources| sources.litellm_setting) + .collect::>(), expected ); } diff --git a/litellm-rust/crates/router-types/src/litellm_params.rs b/litellm-rust/crates/router-types/src/litellm_params.rs index 186a1d98036..c2b4d5a610c 100644 --- a/litellm-rust/crates/router-types/src/litellm_params.rs +++ b/litellm-rust/crates/router-types/src/litellm_params.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, fmt, ops::Deref}; -use litellm_auth_types::{AwsParams, ParamSpec, SecretValue, VertexParams}; +use litellm_auth_types::{AwsKeys, AwsParams, SecretValue, VertexKeys, VertexParams}; use serde::Deserialize; use serde_json::Value; use serde_with::{OneOrMany, formats::PreferMany, serde_as}; @@ -54,10 +54,13 @@ pub struct LitellmParams { } impl LitellmParams { - /// Every connection param spec the credential families declare, for hosts that fold a - /// spec's module global in when a call names none of its spellings. - pub fn specs() -> impl Iterator { - AwsParams::SPECS.iter().chain(VertexParams::SPECS.iter()) + /// Every credential setting that has a `litellm.*` global, for hosts that fold the global + /// in when a call leaves the setting unset. + pub fn litellm_settings() -> impl Iterator { + AwsKeys::SOURCES + .iter() + .chain(VertexKeys::SOURCES.iter()) + .filter(|sources| sources.litellm_setting.is_some()) } /// The wire names a host projects out of a caller's kwargs: the model, the credentials @@ -66,8 +69,8 @@ impl LitellmParams { pub fn fields() -> impl Iterator { ["model", "api_key", "api_base", "api_version"] .into_iter() - .chain(AwsParams::fields()) - .chain(VertexParams::fields()) + .chain(AwsKeys::LITELLM_PARAMS.iter().copied()) + .chain(VertexKeys::LITELLM_PARAMS.iter().copied()) } }