mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
refactor(auth-types): derive AWS and Vertex credential sources and remove ParamSpec
AWS and Vertex params are declared with the Layered derive, which replaces ParamSpec, and vertex_project and vertex_location become typed config settings Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
b61449e35d
commit
e3ad07aa68
33 changed files with 584 additions and 467 deletions
3
litellm-rust/Cargo.lock
generated
3
litellm-rust/Cargo.lock
generated
|
|
@ -3366,6 +3366,7 @@ dependencies = [
|
|||
"aws-types",
|
||||
"litellm-auth-types",
|
||||
"litellm-http",
|
||||
"litellm-layered",
|
||||
"moka",
|
||||
"reqwest 0.12.28",
|
||||
"rstest",
|
||||
|
|
@ -3404,6 +3405,7 @@ dependencies = [
|
|||
"google-cloud-auth",
|
||||
"http 1.4.2",
|
||||
"litellm-auth-types",
|
||||
"litellm-layered",
|
||||
"moka",
|
||||
"rstest",
|
||||
"serde",
|
||||
|
|
@ -3418,6 +3420,7 @@ dependencies = [
|
|||
name = "litellm-auth-types"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"litellm-layered",
|
||||
"rstest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ license.workspace = true
|
|||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
litellm-layered.workspace = true
|
||||
litellm-auth-types.workspace = true
|
||||
litellm-http.workspace = true
|
||||
|
||||
|
|
|
|||
|
|
@ -13,15 +13,14 @@ use aws_sigv4::http_request::{
|
|||
use aws_sigv4::sign::v4;
|
||||
use aws_smithy_runtime_api::client::identity::Identity;
|
||||
|
||||
use litellm_auth_types::AwsParams;
|
||||
use litellm_auth_types::{AwsKeysPartial, AwsParams};
|
||||
use litellm_layered::merge;
|
||||
|
||||
use super::Error;
|
||||
use super::constants::{
|
||||
AWS_ACCESS_KEY_ID, AWS_EXTERNAL_ID, AWS_PROFILE, AWS_PROFILE_NAME, AWS_REGION_NAME,
|
||||
AWS_ROLE_ARN, AWS_ROLE_NAME, AWS_ROLE_SESSION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_NAME,
|
||||
AWS_SESSION_TOKEN, AWS_SIGNED_HEADER_NAMES, AWS_STS_ENDPOINT, AWS_WEB_IDENTITY_TOKEN,
|
||||
AWS_WEB_IDENTITY_TOKEN_FILE, DEFAULT_BEDROCK_REGION, DEFAULT_SESSION_NAME_PREFIX,
|
||||
SIGV4_COMPUTED_HEADER_NAMES,
|
||||
AWS_ACCESS_KEY_ID, AWS_PROFILE, AWS_ROLE_ARN, AWS_ROLE_SESSION_NAME, AWS_SECRET_ACCESS_KEY,
|
||||
AWS_SESSION_TOKEN, AWS_SIGNED_HEADER_NAMES, AWS_WEB_IDENTITY_TOKEN_FILE,
|
||||
DEFAULT_BEDROCK_REGION, DEFAULT_SESSION_NAME_PREFIX, SIGV4_COMPUTED_HEADER_NAMES,
|
||||
};
|
||||
|
||||
const STATIC_CREDENTIALS_TTL: Duration = Duration::from_secs(3600 - 60);
|
||||
|
|
@ -75,21 +74,39 @@ pub struct AwsAuthConfig {
|
|||
|
||||
impl AwsAuthConfig {
|
||||
fn with_environment(self, env_lookup: &(dyn Fn(&str) -> Option<String> + Sync)) -> Self {
|
||||
Self::from(merge([self.keys(), env_keys(env_lookup)]))
|
||||
}
|
||||
|
||||
fn keys(self) -> AwsKeysPartial {
|
||||
AwsKeysPartial {
|
||||
access_key_id: self.access_key_id,
|
||||
secret_access_key: self.secret_access_key,
|
||||
session_token: self.session_token,
|
||||
region: self.region_name,
|
||||
session_name: self.session_name,
|
||||
profile_name: self.profile_name,
|
||||
role_name: self.role_name,
|
||||
web_identity_token: self.web_identity_token,
|
||||
sts_endpoint: self.sts_endpoint,
|
||||
external_id: self.external_id,
|
||||
bedrock_runtime_endpoint: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<AwsKeysPartial> for AwsAuthConfig {
|
||||
fn from(keys: AwsKeysPartial) -> Self {
|
||||
Self {
|
||||
access_key_id: self.access_key_id.or_else(|| env_lookup(AWS_ACCESS_KEY_ID)),
|
||||
secret_access_key: self
|
||||
.secret_access_key
|
||||
.or_else(|| env_lookup(AWS_SECRET_ACCESS_KEY)),
|
||||
session_token: self.session_token.or_else(|| env_lookup(AWS_SESSION_TOKEN)),
|
||||
region_name: self.region_name.or_else(|| env_lookup(AWS_REGION_NAME)),
|
||||
session_name: self.session_name.or_else(|| env_lookup(AWS_SESSION_NAME)),
|
||||
profile_name: self.profile_name.or_else(|| env_lookup(AWS_PROFILE_NAME)),
|
||||
role_name: self.role_name.or_else(|| env_lookup(AWS_ROLE_NAME)),
|
||||
web_identity_token: self
|
||||
.web_identity_token
|
||||
.or_else(|| env_lookup(AWS_WEB_IDENTITY_TOKEN)),
|
||||
sts_endpoint: self.sts_endpoint.or_else(|| env_lookup(AWS_STS_ENDPOINT)),
|
||||
external_id: self.external_id.or_else(|| env_lookup(AWS_EXTERNAL_ID)),
|
||||
access_key_id: keys.access_key_id,
|
||||
secret_access_key: keys.secret_access_key,
|
||||
session_token: keys.session_token,
|
||||
region_name: keys.region,
|
||||
session_name: keys.session_name,
|
||||
profile_name: keys.profile_name,
|
||||
role_name: keys.role_name,
|
||||
web_identity_token: keys.web_identity_token,
|
||||
sts_endpoint: keys.sts_endpoint,
|
||||
external_id: keys.external_id,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -612,9 +629,10 @@ pub fn resolve_aws_region(
|
|||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Option<String> {
|
||||
params
|
||||
.resolve(&AwsParams::REGION, &|_| None)
|
||||
.keys()
|
||||
.region
|
||||
.or_else(|| model_region.map(str::to_string))
|
||||
.or_else(|| AwsParams::REGION.resolve(&|_| None, env_lookup))
|
||||
.or_else(|| env_keys(env_lookup).region)
|
||||
}
|
||||
|
||||
pub fn resolve_bedrock_region(
|
||||
|
|
@ -630,18 +648,11 @@ pub fn aws_auth_config(
|
|||
params: &AwsParams,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> AwsAuthConfig {
|
||||
AwsAuthConfig {
|
||||
access_key_id: params.resolve(&AwsParams::ACCESS_KEY_ID, env_lookup),
|
||||
secret_access_key: params.resolve(&AwsParams::SECRET_ACCESS_KEY, env_lookup),
|
||||
session_token: params.resolve(&AwsParams::SESSION_TOKEN, env_lookup),
|
||||
region_name: params.resolve(&AwsParams::REGION, env_lookup),
|
||||
session_name: params.resolve(&AwsParams::SESSION_NAME, env_lookup),
|
||||
profile_name: params.resolve(&AwsParams::PROFILE_NAME, env_lookup),
|
||||
role_name: params.resolve(&AwsParams::ROLE_NAME, env_lookup),
|
||||
web_identity_token: params.resolve(&AwsParams::WEB_IDENTITY_TOKEN, env_lookup),
|
||||
sts_endpoint: params.resolve(&AwsParams::STS_ENDPOINT, env_lookup),
|
||||
external_id: params.resolve(&AwsParams::EXTERNAL_ID, env_lookup),
|
||||
}
|
||||
AwsAuthConfig::from(merge([params.keys(), env_keys(env_lookup)]))
|
||||
}
|
||||
|
||||
fn env_keys(env_lookup: &dyn Fn(&str) -> Option<String>) -> AwsKeysPartial {
|
||||
AwsKeysPartial::from_env(&env_lookup).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Where the credentials that sign a request come from, decided when the request is
|
||||
|
|
@ -702,7 +713,10 @@ pub fn host_supplied_credentials(params: &AwsParams) -> Option<Credentials> {
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::constants::{AWS_REGION, BEDROCK_SERVICE};
|
||||
use crate::constants::{
|
||||
AWS_PROFILE_NAME, AWS_REGION, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, BEDROCK_SERVICE,
|
||||
};
|
||||
use litellm_auth_types::AwsKeys;
|
||||
|
||||
fn no_env(_: &str) -> Option<String> {
|
||||
None
|
||||
|
|
@ -724,7 +738,7 @@ mod tests {
|
|||
seen.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.all(|name| AwsParams::secret_names().contains(&name.as_str()))
|
||||
.all(|name| AwsKeys::ENV_NAMES.contains(&name.as_str()))
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,22 +1,22 @@
|
|||
use litellm_auth_types::AwsParams;
|
||||
use litellm_auth_types::AwsKeys;
|
||||
|
||||
pub const AWS_ACCESS_KEY_ID: &str = AwsParams::ACCESS_KEY_ID.env[0];
|
||||
pub const AWS_SECRET_ACCESS_KEY: &str = AwsParams::SECRET_ACCESS_KEY.env[0];
|
||||
pub const AWS_SESSION_TOKEN: &str = AwsParams::SESSION_TOKEN.env[0];
|
||||
pub const AWS_REGION_NAME: &str = AwsParams::REGION.env[0];
|
||||
pub const AWS_REGION: &str = AwsParams::REGION.env[1];
|
||||
pub const AWS_ACCESS_KEY_ID: &str = AwsKeys::ACCESS_KEY_ID.env[0];
|
||||
pub const AWS_SECRET_ACCESS_KEY: &str = AwsKeys::SECRET_ACCESS_KEY.env[0];
|
||||
pub const AWS_SESSION_TOKEN: &str = AwsKeys::SESSION_TOKEN.env[0];
|
||||
pub const AWS_REGION_NAME: &str = AwsKeys::REGION.env[0];
|
||||
pub const AWS_REGION: &str = AwsKeys::REGION.env[1];
|
||||
pub const AWS_DEFAULT_REGION: &str = "AWS_DEFAULT_REGION";
|
||||
pub const AWS_BEDROCK_RUNTIME_ENDPOINT: &str = AwsParams::BEDROCK_RUNTIME_ENDPOINT.env[0];
|
||||
pub const AWS_SESSION_NAME: &str = AwsParams::SESSION_NAME.env[0];
|
||||
pub const AWS_PROFILE_NAME: &str = AwsParams::PROFILE_NAME.env[0];
|
||||
pub const AWS_ROLE_NAME: &str = AwsParams::ROLE_NAME.env[0];
|
||||
pub const AWS_WEB_IDENTITY_TOKEN: &str = AwsParams::WEB_IDENTITY_TOKEN.env[0];
|
||||
pub const AWS_BEDROCK_RUNTIME_ENDPOINT: &str = AwsKeys::BEDROCK_RUNTIME_ENDPOINT.env[0];
|
||||
pub const AWS_SESSION_NAME: &str = AwsKeys::SESSION_NAME.env[0];
|
||||
pub const AWS_PROFILE_NAME: &str = AwsKeys::PROFILE_NAME.env[0];
|
||||
pub const AWS_ROLE_NAME: &str = AwsKeys::ROLE_NAME.env[0];
|
||||
pub const AWS_WEB_IDENTITY_TOKEN: &str = AwsKeys::WEB_IDENTITY_TOKEN.env[0];
|
||||
pub const AWS_ROLE_ARN: &str = "AWS_ROLE_ARN";
|
||||
pub const AWS_ROLE_SESSION_NAME: &str = "AWS_ROLE_SESSION_NAME";
|
||||
pub const AWS_PROFILE: &str = "AWS_PROFILE";
|
||||
pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = "AWS_WEB_IDENTITY_TOKEN_FILE";
|
||||
pub const AWS_STS_ENDPOINT: &str = AwsParams::STS_ENDPOINT.env[0];
|
||||
pub const AWS_EXTERNAL_ID: &str = AwsParams::EXTERNAL_ID.env[0];
|
||||
pub const AWS_STS_ENDPOINT: &str = AwsKeys::STS_ENDPOINT.env[0];
|
||||
pub const AWS_EXTERNAL_ID: &str = AwsKeys::EXTERNAL_ID.env[0];
|
||||
pub const AWS_BEARER_TOKEN_BEDROCK: &str = "AWS_BEARER_TOKEN_BEDROCK";
|
||||
/// Headers SigV4 covers, beyond the `x-amz-` / `x-amzn-` prefixes. Mirrors
|
||||
/// Python's `_filter_headers_for_aws_signature` allowlist.
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ repository.workspace = true
|
|||
google-sdk = ["dep:google-cloud-auth", "dep:http"]
|
||||
|
||||
[dependencies]
|
||||
litellm-layered.workspace = true
|
||||
litellm-auth-types.workspace = true
|
||||
|
||||
base64.workspace = true
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@ use base64::{
|
|||
use litellm_auth_types::{Error, ErrorDetail, InputSource, SecretValue, Sourced};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use litellm_auth_types::VertexParams;
|
||||
use litellm_auth_types::{VertexKeys, VertexKeysPartial, VertexParams};
|
||||
use litellm_layered::merge;
|
||||
|
||||
use crate::constants::GOOGLE_APPLICATION_CREDENTIALS_ENV;
|
||||
|
||||
|
|
@ -36,9 +37,9 @@ impl VertexConfig {
|
|||
sources: &BTreeMap<String, InputSource>,
|
||||
) -> Result<Self, Error> {
|
||||
Ok(Self::new(
|
||||
optional_credentials(params, sources, VertexParams::CREDENTIALS.wire)?,
|
||||
optional_string(params, VertexParams::PROJECT.wire)?,
|
||||
optional_string(params, VertexParams::LOCATION.wire)?,
|
||||
optional_credentials(params, sources, VertexKeys::CREDENTIALS.litellm_params)?,
|
||||
optional_string(params, VertexKeys::PROJECT.litellm_params)?,
|
||||
optional_string(params, VertexKeys::LOCATION.litellm_params)?,
|
||||
))
|
||||
}
|
||||
|
||||
|
|
@ -61,16 +62,28 @@ impl VertexConfig {
|
|||
pub fn location(&self) -> Option<&str> {
|
||||
self.location.as_deref()
|
||||
}
|
||||
|
||||
fn keys(&self) -> VertexKeysPartial {
|
||||
VertexKeysPartial {
|
||||
credentials: self
|
||||
.credentials
|
||||
.as_ref()
|
||||
.map(|credentials| credentials.value().expose().to_string()),
|
||||
project: self.project_id.clone(),
|
||||
location: self.location.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn resolved(&self, env_lookup: &dyn Fn(&str) -> Option<String>) -> VertexKeysPartial {
|
||||
merge([self.keys(), env_keys(env_lookup)])
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_vertex_ai_project(
|
||||
config: &VertexConfig,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Option<String> {
|
||||
config
|
||||
.project_id()
|
||||
.map(str::to_string)
|
||||
.or_else(|| VertexParams::PROJECT.resolve(&|_| None, env_lookup))
|
||||
config.resolved(env_lookup).project
|
||||
}
|
||||
|
||||
/// The project a request is billed to when nothing names it: the `project_id` inside the
|
||||
|
|
@ -104,10 +117,11 @@ pub fn get_vertex_ai_location(
|
|||
config: &VertexConfig,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Option<String> {
|
||||
config
|
||||
.location()
|
||||
.map(str::to_string)
|
||||
.or_else(|| VertexParams::LOCATION.resolve(&|_| None, env_lookup))
|
||||
config.resolved(env_lookup).location
|
||||
}
|
||||
|
||||
fn env_keys(env_lookup: &dyn Fn(&str) -> Option<String>) -> VertexKeysPartial {
|
||||
VertexKeysPartial::from_env(&env_lookup).unwrap_or_default()
|
||||
}
|
||||
|
||||
const INLINE_JSON_LEADING_NOISE: [char; 7] = [' ', '\t', '\r', '\n', '\u{feff}', '"', '\''];
|
||||
|
|
@ -156,7 +170,7 @@ pub(crate) fn credential_source(
|
|||
}
|
||||
};
|
||||
}
|
||||
if let Some(configured) = VertexParams::CREDENTIALS.resolve(&|_| None, env_lookup) {
|
||||
if let Some(configured) = config.resolved(env_lookup).credentials {
|
||||
return CredentialSource::Trusted(SecretValue::new(configured));
|
||||
}
|
||||
non_empty_env(env_lookup, GOOGLE_APPLICATION_CREDENTIALS_ENV)
|
||||
|
|
|
|||
|
|
@ -1,14 +1,14 @@
|
|||
use std::sync::LazyLock;
|
||||
|
||||
use litellm_auth_types::VertexParams;
|
||||
use litellm_auth_types::VertexKeys;
|
||||
|
||||
pub const CLOUD_PLATFORM_SCOPE: &str = "https://www.googleapis.com/auth/cloud-platform";
|
||||
pub const GOOGLE_OAUTH_TOKEN_ENDPOINT: &str = "https://oauth2.googleapis.com/token";
|
||||
pub const GOOGLE_APPLICATION_CREDENTIALS_ENV: &str = "GOOGLE_APPLICATION_CREDENTIALS";
|
||||
pub const VERTEX_AI_API_KEY_ENV: &str = "VERTEX_AI_API_KEY";
|
||||
pub const VERTEXAI_API_KEY_ENV: &str = "VERTEXAI_API_KEY";
|
||||
pub const VERTEXAI_CREDENTIALS_ENV: &str = VertexParams::CREDENTIALS.env[0];
|
||||
pub const VERTEX_LOCATION_ENV: &str = VertexParams::LOCATION.env[1];
|
||||
pub const VERTEXAI_CREDENTIALS_ENV: &str = VertexKeys::CREDENTIALS.env[0];
|
||||
pub const VERTEX_LOCATION_ENV: &str = VertexKeys::LOCATION.env[1];
|
||||
|
||||
/// Every environment name the Vertex params and the token acquisition read, for hosts
|
||||
/// that resolve secrets up front.
|
||||
|
|
@ -20,11 +20,7 @@ pub fn secret_names() -> &'static [&'static str] {
|
|||
GOOGLE_APPLICATION_CREDENTIALS_ENV,
|
||||
]
|
||||
.into_iter()
|
||||
.chain(
|
||||
VertexParams::SPECS
|
||||
.iter()
|
||||
.flat_map(|spec| spec.env.iter().copied()),
|
||||
)
|
||||
.chain(VertexKeys::ENV_NAMES.iter().copied())
|
||||
.fold(Vec::new(), |mut names, name| {
|
||||
if !names.contains(&name) {
|
||||
names.push(name);
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ license.workspace = true
|
|||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
litellm-layered.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
subtle.workspace = true
|
||||
|
|
|
|||
|
|
@ -19,10 +19,10 @@ pub enum Error {
|
|||
provider: &'static str,
|
||||
environment_variable: &'static str,
|
||||
},
|
||||
#[error("Missing {provider} {} - {}", .spec.setting, .spec.guidance())]
|
||||
#[error("Missing {provider} {} - {}", .spec.label, .spec.guidance())]
|
||||
MissingParam {
|
||||
provider: &'static str,
|
||||
spec: &'static crate::ParamSpec,
|
||||
spec: &'static litellm_layered::Sources,
|
||||
},
|
||||
#[error("Missing {provider} API Base - {guidance}")]
|
||||
MissingApiBase {
|
||||
|
|
@ -108,3 +108,9 @@ impl PartialEq for ErrorSource {
|
|||
}
|
||||
|
||||
impl Eq for ErrorSource {}
|
||||
|
||||
impl Error {
|
||||
pub fn missing(provider: &'static str, spec: &'static litellm_layered::Sources) -> Self {
|
||||
Self::MissingParam { provider, spec }
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -53,7 +53,8 @@ pub use credential::{
|
|||
};
|
||||
pub use error::{Error, ErrorDetail, ErrorSource};
|
||||
pub use http::CredentialPlacement;
|
||||
pub use params::{AwsParams, ParamSpec, VertexParams};
|
||||
pub use litellm_layered::Sources;
|
||||
pub use params::{AwsKeys, AwsKeysPartial, AwsParams, VertexKeys, VertexKeysPartial, VertexParams};
|
||||
pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy};
|
||||
pub use secret::SecretValue;
|
||||
pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle};
|
||||
|
|
|
|||
|
|
@ -1,22 +1,74 @@
|
|||
use std::sync::LazyLock;
|
||||
|
||||
use litellm_layered::Layered;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
use veil::Redact;
|
||||
|
||||
use super::ParamSpec;
|
||||
|
||||
const fn spec(
|
||||
setting: &'static str,
|
||||
wire: &'static [&'static str],
|
||||
env: &'static [&'static str],
|
||||
) -> ParamSpec {
|
||||
ParamSpec {
|
||||
setting,
|
||||
wire,
|
||||
module_global: None,
|
||||
env,
|
||||
}
|
||||
#[derive(Layered)]
|
||||
pub struct AwsKeys {
|
||||
#[layer(
|
||||
skip_blank,
|
||||
litellm_param = "aws_access_key_id",
|
||||
env = "AWS_ACCESS_KEY_ID"
|
||||
)]
|
||||
pub access_key_id: String,
|
||||
#[layer(
|
||||
skip_blank,
|
||||
litellm_param = "aws_secret_access_key",
|
||||
env = "AWS_SECRET_ACCESS_KEY",
|
||||
secret
|
||||
)]
|
||||
pub secret_access_key: String,
|
||||
#[layer(
|
||||
skip_blank,
|
||||
litellm_param = "aws_session_token",
|
||||
env = "AWS_SESSION_TOKEN",
|
||||
secret
|
||||
)]
|
||||
pub session_token: String,
|
||||
#[layer(skip_blank, litellm_param = "aws_region_name", env = ["AWS_REGION_NAME", "AWS_REGION"])]
|
||||
pub region: String,
|
||||
#[layer(
|
||||
skip_blank,
|
||||
litellm_param = "aws_session_name",
|
||||
env = "AWS_SESSION_NAME"
|
||||
)]
|
||||
pub session_name: String,
|
||||
#[layer(
|
||||
skip_blank,
|
||||
litellm_param = "aws_profile_name",
|
||||
env = "AWS_PROFILE_NAME"
|
||||
)]
|
||||
pub profile_name: String,
|
||||
#[layer(skip_blank, litellm_param = "aws_role_name", env = "AWS_ROLE_NAME")]
|
||||
pub role_name: String,
|
||||
#[layer(
|
||||
skip_blank,
|
||||
litellm_param = "aws_web_identity_token",
|
||||
env = "AWS_WEB_IDENTITY_TOKEN",
|
||||
secret
|
||||
)]
|
||||
pub web_identity_token: String,
|
||||
#[layer(
|
||||
skip_blank,
|
||||
litellm_param = "aws_sts_endpoint",
|
||||
env = "AWS_STS_ENDPOINT",
|
||||
label = "STS endpoint"
|
||||
)]
|
||||
pub sts_endpoint: String,
|
||||
#[layer(
|
||||
skip_blank,
|
||||
litellm_param = "aws_external_id",
|
||||
env = "AWS_EXTERNAL_ID",
|
||||
secret
|
||||
)]
|
||||
pub external_id: String,
|
||||
#[layer(
|
||||
skip_blank,
|
||||
litellm_param = "aws_bedrock_runtime_endpoint",
|
||||
env = "AWS_BEDROCK_RUNTIME_ENDPOINT",
|
||||
label = "Bedrock runtime endpoint"
|
||||
)]
|
||||
pub bedrock_runtime_endpoint: String,
|
||||
}
|
||||
|
||||
/// The `aws_*` fields of Python's `GenericLiteLLMParams`.
|
||||
|
|
@ -51,102 +103,8 @@ pub struct AwsParams {
|
|||
}
|
||||
|
||||
impl AwsParams {
|
||||
pub const ACCESS_KEY_ID: ParamSpec = spec(
|
||||
"access key id",
|
||||
&["aws_access_key_id"],
|
||||
&["AWS_ACCESS_KEY_ID"],
|
||||
);
|
||||
pub const SECRET_ACCESS_KEY: ParamSpec = spec(
|
||||
"secret access key",
|
||||
&["aws_secret_access_key"],
|
||||
&["AWS_SECRET_ACCESS_KEY"],
|
||||
);
|
||||
pub const SESSION_TOKEN: ParamSpec = spec(
|
||||
"session token",
|
||||
&["aws_session_token"],
|
||||
&["AWS_SESSION_TOKEN"],
|
||||
);
|
||||
pub const REGION: ParamSpec = spec(
|
||||
"region",
|
||||
&["aws_region_name"],
|
||||
&["AWS_REGION_NAME", "AWS_REGION"],
|
||||
);
|
||||
pub const SESSION_NAME: ParamSpec =
|
||||
spec("session name", &["aws_session_name"], &["AWS_SESSION_NAME"]);
|
||||
pub const PROFILE_NAME: ParamSpec =
|
||||
spec("profile name", &["aws_profile_name"], &["AWS_PROFILE_NAME"]);
|
||||
pub const ROLE_NAME: ParamSpec = spec("role name", &["aws_role_name"], &["AWS_ROLE_NAME"]);
|
||||
pub const WEB_IDENTITY_TOKEN: ParamSpec = spec(
|
||||
"web identity token",
|
||||
&["aws_web_identity_token"],
|
||||
&["AWS_WEB_IDENTITY_TOKEN"],
|
||||
);
|
||||
pub const STS_ENDPOINT: ParamSpec =
|
||||
spec("STS endpoint", &["aws_sts_endpoint"], &["AWS_STS_ENDPOINT"]);
|
||||
pub const EXTERNAL_ID: ParamSpec =
|
||||
spec("external id", &["aws_external_id"], &["AWS_EXTERNAL_ID"]);
|
||||
pub const BEDROCK_RUNTIME_ENDPOINT: ParamSpec = spec(
|
||||
"Bedrock runtime endpoint",
|
||||
&["aws_bedrock_runtime_endpoint"],
|
||||
&["AWS_BEDROCK_RUNTIME_ENDPOINT"],
|
||||
);
|
||||
|
||||
/// Every `aws_*` param Python reads, in declaration order, with the environment names
|
||||
/// each falls back to.
|
||||
pub const SPECS: [ParamSpec; 11] = [
|
||||
Self::ACCESS_KEY_ID,
|
||||
Self::SECRET_ACCESS_KEY,
|
||||
Self::SESSION_TOKEN,
|
||||
Self::REGION,
|
||||
Self::SESSION_NAME,
|
||||
Self::PROFILE_NAME,
|
||||
Self::ROLE_NAME,
|
||||
Self::WEB_IDENTITY_TOKEN,
|
||||
Self::STS_ENDPOINT,
|
||||
Self::EXTERNAL_ID,
|
||||
Self::BEDROCK_RUNTIME_ENDPOINT,
|
||||
];
|
||||
|
||||
/// The wire names a host projects out of a caller's kwargs, derived from [`Self::SPECS`].
|
||||
pub fn fields() -> impl Iterator<Item = &'static str> {
|
||||
Self::SPECS
|
||||
.iter()
|
||||
.flat_map(|spec| spec.wire.iter().copied())
|
||||
}
|
||||
|
||||
/// The environment names the specs read, for hosts that resolve secrets up front.
|
||||
pub fn secret_names() -> &'static [&'static str] {
|
||||
static NAMES: LazyLock<Vec<&'static str>> = LazyLock::new(|| {
|
||||
AwsParams::SPECS
|
||||
.iter()
|
||||
.flat_map(|spec| spec.env.iter().copied())
|
||||
.fold(Vec::new(), |mut names, name| {
|
||||
if !names.contains(&name) {
|
||||
names.push(name);
|
||||
}
|
||||
names
|
||||
})
|
||||
});
|
||||
&NAMES
|
||||
}
|
||||
|
||||
/// The value under a wire name, read through serde so the names can never drift from
|
||||
/// the struct.
|
||||
pub fn get(&self, wire: &str) -> Option<String> {
|
||||
serde_json::to_value(self)
|
||||
.ok()?
|
||||
.get(wire)?
|
||||
.as_str()
|
||||
.map(str::to_string)
|
||||
}
|
||||
|
||||
/// The spec's value from these params, then the environment.
|
||||
pub fn resolve(
|
||||
&self,
|
||||
spec: &ParamSpec,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Option<String> {
|
||||
spec.resolve(&|name| self.get(name), env_lookup)
|
||||
pub fn keys(&self) -> AwsKeysPartial {
|
||||
AwsKeysPartial::from_litellm_params(&super::as_litellm_params(self))
|
||||
}
|
||||
|
||||
/// Reads the string-valued `aws_*` keys of an untyped params map, ignoring anything else.
|
||||
|
|
|
|||
|
|
@ -1,48 +1,15 @@
|
|||
mod aws;
|
||||
mod vertex;
|
||||
|
||||
pub use aws::AwsParams;
|
||||
pub use vertex::VertexParams;
|
||||
pub use aws::{AwsKeys, AwsKeysPartial, AwsParams};
|
||||
pub use vertex::{VertexKeys, VertexKeysPartial, VertexParams};
|
||||
|
||||
/// One connection param of Python's `CredentialLiteLLMParams` and every place Python reads
|
||||
/// it from, in order: the wire names on a call or a deployment, the `litellm.<name>` module
|
||||
/// global a host folds in, then the environment names.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct ParamSpec {
|
||||
pub setting: &'static str,
|
||||
pub wire: &'static [&'static str],
|
||||
pub module_global: Option<&'static str>,
|
||||
pub env: &'static [&'static str],
|
||||
}
|
||||
use serde::Serialize;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
impl ParamSpec {
|
||||
/// The first non-blank value: each wire name through `params`, then each environment
|
||||
/// name through `env`.
|
||||
pub fn resolve(
|
||||
&self,
|
||||
params: &dyn Fn(&str) -> Option<String>,
|
||||
env: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Option<String> {
|
||||
self.wire
|
||||
.iter()
|
||||
.map(|name| params(name))
|
||||
.chain(self.env.iter().map(|name| env(name)))
|
||||
.find_map(|value| value.filter(|value| !value.trim().is_empty()))
|
||||
}
|
||||
|
||||
pub fn missing(&'static self, provider: &'static str) -> crate::Error {
|
||||
crate::Error::MissingParam {
|
||||
provider,
|
||||
spec: self,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn guidance(&self) -> String {
|
||||
let wire = self.wire.join(" or ");
|
||||
let env = self.env.join(" or ");
|
||||
match self.module_global {
|
||||
Some(global) => format!("pass {wire}, set litellm.{global}, or set {env}"),
|
||||
None => format!("pass {wire} or set {env}"),
|
||||
}
|
||||
fn as_litellm_params(params: &impl Serialize) -> Map<String, Value> {
|
||||
match serde_json::to_value(params) {
|
||||
Ok(Value::Object(map)) => map,
|
||||
_ => Map::new(),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,30 @@
|
|||
use litellm_layered::Layered;
|
||||
use serde::{Deserialize, Deserializer, Serialize, de::Unexpected};
|
||||
use serde_json::Value;
|
||||
use veil::Redact;
|
||||
|
||||
use super::ParamSpec;
|
||||
#[derive(Layered)]
|
||||
pub struct VertexKeys {
|
||||
#[layer(skip_blank,
|
||||
litellm_param = ["vertex_credentials", "vertex_ai_credentials"],
|
||||
deserialize_with = optional_credential_text,
|
||||
env = "VERTEXAI_CREDENTIALS",
|
||||
secret
|
||||
)]
|
||||
pub credentials: String,
|
||||
#[layer(skip_blank,
|
||||
litellm_param = ["vertex_project", "vertex_ai_project"],
|
||||
litellm_setting = "vertex_project",
|
||||
env = "VERTEXAI_PROJECT",
|
||||
)]
|
||||
pub project: String,
|
||||
#[layer(skip_blank,
|
||||
litellm_param = ["vertex_location", "vertex_ai_location"],
|
||||
litellm_setting = "vertex_location",
|
||||
env = ["VERTEXAI_LOCATION", "VERTEX_LOCATION"],
|
||||
)]
|
||||
pub location: String,
|
||||
}
|
||||
|
||||
/// The Vertex AI fields of Python's `GenericLiteLLMParams`, both the current names and the
|
||||
/// `vertex_ai_*` spellings `VertexBase.safe_get_vertex_ai_*` still read.
|
||||
|
|
@ -25,65 +47,20 @@ pub struct VertexParams {
|
|||
}
|
||||
|
||||
impl VertexParams {
|
||||
pub const CREDENTIALS: ParamSpec = ParamSpec {
|
||||
setting: "credentials",
|
||||
wire: &["vertex_credentials", "vertex_ai_credentials"],
|
||||
module_global: None,
|
||||
env: &["VERTEXAI_CREDENTIALS"],
|
||||
};
|
||||
pub const PROJECT: ParamSpec = ParamSpec {
|
||||
setting: "project",
|
||||
wire: &["vertex_project", "vertex_ai_project"],
|
||||
module_global: Some("vertex_project"),
|
||||
env: &["VERTEXAI_PROJECT"],
|
||||
};
|
||||
pub const LOCATION: ParamSpec = ParamSpec {
|
||||
setting: "location",
|
||||
wire: &["vertex_location", "vertex_ai_location"],
|
||||
module_global: Some("vertex_location"),
|
||||
env: &["VERTEXAI_LOCATION", "VERTEX_LOCATION"],
|
||||
};
|
||||
|
||||
/// Every Vertex AI param Python reads, with both spellings, the module global
|
||||
/// `VertexBase.get_vertex_ai_*` consults, and the environment names it falls back to.
|
||||
pub const SPECS: [ParamSpec; 3] = [Self::CREDENTIALS, Self::PROJECT, Self::LOCATION];
|
||||
|
||||
/// The wire names a host projects out of a caller's kwargs, derived from [`Self::SPECS`].
|
||||
pub fn fields() -> impl Iterator<Item = &'static str> {
|
||||
Self::SPECS
|
||||
.iter()
|
||||
.flat_map(|spec| spec.wire.iter().copied())
|
||||
}
|
||||
|
||||
/// The value under a wire name, read through serde so the names can never drift from
|
||||
/// the struct.
|
||||
pub fn get(&self, wire: &str) -> Option<String> {
|
||||
serde_json::to_value(self)
|
||||
.ok()?
|
||||
.get(wire)?
|
||||
.as_str()
|
||||
.map(str::to_string)
|
||||
}
|
||||
|
||||
/// The spec's value from these params, then the environment.
|
||||
pub fn resolve(
|
||||
&self,
|
||||
spec: &ParamSpec,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Option<String> {
|
||||
spec.resolve(&|name| self.get(name), env_lookup)
|
||||
pub fn keys(&self) -> VertexKeysPartial {
|
||||
VertexKeysPartial::from_litellm_params(&super::as_litellm_params(self))
|
||||
}
|
||||
|
||||
pub fn credentials(&self) -> Option<String> {
|
||||
self.resolve(&Self::CREDENTIALS, &|_| None)
|
||||
self.keys().credentials
|
||||
}
|
||||
|
||||
pub fn project(&self) -> Option<String> {
|
||||
self.resolve(&Self::PROJECT, &|_| None)
|
||||
self.keys().project
|
||||
}
|
||||
|
||||
pub fn location(&self) -> Option<String> {
|
||||
self.resolve(&Self::LOCATION, &|_| None)
|
||||
self.keys().location
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,26 +1,27 @@
|
|||
use litellm_auth_types::AwsParams;
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use litellm_auth_types::{AwsKeys, AwsParams};
|
||||
use rstest::rstest;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
#[rstest]
|
||||
fn fields_name_every_param_once_and_in_declaration_order() {
|
||||
let filled: Map<String, Value> = AwsParams::fields()
|
||||
fn fields_name_every_param_once() {
|
||||
let filled: Map<String, Value> = AwsKeys::LITELLM_PARAMS
|
||||
.iter()
|
||||
.map(|name| (name.to_string(), Value::from(format!("value-of-{name}"))))
|
||||
.collect();
|
||||
let typed = AwsParams::from_optional_params(&filled);
|
||||
let serialized = serde_json::to_value(&typed).unwrap();
|
||||
assert_eq!(
|
||||
serialized
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.keys()
|
||||
.map(String::as_str)
|
||||
.collect::<Vec<_>>(),
|
||||
AwsParams::fields().collect::<Vec<_>>()
|
||||
);
|
||||
assert_eq!(serialized, Value::Object(filled.clone()));
|
||||
assert_eq!(
|
||||
serde_json::from_value::<AwsParams>(Value::Object(filled)).unwrap(),
|
||||
typed
|
||||
);
|
||||
assert_eq!(
|
||||
AwsKeys::LITELLM_PARAMS
|
||||
.iter()
|
||||
.collect::<BTreeSet<_>>()
|
||||
.len(),
|
||||
AwsKeys::LITELLM_PARAMS.len()
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,20 +1,16 @@
|
|||
use litellm_auth_types::{AwsParams, ParamSpec};
|
||||
use litellm_auth_types::{AwsKeys, Error, Sources, VertexKeys, VertexKeysPartial, VertexParams};
|
||||
use litellm_layered::{Env, merge};
|
||||
use rstest::rstest;
|
||||
|
||||
const LOCATION: ParamSpec = ParamSpec {
|
||||
setting: "location",
|
||||
wire: &["vertex_location", "vertex_ai_location"],
|
||||
module_global: Some("vertex_location"),
|
||||
env: &["VERTEXAI_LOCATION", "VERTEX_LOCATION"],
|
||||
};
|
||||
|
||||
fn lookup(entries: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
|
||||
move |name| {
|
||||
fn vertex_params(entries: &[(&str, &str)]) -> VertexParams {
|
||||
serde_json::from_value(
|
||||
entries
|
||||
.iter()
|
||||
.find(|(key, _)| *key == name)
|
||||
.map(|(_, value)| value.to_string())
|
||||
}
|
||||
.map(|(name, value)| (name.to_string(), serde_json::Value::from(*value)))
|
||||
.collect::<serde_json::Map<_, _>>()
|
||||
.into(),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
|
|
@ -38,35 +34,32 @@ fn lookup(entries: &'static [(&'static str, &'static str)]) -> impl Fn(&str) ->
|
|||
&[("VERTEX_LOCATION", "second"), ("VERTEXAI_LOCATION", "first")],
|
||||
Some("first")
|
||||
)]
|
||||
#[case::values_are_kept_untrimmed(&[("vertex_location", " eu ")], &[], Some(" eu "))]
|
||||
#[case::nothing_set(&[], &[], None)]
|
||||
fn resolution_walks_wire_names_then_environment_names(
|
||||
#[case] params: &'static [(&'static str, &'static str)],
|
||||
#[case] env: &'static [(&'static str, &'static str)],
|
||||
fn params_are_stacked_above_the_environment(
|
||||
#[case] params: &[(&str, &str)],
|
||||
#[case] env: &[(&str, &str)],
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
assert_eq!(
|
||||
LOCATION.resolve(&lookup(params), &lookup(env)).as_deref(),
|
||||
expected
|
||||
);
|
||||
let merged = merge([
|
||||
vertex_params(params).keys(),
|
||||
VertexKeysPartial::from_env(&Env::of(env)).unwrap(),
|
||||
]);
|
||||
assert_eq!(merged.location.as_deref(), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::without_a_module_global(
|
||||
&AwsParams::REGION,
|
||||
#[case::without_a_litellm_setting(
|
||||
AwsKeys::REGION,
|
||||
"Missing AWS region - pass aws_region_name or set AWS_REGION_NAME or AWS_REGION"
|
||||
)]
|
||||
#[case::with_a_module_global(
|
||||
&LOCATION,
|
||||
#[case::with_a_litellm_setting(
|
||||
VertexKeys::LOCATION,
|
||||
"Missing AWS location - pass vertex_location or vertex_ai_location, set litellm.vertex_location, or set VERTEXAI_LOCATION or VERTEX_LOCATION"
|
||||
)]
|
||||
fn a_missing_param_names_every_place_the_spec_reads(
|
||||
#[case] spec: &'static ParamSpec,
|
||||
fn a_missing_param_names_every_source_the_group_reads(
|
||||
#[case] sources: &'static Sources,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
let message = spec.missing("AWS").to_string();
|
||||
|
||||
assert_eq!(message, expected);
|
||||
assert!(spec.wire.iter().all(|name| message.contains(name)));
|
||||
assert!(spec.env.iter().all(|name| message.contains(name)));
|
||||
assert_eq!(message.contains("litellm."), spec.module_global.is_some());
|
||||
assert_eq!(Error::missing("AWS", sources).to_string(), expected);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
use std::collections::BTreeSet;
|
||||
|
||||
use litellm_auth_types::VertexParams;
|
||||
use litellm_auth_types::{VertexKeys, VertexKeysPartial, VertexParams};
|
||||
use litellm_layered::{Env, merge};
|
||||
use rstest::rstest;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
|
|
@ -61,8 +62,9 @@ fn a_param_of_the_wrong_type_is_rejected(#[case] params: Value) {
|
|||
|
||||
#[rstest]
|
||||
fn fields_name_every_param_once() {
|
||||
let filled: Value = VertexParams::fields()
|
||||
.map(|name| (name.to_string(), Value::from(name)))
|
||||
let filled: Value = VertexKeys::LITELLM_PARAMS
|
||||
.iter()
|
||||
.map(|name| (name.to_string(), Value::from(*name)))
|
||||
.collect::<Map<_, _>>()
|
||||
.into();
|
||||
let params: VertexParams = serde_json::from_value(filled).unwrap();
|
||||
|
|
@ -76,8 +78,11 @@ fn fields_name_every_param_once() {
|
|||
};
|
||||
assert_eq!(params, expected);
|
||||
assert_eq!(
|
||||
VertexParams::fields().collect::<BTreeSet<_>>().len(),
|
||||
VertexParams::fields().count()
|
||||
VertexKeys::LITELLM_PARAMS
|
||||
.iter()
|
||||
.collect::<BTreeSet<_>>()
|
||||
.len(),
|
||||
VertexKeys::LITELLM_PARAMS.len()
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -101,7 +106,7 @@ fn debug_output_does_not_expose_credentials() {
|
|||
#[case::legacy_spelling(None, Some("legacy"), &[("VERTEXAI_PROJECT", "env")], Some("legacy"))]
|
||||
#[case::blank_params_fall_to_the_environment(Some(" "), None, &[("VERTEXAI_PROJECT", "env")], Some("env"))]
|
||||
#[case::nothing(None, None, &[], None)]
|
||||
fn a_spec_resolves_from_the_params_then_the_environment(
|
||||
fn a_setting_resolves_from_the_params_then_the_environment(
|
||||
#[case] vertex_project: Option<&str>,
|
||||
#[case] vertex_ai_project: Option<&str>,
|
||||
#[case] environment: &[(&str, &str)],
|
||||
|
|
@ -112,15 +117,10 @@ fn a_spec_resolves_from_the_params_then_the_environment(
|
|||
vertex_ai_project: vertex_ai_project.map(str::to_string),
|
||||
..VertexParams::default()
|
||||
};
|
||||
let env = |name: &str| {
|
||||
environment
|
||||
.iter()
|
||||
.find(|(key, _)| *key == name)
|
||||
.map(|(_, value)| value.to_string())
|
||||
};
|
||||
let merged = merge([
|
||||
params.keys(),
|
||||
VertexKeysPartial::from_env(&Env::of(environment)).unwrap(),
|
||||
]);
|
||||
|
||||
assert_eq!(
|
||||
params.resolve(&VertexParams::PROJECT, &env).as_deref(),
|
||||
expected
|
||||
);
|
||||
assert_eq!(merged.project.as_deref(), expected);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -216,6 +216,8 @@ pub struct LiteLlmSettings {
|
|||
pub aiohttp_trust_env: Option<bool>,
|
||||
pub disable_aiohttp_trust_env: Option<bool>,
|
||||
pub disable_aiohttp_transport: Option<bool>,
|
||||
pub vertex_project: Option<String>,
|
||||
pub vertex_location: Option<String>,
|
||||
pub drop_params: Option<Spelled<bool>>,
|
||||
pub request_timeout: Option<Spelled<f64>>,
|
||||
pub num_retries: Option<u64>,
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ mod auth;
|
|||
mod error;
|
||||
mod http;
|
||||
mod secrets;
|
||||
mod vertex;
|
||||
pub use error::Error;
|
||||
|
||||
use std::{sync::Arc, time::Instant};
|
||||
|
|
@ -50,7 +51,10 @@ pub fn build_inference(config: &Config, process: &Env) -> Result<Arc<Gateway>, E
|
|||
ocr,
|
||||
env,
|
||||
secrets,
|
||||
ModelRouter::from_model_list(&config.model_list),
|
||||
ModelRouter::from_model_list(&vertex::with_written_settings(
|
||||
&config.model_list,
|
||||
&config.litellm_settings,
|
||||
)),
|
||||
)?))
|
||||
}
|
||||
|
||||
|
|
|
|||
89
litellm-rust/crates/gateway/src/vertex.rs
Normal file
89
litellm-rust/crates/gateway/src/vertex.rs
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
use litellm_auth_types::{VertexKeysPartial, VertexParams};
|
||||
use litellm_config::{LiteLlmSettings, LitellmParams, Model};
|
||||
use litellm_layered::merge;
|
||||
|
||||
pub(crate) fn with_written_settings(models: &[Model], settings: &LiteLlmSettings) -> Vec<Model> {
|
||||
let file = written(settings);
|
||||
models
|
||||
.iter()
|
||||
.map(|model| {
|
||||
let keys = merge([model.litellm_params.vertex.keys(), file.clone()]);
|
||||
Model {
|
||||
litellm_params: LitellmParams {
|
||||
vertex: VertexParams {
|
||||
vertex_project: keys.project,
|
||||
vertex_location: keys.location,
|
||||
..model.litellm_params.vertex.clone()
|
||||
},
|
||||
..model.litellm_params.clone()
|
||||
},
|
||||
..model.clone()
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn written(settings: &LiteLlmSettings) -> VertexKeysPartial {
|
||||
VertexKeysPartial {
|
||||
credentials: None,
|
||||
project: settings.vertex_project.clone(),
|
||||
location: settings.vertex_location.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use litellm_auth_types::VertexKeys;
|
||||
use litellm_config::Config;
|
||||
use litellm_http::HttpKeys;
|
||||
use rstest::rstest;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn models(yaml: &str) -> Vec<Model> {
|
||||
let config = Config::from_yaml(yaml).unwrap();
|
||||
with_written_settings(&config.model_list, &config.litellm_settings)
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::file_fills_an_unset_deployment(
|
||||
"model_list:\n - model_name: m\n litellm_params:\n model: vertex_ai/claude\nlitellm_settings:\n vertex_project: file-project\n",
|
||||
Some("file-project")
|
||||
)]
|
||||
#[case::the_deployment_wins(
|
||||
"model_list:\n - model_name: m\n litellm_params:\n model: vertex_ai/claude\n vertex_ai_project: deployment\nlitellm_settings:\n vertex_project: file-project\n",
|
||||
Some("deployment")
|
||||
)]
|
||||
#[case::a_blank_deployment_value_takes_the_file(
|
||||
"model_list:\n - model_name: m\n litellm_params:\n model: vertex_ai/claude\n vertex_project: ' '\nlitellm_settings:\n vertex_project: file-project\n",
|
||||
Some("file-project")
|
||||
)]
|
||||
#[case::neither(
|
||||
"model_list:\n - model_name: m\n litellm_params:\n model: vertex_ai/claude\n",
|
||||
None
|
||||
)]
|
||||
fn deployment_params_are_stacked_above_the_config_file(
|
||||
#[case] yaml: &str,
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
assert_eq!(
|
||||
models(yaml)[0].litellm_params.vertex.project().as_deref(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn every_litellm_setting_a_group_declares_is_a_typed_config_field() {
|
||||
let names = HttpKeys::LITELLM_SETTINGS
|
||||
.iter()
|
||||
.chain(VertexKeys::LITELLM_SETTINGS)
|
||||
.map(|(name, _)| ((*name).to_owned(), Value::Null))
|
||||
.collect::<Map<_, _>>();
|
||||
let settings: LiteLlmSettings = serde_json::from_value(Value::Object(names)).unwrap();
|
||||
assert_eq!(
|
||||
settings.additional_fields.keys().collect::<Vec<_>>(),
|
||||
Vec::<&String>::new()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -233,8 +233,19 @@ environment_variables:
|
|||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::project_on_the_deployment(
|
||||
" vertex_project: test-project\n vertex_location: us-east5\n",
|
||||
""
|
||||
)]
|
||||
#[case::project_only_in_litellm_settings(
|
||||
"",
|
||||
"litellm_settings:\n vertex_project: test-project\n vertex_location: us-east5\n"
|
||||
)]
|
||||
#[tokio::test]
|
||||
async fn messages_auth_reads_credentials_from_config_environment_variables() {
|
||||
async fn messages_auth_reads_credentials_from_config_environment_variables(
|
||||
#[case] deployment: &str,
|
||||
#[case] litellm_settings: &str,
|
||||
) {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let credentials = directory.path().join("service-account.json");
|
||||
std::fs::write(&credentials, "not a service account").unwrap();
|
||||
|
|
@ -243,13 +254,11 @@ async fn messages_auth_reads_credentials_from_config_environment_variables() {
|
|||
- model_name: public/vertex
|
||||
litellm_params:
|
||||
model: vertex_ai/claude-sonnet-4-5
|
||||
vertex_project: test-project
|
||||
vertex_location: us-east5
|
||||
general_settings:
|
||||
{deployment}general_settings:
|
||||
master_key: gateway-key
|
||||
environment_variables:
|
||||
GOOGLE_APPLICATION_CREDENTIALS: '{}'
|
||||
",
|
||||
{litellm_settings}",
|
||||
credentials.display()
|
||||
))
|
||||
.unwrap();
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ impl BaseOcrConfig for TextractAnalyzeDocumentConfig {
|
|||
type Environment = TextractEnvironment;
|
||||
|
||||
fn secret_names(&self) -> Vec<&'static str> {
|
||||
litellm_auth::AwsParams::secret_names().to_vec()
|
||||
litellm_auth::AwsKeys::ENV_NAMES.to_vec()
|
||||
}
|
||||
|
||||
fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] {
|
||||
|
|
|
|||
|
|
@ -239,8 +239,12 @@ pub(super) async fn environment(
|
|||
) -> Result<TextractEnvironment, Error> {
|
||||
let env_lookup = |name: &str| request.connection.secret(name);
|
||||
let params = AwsParams::from_optional_params(&request.optional_params);
|
||||
let region = resolve_aws_region(None, ¶ms, &env_lookup)
|
||||
.ok_or_else(|| Error::Auth(AwsParams::REGION.missing("AWS")))?;
|
||||
let region = resolve_aws_region(None, ¶ms, &env_lookup).ok_or_else(|| {
|
||||
Error::Auth(litellm_auth::Error::missing(
|
||||
"AWS",
|
||||
litellm_auth::AwsKeys::REGION,
|
||||
))
|
||||
})?;
|
||||
let signer = SigV4Signer::resolve(
|
||||
auth,
|
||||
region.clone(),
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ impl BaseOcrConfig for TextractDetectTextConfig {
|
|||
type Environment = TextractEnvironment;
|
||||
|
||||
fn secret_names(&self) -> Vec<&'static str> {
|
||||
litellm_auth::AwsParams::secret_names().to_vec()
|
||||
litellm_auth::AwsKeys::ENV_NAMES.to_vec()
|
||||
}
|
||||
|
||||
fn get_health_check_document(&self) -> OcrDocument {
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ fn optional_string<'a>(params: &'a Map<String, Value>, key: &str) -> Option<&'a
|
|||
|
||||
impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
|
||||
fn secret_names(&self) -> Vec<&'static str> {
|
||||
litellm_auth::AwsParams::secret_names().to_vec()
|
||||
litellm_auth::AwsKeys::ENV_NAMES.to_vec()
|
||||
}
|
||||
|
||||
fn get_supported_openai_params(&self) -> &'static [&'static str] {
|
||||
|
|
|
|||
|
|
@ -94,7 +94,7 @@ pub const BEDROCK_CHAT_COMPLETIONS_CONFIG: AmazonConverseConfig = AmazonConverse
|
|||
|
||||
impl BaseConfig for AmazonConverseConfig {
|
||||
fn secret_names(&self) -> Vec<&'static str> {
|
||||
litellm_auth::AwsParams::secret_names()
|
||||
litellm_auth::AwsKeys::ENV_NAMES
|
||||
.iter()
|
||||
.copied()
|
||||
.chain([AWS_BEARER_TOKEN_BEDROCK])
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
use std::sync::LazyLock;
|
||||
|
||||
use litellm_auth::{AwsParams, CredentialPlacement, ParamSpec, SecretValue};
|
||||
use litellm_auth::{CredentialPlacement, SecretValue};
|
||||
use litellm_auth_aws::{AwsCredentialSource, resolve_aws_region};
|
||||
use litellm_core_utils::settings::resolve_non_empty;
|
||||
use litellm_http::request::{header_value, with_header, without_headers};
|
||||
use litellm_layered::Layered;
|
||||
use litellm_llms_types::{
|
||||
formats::messages::{MessagesOptionalParams, MessagesRequest},
|
||||
providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
|
||||
|
|
@ -26,20 +27,26 @@ use crate::{
|
|||
|
||||
const SERVICE: &str = "aws-external-anthropic";
|
||||
const BASE_ENVS: &[&str] = &["ANTHROPIC_AWS_BASE_URL", "ANTHROPIC_AWS_API_BASE"];
|
||||
const WORKSPACE_ENVS: &[&str] = &["ANTHROPIC_AWS_WORKSPACE_ID", "ANTHROPIC_WORKSPACE_ID"];
|
||||
const WORKSPACE_FIELDS: &[&str] = &[
|
||||
"workspace_id",
|
||||
"aws_workspace_id",
|
||||
"anthropic-workspace-id",
|
||||
"anthropic_workspace_id",
|
||||
];
|
||||
const WORKSPACE_ENVS: &[&str] = WorkspaceKeys::ENV_NAMES;
|
||||
const WORKSPACE_FIELDS: &[&str] = WorkspaceKeys::LITELLM_PARAMS;
|
||||
const WORKSPACE_HEADER: &str = "anthropic-workspace-id";
|
||||
static WORKSPACE: ParamSpec = ParamSpec {
|
||||
setting: "workspace_id",
|
||||
wire: WORKSPACE_FIELDS,
|
||||
module_global: None,
|
||||
env: WORKSPACE_ENVS,
|
||||
};
|
||||
|
||||
#[derive(Layered)]
|
||||
#[expect(dead_code, reason = "only WorkspaceKeysPartial is built")]
|
||||
struct WorkspaceKeys {
|
||||
#[layer(
|
||||
skip_blank,
|
||||
litellm_param = [
|
||||
"workspace_id",
|
||||
"aws_workspace_id",
|
||||
"anthropic-workspace-id",
|
||||
"anthropic_workspace_id",
|
||||
],
|
||||
env = ["ANTHROPIC_AWS_WORKSPACE_ID", "ANTHROPIC_WORKSPACE_ID"],
|
||||
label = "workspace_id"
|
||||
)]
|
||||
workspace_id: String,
|
||||
}
|
||||
const UNSUPPORTED_OVERRIDE: &str = "claude_platform_unsupported_params";
|
||||
|
||||
pub struct BedrockClaudePlatformMessagesConfig;
|
||||
|
|
@ -72,8 +79,12 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
|
|||
.or_else(|| workspace_value(params.extra.get(*field)))
|
||||
})
|
||||
});
|
||||
let workspace = resolve_non_empty(workspace, env, WORKSPACE_ENVS)
|
||||
.ok_or_else(|| Error::Auth(WORKSPACE.missing("Claude Platform on AWS")))?;
|
||||
let workspace = resolve_non_empty(workspace, env, WORKSPACE_ENVS).ok_or_else(|| {
|
||||
Error::Auth(litellm_auth::Error::missing(
|
||||
"Claude Platform on AWS",
|
||||
WorkspaceKeys::WORKSPACE_ID,
|
||||
))
|
||||
})?;
|
||||
let headers = with_header(headers, WORKSPACE_HEADER, workspace);
|
||||
let key = resolve_non_empty(api_key, env, &["ANTHROPIC_AWS_API_KEY"]);
|
||||
let headers = without_headers(headers, &["authorization"]);
|
||||
|
|
@ -150,7 +161,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
|
|||
.chain(BASE_ENVS.iter().copied())
|
||||
.chain(WORKSPACE_ENVS.iter().copied())
|
||||
.chain(["AWS_DEFAULT_REGION"])
|
||||
.chain(AwsParams::secret_names().iter().copied())
|
||||
.chain(litellm_auth::AwsKeys::ENV_NAMES.iter().copied())
|
||||
.collect()
|
||||
});
|
||||
&NAMES
|
||||
|
|
@ -199,7 +210,12 @@ fn platform_region(
|
|||
let region = host_region
|
||||
.or_else(|| resolve_aws_region(None, ¶ms.aws, env))
|
||||
.or_else(|| resolve_non_empty(None, env, &["AWS_DEFAULT_REGION"]))
|
||||
.ok_or_else(|| Error::Auth(AwsParams::REGION.missing("Claude Platform on AWS")))?;
|
||||
.ok_or_else(|| {
|
||||
Error::Auth(litellm_auth::Error::missing(
|
||||
"Claude Platform on AWS",
|
||||
litellm_auth::AwsKeys::REGION,
|
||||
))
|
||||
})?;
|
||||
validated_region(region)
|
||||
}
|
||||
|
||||
|
|
@ -250,6 +266,7 @@ fn invalid_request(error: serde_json::Error) -> Error {
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use litellm_auth::AwsParams;
|
||||
use litellm_llms_types::providers::anthropic::AnthropicBeta;
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
|
@ -267,7 +284,10 @@ mod tests {
|
|||
&|_| None
|
||||
)
|
||||
.unwrap_err(),
|
||||
Error::Auth(WORKSPACE.missing("Claude Platform on AWS"))
|
||||
Error::Auth(litellm_auth::Error::missing(
|
||||
"Claude Platform on AWS",
|
||||
WorkspaceKeys::WORKSPACE_ID,
|
||||
))
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
use std::sync::LazyLock;
|
||||
|
||||
use litellm_auth::{AwsParams, CredentialPlacement, SecretValue};
|
||||
use litellm_auth::{CredentialPlacement, SecretValue};
|
||||
use litellm_auth_aws::AwsCredentialSource;
|
||||
use litellm_auth_aws::bedrock_model_id_and_region;
|
||||
use litellm_core_utils::settings::resolve_non_empty;
|
||||
|
|
@ -164,7 +164,7 @@ impl BaseMessagesConfig for BedrockMantleAnthropicMessagesConfig {
|
|||
.into_iter()
|
||||
.chain(REGION_ENVS.iter().copied())
|
||||
.chain(
|
||||
AwsParams::secret_names()
|
||||
litellm_auth::AwsKeys::ENV_NAMES
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|name| !REGION_ENVS.contains(name)),
|
||||
|
|
@ -263,6 +263,7 @@ fn mantle_region(
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use litellm_auth::AwsParams;
|
||||
use litellm_http::request::header_value;
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
use litellm_auth::VertexParams;
|
||||
use litellm_auth_gcp::{
|
||||
VertexConfig, get_vertex_ai_location, get_vertex_ai_project,
|
||||
get_vertex_ai_project_from_credentials, secret_names,
|
||||
|
|
@ -64,7 +63,12 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
|
|||
let config = VertexConfig::from_params(&litellm_params.vertex);
|
||||
let project = get_vertex_ai_project(&config, env_lookup)
|
||||
.or_else(|| get_vertex_ai_project_from_credentials(&config, env_lookup))
|
||||
.ok_or_else(|| Error::Auth(VertexParams::PROJECT.missing("Vertex AI")))?;
|
||||
.ok_or_else(|| {
|
||||
Error::Auth(litellm_auth::Error::missing(
|
||||
"Vertex AI",
|
||||
litellm_auth::VertexKeys::PROJECT,
|
||||
))
|
||||
})?;
|
||||
let location = get_vertex_ai_location(&config, env_lookup)
|
||||
.unwrap_or_else(|| DEFAULT_LOCATION.to_string());
|
||||
complete_vertex_anthropic_url(api_base, &project, &location, model, stream)
|
||||
|
|
@ -227,6 +231,7 @@ fn vertex_feature_betas(request: &MessagesRequest) -> BetaSet {
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use litellm_auth::VertexParams;
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
|
|
@ -422,7 +427,7 @@ mod tests {
|
|||
error,
|
||||
Error::Auth(litellm_auth::Error::MissingParam {
|
||||
provider: "Vertex AI",
|
||||
spec: &VertexParams::PROJECT,
|
||||
spec: litellm_auth::VertexKeys::PROJECT,
|
||||
})
|
||||
));
|
||||
assert_eq!(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,74 @@
|
|||
use litellm_auth::Sources;
|
||||
use pyo3::prelude::*;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::{
|
||||
coercion::FieldSpec,
|
||||
python_settings::{PythonSettings, Snapshot},
|
||||
};
|
||||
|
||||
/// The settings among `sources` that have a `litellm.*` global and that `params` leaves unset,
|
||||
/// counting a blank value as unset the way Python's `or` chain does.
|
||||
pub(crate) fn unset<'s>(
|
||||
params: &Map<String, Value>,
|
||||
sources: impl IntoIterator<Item = &'s Sources>,
|
||||
) -> Vec<&'s Sources> {
|
||||
sources
|
||||
.into_iter()
|
||||
.filter(|sources| sources.litellm_setting.is_some())
|
||||
.filter(|sources| {
|
||||
!sources
|
||||
.litellm_params
|
||||
.iter()
|
||||
.any(|name| params.get(*name).is_some_and(is_set))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn is_set(value: &Value) -> bool {
|
||||
match value {
|
||||
Value::Null => false,
|
||||
Value::String(text) => !text.trim().is_empty(),
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads the globals of `unset` through the `ProviderDefaults` accessor and places each truthy
|
||||
/// one under its setting's first param name, replacing a blank value the call passed.
|
||||
pub(crate) fn fold(
|
||||
py: Python<'_>,
|
||||
params: Map<String, Value>,
|
||||
unset: &[&Sources],
|
||||
) -> PyResult<Map<String, Value>> {
|
||||
if unset.is_empty() {
|
||||
return Ok(params);
|
||||
}
|
||||
let Some(defaults) = PythonSettings::ProviderDefaults.read_or_unset(py)? else {
|
||||
return Ok(params);
|
||||
};
|
||||
fold_from(params, unset, &defaults)
|
||||
}
|
||||
|
||||
pub(crate) fn fold_from(
|
||||
params: Map<String, Value>,
|
||||
unset: &[&Sources],
|
||||
defaults: &Snapshot<'_>,
|
||||
) -> PyResult<Map<String, Value>> {
|
||||
let globals = unset
|
||||
.iter()
|
||||
.filter_map(|sources| {
|
||||
let setting = sources.litellm_setting?;
|
||||
let name = sources.litellm_params.first()?;
|
||||
let spec = FieldSpec::new(setting, |field| field.falsy_optional_string());
|
||||
Some(
|
||||
defaults
|
||||
.read(&spec)
|
||||
.map(|value| value.map(|value| (name.to_string(), Value::String(value)))),
|
||||
)
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
Ok(params
|
||||
.into_iter()
|
||||
.chain(globals.into_iter().flatten())
|
||||
.collect())
|
||||
}
|
||||
|
|
@ -2,6 +2,7 @@ use crate::cache::{CacheCall, Cached, PythonCache, Selection};
|
|||
use litellm_host_python::{PythonHostCalls, PythonOwned};
|
||||
|
||||
use bytes::Bytes;
|
||||
use litellm_auth::Sources;
|
||||
use litellm_host_python::{InvokeError, PythonBinding, from_py, to_py};
|
||||
use litellm_http::transport::Error as TransportError;
|
||||
use litellm_inference_messages::{
|
||||
|
|
@ -22,8 +23,7 @@ use serde_json::{Map, Value};
|
|||
use crate::{
|
||||
errors::{RustUpstreamError, route_error_to_pyerr},
|
||||
marshal::{optional_timeout, project_optional_fields, public_response, python_timeout_seconds},
|
||||
python_settings::missing_module,
|
||||
routes::RequestView,
|
||||
routes::{RequestView, credential_globals},
|
||||
};
|
||||
|
||||
const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.messages";
|
||||
|
|
@ -41,31 +41,16 @@ fn merge_headers(
|
|||
(!merged.is_empty()).then_some(merged)
|
||||
}
|
||||
|
||||
/// Reads `litellm.<name>`, the module global a param spec names, the way
|
||||
/// `VertexBase.get_vertex_ai_project` reads `litellm.vertex_project`.
|
||||
fn module_global<'py>(py: Python<'py>, name: &str) -> PyResult<Option<Bound<'py, PyAny>>> {
|
||||
let module = match py.import("litellm") {
|
||||
Ok(module) => module,
|
||||
Err(error) if missing_module(py, &error, "litellm")? => return Ok(None),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let value = module.getattr(name)?;
|
||||
Ok((!value.is_none()).then_some(value))
|
||||
}
|
||||
|
||||
/// The caller's litellm params, read from the kwargs by the names the typed params declare,
|
||||
/// so a key the configs do not read is never converted; a spec whose spellings the call all
|
||||
/// leaves out falls back to the module global it names, which stays the host's concern.
|
||||
/// so a key the configs do not read is never converted. A setting the call leaves unset or
|
||||
/// blank takes its `litellm.*` global, as Python's `or` chain does.
|
||||
fn project_litellm_params<'py>(
|
||||
argument: impl Fn(&str) -> PyResult<Option<Bound<'py, PyAny>>>,
|
||||
global: impl Fn(&str) -> PyResult<Option<Bound<'py, PyAny>>>,
|
||||
fold: impl Fn(Map<String, Value>, &[&Sources]) -> PyResult<Map<String, Value>>,
|
||||
) -> PyResult<Result<LitellmParams, Error>> {
|
||||
let fields = project_optional_fields(LitellmParams::fields(), &argument)?;
|
||||
let globals = LitellmParams::specs()
|
||||
.filter(|spec| !spec.wire.iter().any(|name| fields.contains_key(*name)))
|
||||
.filter_map(|spec| spec.module_global);
|
||||
let folded = project_optional_fields(globals, &global)?;
|
||||
Ok(litellm_params(fields.into_iter().chain(folded).collect()))
|
||||
let unset = credential_globals::unset(&fields, LitellmParams::litellm_settings());
|
||||
Ok(litellm_params(fold(fields, &unset)?))
|
||||
}
|
||||
|
||||
fn native_error(py: Python<'_>, error: Error) -> PyResult<PyErr> {
|
||||
|
|
@ -136,7 +121,9 @@ impl MessagesPythonHost {
|
|||
let api_base = string("api_base")?;
|
||||
let extra_headers = self.merged_headers(py, arguments)?;
|
||||
let provider_specific_header = self.provider_specific_header(py, arguments)?;
|
||||
let litellm_params = project_litellm_params(argument, |name| module_global(py, name))?;
|
||||
let litellm_params = project_litellm_params(argument, |fields, unset| {
|
||||
credential_globals::fold(py, fields, unset)
|
||||
})?;
|
||||
Ok(messages_body(body).and_then(|body| {
|
||||
Ok(MessagesCall {
|
||||
body,
|
||||
|
|
@ -336,6 +323,7 @@ mod tests {
|
|||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
use crate::python_settings::PythonSettings;
|
||||
|
||||
fn map(value: Value) -> Map<String, Value> {
|
||||
serde_json::from_value(value).unwrap()
|
||||
|
|
@ -404,7 +392,12 @@ mod tests {
|
|||
json!({"vertex_project": "from-global"}),
|
||||
Ok(json!({"model": "m", "vertex_project": "from-global"})),
|
||||
)]
|
||||
#[case::a_global_of_the_wrong_type_is_a_request_error(
|
||||
#[case::a_blank_value_in_the_call_falls_back(
|
||||
json!({"model": "m", "vertex_project": "", "vertex_ai_project": " "}),
|
||||
json!({"vertex_project": "from-global"}),
|
||||
Ok(json!({"model": "m", "vertex_project": "from-global", "vertex_ai_project": " "})),
|
||||
)]
|
||||
#[case::a_global_of_the_wrong_type_is_an_error(
|
||||
json!({"model": "m"}),
|
||||
json!({"vertex_location": 5}),
|
||||
Err(()),
|
||||
|
|
@ -447,19 +440,27 @@ mod tests {
|
|||
.unwrap()
|
||||
};
|
||||
let kwargs = dict(&kwargs);
|
||||
let globals = dict(&globals);
|
||||
let defaults = dict(&json!({"vertex_project": null, "vertex_location": null}));
|
||||
defaults.update(dict(&globals).as_mapping()).unwrap();
|
||||
let namespace = py
|
||||
.import("types")
|
||||
.unwrap()
|
||||
.getattr("SimpleNamespace")
|
||||
.unwrap()
|
||||
.call((), Some(&defaults))
|
||||
.unwrap();
|
||||
let snapshot = PythonSettings::ProviderDefaults.snapshot(namespace);
|
||||
let bound = PyDict::new(py);
|
||||
let projected = project_litellm_params(
|
||||
|name| present(&kwargs, &bound, name),
|
||||
|name| present(&globals, &bound, name),
|
||||
)
|
||||
.unwrap();
|
||||
|fields, unset| credential_globals::fold_from(fields, unset, &snapshot),
|
||||
);
|
||||
match (projected, expected) {
|
||||
(Ok(projected), Ok(fields)) => assert_eq!(
|
||||
(Ok(Ok(projected)), Ok(fields)) => assert_eq!(
|
||||
projected,
|
||||
litellm_params(serde_json::from_value(fields).unwrap()).unwrap()
|
||||
),
|
||||
(Err(error), Err(())) => assert!(matches!(error, Error::InvalidRequest(_))),
|
||||
(Err(_) | Ok(Err(_)), Err(())) => {}
|
||||
(projected, expected) => panic!("got {projected:?}, expected {expected:?}"),
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
pub(crate) mod audio_transcription;
|
||||
pub(crate) mod chat_completions;
|
||||
pub(crate) mod clickhouse_spend;
|
||||
mod credential_globals;
|
||||
pub(crate) mod embeddings;
|
||||
mod inference;
|
||||
pub(crate) mod messages;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
use litellm_auth::SecretValue;
|
||||
use litellm_auth::VertexParams;
|
||||
use litellm_auth::{Sources, VertexKeys};
|
||||
use litellm_host_python::{from_py, present};
|
||||
use litellm_inference_ocr::{
|
||||
types::{LiteLLMOcrRequest, OcrDocumentInput},
|
||||
|
|
@ -11,10 +11,9 @@ use serde_json::{Map, Value};
|
|||
|
||||
use super::{document::FileDocumentInput, errors::to_pyerr as ocr_error_to_pyerr};
|
||||
use crate::{
|
||||
coercion::FieldSpec,
|
||||
credentials::{self, CallerTokenProvider},
|
||||
marshal::{project_optional_fields, python_timeout_seconds, request_input_sources},
|
||||
python_settings::{PythonSettings, Snapshot},
|
||||
routes::credential_globals,
|
||||
};
|
||||
|
||||
/// What the host keeps after projection: the caller's token callable that answers the
|
||||
|
|
@ -126,16 +125,9 @@ pub(super) fn project_request(
|
|||
let names = specs.iter().map(|spec| spec.name).collect::<Vec<_>>();
|
||||
let optional_params =
|
||||
project_optional_fields(names.iter().copied(), |name| present(kwargs, bound, name))?;
|
||||
let globals = module_globals_to_read(&optional_params, &names);
|
||||
let defaults = if globals.is_empty() {
|
||||
None
|
||||
} else {
|
||||
PythonSettings::ProviderDefaults.read_or_unset(bound.py())?
|
||||
};
|
||||
let optional_params = match defaults {
|
||||
Some(defaults) => fold_module_globals(bound.py(), optional_params, &globals, &defaults)?,
|
||||
None => optional_params,
|
||||
};
|
||||
let unset = unset_vertex_settings(&optional_params, &names);
|
||||
let optional_params = credential_globals::fold(bound.py(), optional_params, &unset)
|
||||
.map_err(|error| settings_error(bound.py(), error))?;
|
||||
let input_sources = request_input_sources(
|
||||
kwargs,
|
||||
names
|
||||
|
|
@ -169,48 +161,21 @@ pub(super) fn project_request(
|
|||
))
|
||||
}
|
||||
|
||||
/// Python's `VertexBase.get_vertex_ai_project` and `get_vertex_ai_location` fall back to a
|
||||
/// `litellm.<name>` module global when a call names neither spelling: the globals of the
|
||||
/// specs this route consumes whose wire names the call left out.
|
||||
fn module_globals_to_read(
|
||||
/// The Vertex settings this route consumes that the call leaves unset, whose `litellm.*`
|
||||
/// globals `VertexBase.get_vertex_ai_project` and `get_vertex_ai_location` fall back to.
|
||||
fn unset_vertex_settings(
|
||||
optional_params: &Map<String, Value>,
|
||||
consumed: &[&str],
|
||||
) -> Vec<&'static str> {
|
||||
VertexParams::SPECS
|
||||
.iter()
|
||||
.filter(|spec| spec.wire.iter().all(|name| consumed.contains(name)))
|
||||
.filter(|spec| {
|
||||
!spec
|
||||
.wire
|
||||
) -> Vec<&'static Sources> {
|
||||
credential_globals::unset(
|
||||
optional_params,
|
||||
VertexKeys::SOURCES.iter().filter(|sources| {
|
||||
sources
|
||||
.litellm_params
|
||||
.iter()
|
||||
.any(|name| optional_params.contains_key(*name))
|
||||
})
|
||||
.filter_map(|spec| spec.module_global)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Folds each module global in under the name its spec declares, skipping falsy values the
|
||||
/// way Python's `or` chain does.
|
||||
fn fold_module_globals(
|
||||
py: Python<'_>,
|
||||
optional_params: Map<String, Value>,
|
||||
globals: &[&'static str],
|
||||
defaults: &Snapshot<'_>,
|
||||
) -> PyResult<Map<String, Value>> {
|
||||
let read = globals
|
||||
.iter()
|
||||
.map(|name| {
|
||||
let spec = FieldSpec::new(name, |field| field.falsy_optional_string());
|
||||
Ok(defaults
|
||||
.read(&spec)
|
||||
.map_err(|error| settings_error(py, error.into()))?
|
||||
.map(|value| (name.to_string(), Value::String(value))))
|
||||
})
|
||||
.collect::<PyResult<Vec<_>>>()?;
|
||||
Ok(optional_params
|
||||
.into_iter()
|
||||
.chain(read.into_iter().flatten())
|
||||
.collect())
|
||||
.all(|name| consumed.contains(name))
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
/// A misconfigured litellm setting is the operator's error, not the request's or the
|
||||
|
|
@ -228,6 +193,7 @@ mod tests {
|
|||
use pyo3::exceptions::PyValueError;
|
||||
|
||||
use super::*;
|
||||
use crate::python_settings::PythonSettings;
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::global_fills_a_missing_project(
|
||||
|
|
@ -263,11 +229,10 @@ mod tests {
|
|||
)
|
||||
.unwrap();
|
||||
let snapshot = PythonSettings::ProviderDefaults.snapshot(namespace);
|
||||
let consumed = VertexParams::fields().collect::<Vec<_>>();
|
||||
let params = params.as_object().cloned().unwrap();
|
||||
let globals = module_globals_to_read(¶ms, &consumed);
|
||||
let unset = unset_vertex_settings(¶ms, VertexKeys::LITELLM_PARAMS);
|
||||
|
||||
let folded = fold_module_globals(py, params, &globals, &snapshot).unwrap();
|
||||
let folded = credential_globals::fold_from(params, &unset, &snapshot).unwrap();
|
||||
|
||||
assert_eq!(Value::Object(folded), expected);
|
||||
});
|
||||
|
|
@ -286,8 +251,11 @@ mod tests {
|
|||
.unwrap();
|
||||
let snapshot = PythonSettings::ProviderDefaults.snapshot(namespace);
|
||||
|
||||
let error =
|
||||
fold_module_globals(py, Map::new(), &["vertex_project"], &snapshot).unwrap_err();
|
||||
let error = settings_error(
|
||||
py,
|
||||
credential_globals::fold_from(Map::new(), &[VertexKeys::PROJECT], &snapshot)
|
||||
.unwrap_err(),
|
||||
);
|
||||
|
||||
assert!(error.is_instance_of::<PyValueError>(py));
|
||||
assert!(
|
||||
|
|
@ -323,7 +291,10 @@ mod tests {
|
|||
#[case] expected: &[&str],
|
||||
) {
|
||||
assert_eq!(
|
||||
module_globals_to_read(params.as_object().unwrap(), consumed),
|
||||
unset_vertex_settings(params.as_object().unwrap(), consumed)
|
||||
.iter()
|
||||
.filter_map(|sources| sources.litellm_setting)
|
||||
.collect::<Vec<_>>(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
use std::{collections::BTreeMap, fmt, ops::Deref};
|
||||
|
||||
use litellm_auth_types::{AwsParams, ParamSpec, SecretValue, VertexParams};
|
||||
use litellm_auth_types::{AwsKeys, AwsParams, SecretValue, VertexKeys, VertexParams};
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
use serde_with::{OneOrMany, formats::PreferMany, serde_as};
|
||||
|
|
@ -54,10 +54,13 @@ pub struct LitellmParams {
|
|||
}
|
||||
|
||||
impl LitellmParams {
|
||||
/// Every connection param spec the credential families declare, for hosts that fold a
|
||||
/// spec's module global in when a call names none of its spellings.
|
||||
pub fn specs() -> impl Iterator<Item = &'static ParamSpec> {
|
||||
AwsParams::SPECS.iter().chain(VertexParams::SPECS.iter())
|
||||
/// Every credential setting that has a `litellm.*` global, for hosts that fold the global
|
||||
/// in when a call leaves the setting unset.
|
||||
pub fn litellm_settings() -> impl Iterator<Item = &'static litellm_auth_types::Sources> {
|
||||
AwsKeys::SOURCES
|
||||
.iter()
|
||||
.chain(VertexKeys::SOURCES.iter())
|
||||
.filter(|sources| sources.litellm_setting.is_some())
|
||||
}
|
||||
|
||||
/// The wire names a host projects out of a caller's kwargs: the model, the credentials
|
||||
|
|
@ -66,8 +69,8 @@ impl LitellmParams {
|
|||
pub fn fields() -> impl Iterator<Item = &'static str> {
|
||||
["model", "api_key", "api_base", "api_version"]
|
||||
.into_iter()
|
||||
.chain(AwsParams::fields())
|
||||
.chain(VertexParams::fields())
|
||||
.chain(AwsKeys::LITELLM_PARAMS.iter().copied())
|
||||
.chain(VertexKeys::LITELLM_PARAMS.iter().copied())
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue