scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-12 03:48:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 57

Merge 075c58a92a into a7a3a6b375

Browse source
This commit is contained in:
yujonglee 2026-10-11 13:26:01 +00:00 • committed by GitHub
commit 13e0e71a76
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
42 changed files with 1214 additions and 431 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
litellm-rust/crates/auth-gcp/src/config.rs
View file

@ -50,6 +50,14 @@ impl VertexConfig {
)
}
pub fn with_project_and_location(self, project_id: Option<String>, location: String) -> Self {
Self::new(
self.credentials,
project_id.or(self.project_id),
Some(location),
)
}
pub fn project_id(&self) -> Option<&str> {
self.project_id.as_deref()
}

7
litellm-rust/crates/auth-types/src/params/aws.rs
View file

@ -11,12 +11,7 @@ const fn spec(
wire: &'static [&'static str],
env: &'static [&'static str],
) -> ParamSpec {
ParamSpec {
setting,
wire,
module_global: None,
env,
}
ParamSpec { setting, wire, env }
}
/// The `aws_*` fields of Python's `GenericLiteLLMParams`.

15
litellm-rust/crates/auth-types/src/params/mod.rs
View file

@ -5,13 +5,11 @@ pub use aws::AwsParams;
pub use vertex::VertexParams;
/// One connection param of Python's `CredentialLiteLLMParams` and every place Python reads
/// it from, in order: the wire names on a call or a deployment, the `litellm.<name>` module
/// global a host folds in, then the environment names.
/// it from, in order: the wire names on a call or a deployment, then the environment names.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct ParamSpec {
pub setting: &'static str,
pub wire: &'static [&'static str],
pub module_global: Option<&'static str>,
pub env: &'static [&'static str],
}
@ -38,11 +36,10 @@ impl ParamSpec {
}
pub(crate) fn guidance(&self) -> String {
let wire = self.wire.join(" or ");
let env = self.env.join(" or ");
match self.module_global {
Some(global) => format!("pass {wire}, set litellm.{global}, or set {env}"),
None => format!("pass {wire} or set {env}"),
}
format!(
"pass {} or set {}",
self.wire.join(" or "),
self.env.join(" or ")
)
}
}

3
litellm-rust/crates/auth-types/src/params/vertex.rs
View file

@ -28,19 +28,16 @@ impl VertexParams {
pub const CREDENTIALS: ParamSpec = ParamSpec {
setting: "credentials",
wire: &["vertex_credentials", "vertex_ai_credentials"],
module_global: None,
env: &["VERTEXAI_CREDENTIALS"],
};
pub const PROJECT: ParamSpec = ParamSpec {
setting: "project",
wire: &["vertex_project", "vertex_ai_project"],
module_global: Some("vertex_project"),
env: &["VERTEXAI_PROJECT"],
};
pub const LOCATION: ParamSpec = ParamSpec {
setting: "location",
wire: &["vertex_location", "vertex_ai_location"],
module_global: Some("vertex_location"),
env: &["VERTEXAI_LOCATION", "VERTEX_LOCATION"],
};

9
litellm-rust/crates/auth-types/tests/params.rs
View file

@ -4,7 +4,6 @@ use rstest::rstest;
const LOCATION: ParamSpec = ParamSpec {
setting: "location",
wire: &["vertex_location", "vertex_ai_location"],
module_global: Some("vertex_location"),
env: &["VERTEXAI_LOCATION", "VERTEX_LOCATION"],
};
@ -51,13 +50,13 @@ fn resolution_walks_wire_names_then_environment_names(
}
#[rstest]
#[case::without_a_module_global(
#[case::aws(
&AwsParams::REGION,
"Missing AWS region - pass aws_region_name or set AWS_REGION_NAME or AWS_REGION"
)]
#[case::with_a_module_global(
#[case::several_wire_names(
&LOCATION,
"Missing AWS location - pass vertex_location or vertex_ai_location, set litellm.vertex_location, or set VERTEXAI_LOCATION or VERTEX_LOCATION"
"Missing AWS location - pass vertex_location or vertex_ai_location or set VERTEXAI_LOCATION or VERTEX_LOCATION"
)]
fn a_missing_param_names_every_place_the_spec_reads(
#[case] spec: &'static ParamSpec,
@ -68,5 +67,5 @@ fn a_missing_param_names_every_place_the_spec_reads(
assert_eq!(message, expected);
assert!(spec.wire.iter().all(|name| message.contains(name)));
assert!(spec.env.iter().all(|name| message.contains(name)));
assert_eq!(message.contains("litellm."), spec.module_global.is_some());
assert!(!message.contains("litellm."));
}

72
litellm-rust/crates/core-utils/src/settings.rs
View file

@ -53,6 +53,55 @@ pub fn resolve_non_empty(
})
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum SettingSource {
Explicit,
RuntimeDefault,
Environment(&'static str),
Derived,
ProviderDefault,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Selected<T> {
pub value: T,
pub source: SettingSource,
}
impl<T> Selected<T> {
pub fn new(value: T, source: SettingSource) -> Self {
Self { value, source }
}
}
/// The first nonblank value of an explicit argument, a runtime default, then each
/// environment name in order, keeping which layer supplied it.
pub fn select_non_blank(
explicit: Option<&str>,
runtime_default: Option<&str>,
env_lookup: &dyn Fn(&str) -> Option<String>,
env_names: &[&'static str],
) -> Option<Selected<String>> {
[
(SettingSource::Explicit, explicit),
(SettingSource::RuntimeDefault, runtime_default),
]
.into_iter()
.find_map(|(source, value)| {
value
.map(str::trim)
.filter(|value| !value.is_empty())
.map(|value| Selected::new(value.to_string(), source))
})
.or_else(|| {
env_names.iter().find_map(|name| {
env_lookup(name)
.filter(|value| !value.trim().is_empty())
.map(|value| Selected::new(value, SettingSource::Environment(name)))
})
})
}
pub trait Layer: Default {
fn or(self, lower: Self) -> Self;
}
@ -159,4 +208,27 @@ mod tests {
fn merging_no_layers_yields_the_empty_layer() {
assert_eq!(merge(Vec::<Pair>::new()), Pair::default());
}
#[rstest::rstest]
#[case::explicit_beats_everything(Some("arg"), Some("default"), &[("FIRST", "env")], Some(("arg", SettingSource::Explicit)))]
#[case::runtime_default_beats_the_environment(None, Some("default"), &[("FIRST", "env")], Some(("default", SettingSource::RuntimeDefault)))]
#[case::blank_layers_are_skipped(Some(" "), Some(""), &[("FIRST", " "), ("SECOND", "env")], Some(("env", SettingSource::Environment("SECOND"))))]
#[case::environment_names_in_order(None, None, &[("SECOND", "b"), ("FIRST", "a")], Some(("a", SettingSource::Environment("FIRST"))))]
#[case::nothing_set(None, None, &[], None)]
fn selection_keeps_the_layer_that_supplied_the_value(
#[case] explicit: Option<&str>,
#[case] runtime_default: Option<&str>,
#[case] env: &'static [(&'static str, &'static str)],
#[case] expected: Option<(&str, SettingSource)>,
) {
assert_eq!(
select_non_blank(
explicit,
runtime_default,
&env_of(env),
&["FIRST", "SECOND"]
),
expected.map(|(value, source)| Selected::new(value.to_string(), source))
);
}
}

1
litellm-rust/crates/gateway-inference/src/messages.rs
View file

@ -80,6 +80,7 @@ fn project(
api_base: deployment.api_base.clone(),
custom_llm_provider: deployment.custom_llm_provider.clone(),
litellm_params: deployment.litellm_params.clone(),
runtime_defaults: Default::default(),
extra_headers: None,
provider_specific_header: anthropic_api_headers(headers),
timeout: deployment.timeout,

31
litellm-rust/crates/inference-messages/src/prepare.rs
View file

@ -7,7 +7,8 @@ use litellm_core_utils::{
};
use litellm_http::request::with_default_headers;
use litellm_llms::base_llm::{
auth::ValidatedEnvironment, messages::context::MessagesTransformContext,
auth::ValidatedEnvironment, connection::ConnectionInputs,
messages::context::MessagesTransformContext,
};
use litellm_llms_types::formats::messages::MessagesRequest;
use litellm_secrets::source::SecretSource;
@ -70,6 +71,7 @@ fn prepare_provider_request(
api_key,
api_base,
litellm_params,
runtime_defaults,
extra_headers,
provider_specific_header,
timeout,
@ -95,28 +97,26 @@ fn prepare_provider_request(
let forwarded = string_headers(Some(
extra_headers.into_iter().flatten().chain(scoped).collect(),
))?;
let validated = config.validate_environment(
let connection = config.prepare_connection(
ConnectionInputs {
api_key: api_key.as_deref(),
api_base: api_base.as_deref(),
litellm_params: &litellm_params,
runtime_defaults: &runtime_defaults,
},
forwarded,
api_key.as_deref(),
&transformed.model,
&litellm_params,
transformed.params.stream == Some(true),
&env_lookup,
)?;
let environment = ValidatedEnvironment {
headers: config.request_headers(
with_default_headers(validated.headers, config.default_headers()),
with_default_headers(connection.headers, config.default_headers()),
&transformed,
),
auth: validated.auth,
auth: connection.auth,
};
let url = config.get_complete_url(
api_base.as_deref(),
&transformed.model,
&litellm_params,
transformed.params.stream == Some(true),
&env_lookup,
)?;
let url = connection.url;
Ok(ProviderMessagesRequest {
provider,
@ -232,6 +232,7 @@ mod tests {
api_base: None,
custom_llm_provider: Some("anthropic".into()),
litellm_params: Default::default(),
runtime_defaults: Default::default(),
extra_headers: None,
provider_specific_header: None,
timeout: None,
@ -259,6 +260,7 @@ mod tests {
api_base: Some("https://anthropic.test".into()),
custom_llm_provider: Some("anthropic".into()),
litellm_params: Default::default(),
runtime_defaults: Default::default(),
extra_headers: None,
provider_specific_header: None,
timeout: None,
@ -367,6 +369,7 @@ mod tests {
api_base: Some("https://resource.services.ai.azure.com".into()),
custom_llm_provider: custom_llm_provider.map(Into::into),
litellm_params: Default::default(),
runtime_defaults: Default::default(),
extra_headers: Some(Map::from_iter([("x-priority".into(), json!("extra"))])),
provider_specific_header: Some(configured),
timeout: None,

1
litellm-rust/crates/inference-messages/src/types.rs
View file

@ -19,6 +19,7 @@ pub struct MessagesCall {
pub api_base: Option<String>,
pub custom_llm_provider: Option<String>,
pub litellm_params: LitellmParams,
pub runtime_defaults: litellm_llms::base_llm::connection::RuntimeDefaults,
pub extra_headers: Option<Map<String, Value>>,
pub provider_specific_header: Option<ProviderSpecificHeaders>,
pub timeout: Option<Duration>,

2
litellm-rust/crates/inference-messages/tests/caching.rs
View file

@ -206,7 +206,7 @@ async fn messages_cache_identity_follows_resolved_configuration_and_request_call
let cache = ScopedCache::new(cache.clone(), CacheScope::Shared);
support::messages_route(secrets.clone()).with_cache(cache).execute(MessagesCall {
body: serde_json::from_value(json!({"model":"anthropic/cache-test-model","messages":[{"role":"user","content":"hello"}],"max_tokens":32})).unwrap(),
api_key:None,api_base:None,custom_llm_provider:None,litellm_params:Default::default(),extra_headers:None,provider_specific_header:None,timeout:None,shaping:Default::default(),
api_key:None,api_base:None,custom_llm_provider:None,litellm_params:Default::default(),runtime_defaults:Default::default(),extra_headers:None,provider_specific_header:None,timeout:None,shaping:Default::default(),
}, &hooks, None).await.unwrap();
}
assert_eq!(hooks.calls.load(Ordering::SeqCst), 4);

1
litellm-rust/crates/inference-messages/tests/messages/main.rs
View file

@ -82,6 +82,7 @@ fn call() -> MessagesCall {
api_base: None,
custom_llm_provider: Some("anthropic".into()),
litellm_params: Default::default(),
runtime_defaults: Default::default(),
extra_headers: None,
provider_specific_header: None,
timeout: Some(Duration::from_secs(5)),

11
litellm-rust/crates/inference-messages/tests/messages/request.rs
View file

@ -69,6 +69,7 @@ async fn credentials_become_exactly_one_auth_header(
run_message(MessagesCall {
custom_llm_provider: Some(provider.into()),
litellm_params: Default::default(),
runtime_defaults: Default::default(),
api_key: api_key.map(Into::into),
api_base: Some(upstream.uri()),
extra_headers: headers(extra_headers.iter().copied()),
@ -98,6 +99,7 @@ async fn a_call_without_credentials_fails_before_sending(
let error = run(MessagesCall {
custom_llm_provider: Some(provider.into()),
litellm_params: Default::default(),
runtime_defaults: Default::default(),
api_base: Some(upstream.uri()),
..call
})
@ -140,6 +142,7 @@ async fn each_provider_posts_to_its_messages_endpoint(
run_message(MessagesCall {
custom_llm_provider: provider.map(Into::into),
litellm_params: Default::default(),
runtime_defaults: Default::default(),
api_key: Some("sk".into()),
api_base: Some(format!("{}{base_suffix}", upstream.uri())),
..with_model(call, model)
@ -171,6 +174,7 @@ async fn unsupported_providers_are_rejected_before_sending(
let error = run(MessagesCall {
custom_llm_provider: provider.map(Into::into),
litellm_params: Default::default(),
runtime_defaults: Default::default(),
api_key: Some("sk".into()),
api_base: Some(UNREACHABLE_BASE.into()),
..with_model(call, model)
@ -224,6 +228,7 @@ async fn cache_scope_removal_is_selected_by_the_provider(
run_message(MessagesCall {
custom_llm_provider: Some(provider.into()),
litellm_params: Default::default(),
runtime_defaults: Default::default(),
api_key: Some("sk-azure".into()),
api_base: Some(upstream.uri()),
body: body(json!({
@ -369,6 +374,7 @@ async fn caller_protocol_headers_win_over_the_defaults(call: MessagesCall, #[cas
run_message(MessagesCall {
custom_llm_provider: Some(provider.into()),
litellm_params: Default::default(),
runtime_defaults: Default::default(),
api_key: Some("sk".into()),
api_base: Some(upstream.uri()),
extra_headers: headers([
@ -421,6 +427,7 @@ async fn unsupported_params_are_dropped_under_drop_params_and_rejected_without_i
body: call.body.clone(),
custom_llm_provider: call.custom_llm_provider.clone(),
litellm_params: Default::default(),
runtime_defaults: Default::default(),
extra_headers: None,
provider_specific_header: None,
timeout: call.timeout,
@ -600,6 +607,7 @@ async fn metadata_is_reduced_to_the_user_id(call: MessagesCall, #[case] provider
MessagesCall {
custom_llm_provider: Some(provider.into()),
litellm_params: Default::default(),
runtime_defaults: Default::default(),
api_key: Some("sk".into()),
api_base: Some(upstream.uri()),
..call
@ -625,6 +633,7 @@ async fn deepseek_sends_neither_billing_blocks_nor_the_custom_tool_discriminator
MessagesCall {
custom_llm_provider: Some("deepseek".into()),
litellm_params: Default::default(),
runtime_defaults: Default::default(),
api_key: Some("sk".into()),
api_base: Some(upstream.uri()),
..call
@ -747,6 +756,7 @@ async fn system_message_folding_is_selected_by_the_provider(
MessagesCall {
custom_llm_provider: Some(provider.into()),
litellm_params: Default::default(),
runtime_defaults: Default::default(),
api_key: Some("sk-azure".into()),
api_base: Some(upstream.uri()),
shaping: MessagesShaping {
@ -812,6 +822,7 @@ async fn provider_validation_runs_before_caller_parameter_removal(
MessagesCall {
custom_llm_provider: Some(provider.into()),
litellm_params: Default::default(),
runtime_defaults: Default::default(),
api_key: Some("sk-test".into()),
api_base: Some(upstream.uri()),
shaping: MessagesShaping {

1
litellm-rust/crates/inference-messages/tests/messages/response.rs
View file

@ -105,6 +105,7 @@ async fn the_provider_message_is_returned(call: MessagesCall, #[case] provider:
let message = run_message(MessagesCall {
custom_llm_provider: Some(provider.into()),
litellm_params: Default::default(),
runtime_defaults: Default::default(),
api_key: Some("sk".into()),
api_base: Some(upstream.uri()),
..call

2
litellm-rust/crates/inference-messages/tests/messages/secrets.rs
View file

@ -46,6 +46,7 @@ async fn the_credential_and_base_come_from_the_secret_source(
MessagesCall {
custom_llm_provider: Some(provider.into()),
litellm_params: Default::default(),
runtime_defaults: Default::default(),
..call
},
)
@ -198,6 +199,7 @@ async fn azure_without_a_base_anywhere_fails_before_sending(call: MessagesCall)
MessagesCall {
custom_llm_provider: Some("azure_ai".into()),
litellm_params: Default::default(),
runtime_defaults: Default::default(),
..call
},
)

2
litellm-rust/crates/inference-messages/tests/messages/stream.rs
View file

@ -347,6 +347,7 @@ async fn the_sdk_returns_stream_headers_and_every_sse_byte(
MessagesCall {
custom_llm_provider: Some(provider.into()),
litellm_params: Default::default(),
runtime_defaults: Default::default(),
..streaming(call, upstream.uri())
},
&(),
@ -472,6 +473,7 @@ async fn a_host_on_anthropic_sse_is_relayed_byte_for_byte(call: MessagesCall) {
MessagesCall {
custom_llm_provider: Some("azure_ai".into()),
litellm_params: Default::default(),
runtime_defaults: Default::default(),
..streaming(call, upstream.uri())
},
usize::MAX,

2
litellm-rust/crates/inference-ocr/src/prepare.rs
View file

@ -54,6 +54,7 @@ pub(crate) fn prepare_request(
optional_params,
input_sources,
azure_ad_token_provider,
runtime_defaults,
..
} = request;
PreparedOcrRequest {
@ -64,6 +65,7 @@ pub(crate) fn prepare_request(
optional_params,
input_sources,
azure_ad_token_provider,
runtime_defaults,
}
}

12
litellm-rust/crates/inference-ocr/src/types.rs
View file

@ -1,3 +1,4 @@
use litellm_llms::base_llm::connection::RuntimeDefaults;
use std::{collections::BTreeMap, path::PathBuf, time::Duration};
use bytes::Bytes;
@ -81,6 +82,7 @@ pub struct LiteLLMOcrRequest<D = OcrDocumentInput> {
pub optional_params: CallArguments,
pub input_sources: BTreeMap<String, InputSource>,
pub azure_ad_token_provider: Option<TokenProviderHandle>,
pub runtime_defaults: RuntimeDefaults,
pub(crate) config: OcrConfigKind,
}
@ -123,12 +125,20 @@ impl LiteLLMOcrRequest {
optional_params,
input_sources: BTreeMap::new(),
azure_ad_token_provider: None,
runtime_defaults: RuntimeDefaults::default(),
config,
})
}
}
impl<D> LiteLLMOcrRequest<D> {
pub fn with_runtime_defaults(self, runtime_defaults: RuntimeDefaults) -> Self {
Self {
runtime_defaults,
..self
}
}
pub fn map_document<T, E>(
self,
map: impl FnOnce(D) -> Result<T, E>,
@ -141,6 +151,7 @@ impl<D> LiteLLMOcrRequest<D> {
optional_params: self.optional_params,
input_sources: self.input_sources,
azure_ad_token_provider: self.azure_ad_token_provider,
runtime_defaults: self.runtime_defaults,
config: self.config,
})
}
@ -154,6 +165,7 @@ impl<D> LiteLLMOcrRequest<D> {
optional_params: self.optional_params,
input_sources: self.input_sources,
azure_ad_token_provider: self.azure_ad_token_provider,
runtime_defaults: self.runtime_defaults,
config: self.config,
}
}

4
litellm-rust/crates/llms/src/anthropic/common_utils.rs
View file

@ -150,8 +150,10 @@ pub fn complete_anthropic_url(
api_base: Option<&str>,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> String {
let api_base = resolve_anthropic_api_base(api_base, env_lookup);
messages_url(&resolve_anthropic_api_base(api_base, env_lookup))
}
pub fn messages_url(api_base: &str) -> String {
let api_base = api_base.trim_end_matches('/');
if api_base.ends_with(MESSAGES_PATH) {
return api_base.to_string();

140
litellm-rust/crates/llms/src/anthropic/connection.rs Normal file
View file

@ -0,0 +1,140 @@
use litellm_core_utils::settings::{Selected, SettingSource, select_non_blank};
use litellm_llms_types::providers::anthropic::API_BASE;
use super::common_utils::{ANTHROPIC_API_BASE_ENV, ANTHROPIC_BASE_URL_ENV, messages_url};
use crate::base_llm::connection::{
InvalidApiBase, RuntimeDefaults, complete_url, path_segments, validated_api_base,
};
/// The Anthropic host every Anthropic operation sends to, chosen once from the explicit
/// `api_base`, the host's runtime default, `ANTHROPIC_API_BASE`, `ANTHROPIC_BASE_URL`, then
/// the public API.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AnthropicConnection {
api_base: Selected<String>,
}
impl AnthropicConnection {
pub fn resolve(
api_base: Option<&str>,
runtime_defaults: &RuntimeDefaults,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<Self, InvalidApiBase> {
let api_base = select_non_blank(
api_base,
runtime_defaults.api_base.as_deref(),
env_lookup,
&[ANTHROPIC_API_BASE_ENV, ANTHROPIC_BASE_URL_ENV],
)
.unwrap_or_else(|| Selected::new(API_BASE.to_string(), SettingSource::ProviderDefault));
Ok(Self {
api_base: validated_api_base(api_base)?,
})
}
pub fn api_base(&self) -> &str {
&self.api_base.value
}
pub fn api_base_source(&self) -> SettingSource {
self.api_base.source
}
pub fn messages_url(&self) -> String {
messages_url(self.api_base())
}
pub fn endpoint(&self, path: &str) -> Result<String, InvalidApiBase> {
complete_url(
self.api_base().trim_end_matches('/'),
&path_segments(path).collect::<Vec<_>>(),
)
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
fn env(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
move |name| {
values
.iter()
.find(|(key, _)| *key == name)
.map(|(_, value)| value.to_string())
}
}
fn runtime(api_base: Option<&str>) -> RuntimeDefaults {
RuntimeDefaults {
api_base: api_base.map(Into::into),
..RuntimeDefaults::default()
}
}
#[rstest]
#[case::explicit_beats_both_aliases(
Some("https://explicit.test"),
None,
&[("ANTHROPIC_API_BASE", "https://a.test"), ("ANTHROPIC_BASE_URL", "https://b.test")],
"https://explicit.test",
SettingSource::Explicit
)]
#[case::runtime_default_beats_the_environment(
None,
Some("https://runtime.test"),
&[("ANTHROPIC_API_BASE", "https://a.test")],
"https://runtime.test",
SettingSource::RuntimeDefault
)]
#[case::api_base_alias_before_base_url_alias(
None,
None,
&[("ANTHROPIC_BASE_URL", "https://b.test"), ("ANTHROPIC_API_BASE", "https://a.test")],
"https://a.test",
SettingSource::Environment("ANTHROPIC_API_BASE")
)]
#[case::base_url_alias_alone(
None,
None,
&[("ANTHROPIC_BASE_URL", "https://b.test")],
"https://b.test",
SettingSource::Environment("ANTHROPIC_BASE_URL")
)]
#[case::blank_overrides_fall_through(
Some(" "),
Some(""),
&[],
"https://api.anthropic.com",
SettingSource::ProviderDefault
)]
fn the_api_base_follows_the_layer_precedence(
#[case] explicit: Option<&str>,
#[case] runtime_default: Option<&str>,
#[case] vars: &'static [(&'static str, &'static str)],
#[case] expected: &str,
#[case] source: SettingSource,
) {
let connection =
AnthropicConnection::resolve(explicit, &runtime(runtime_default), &env(vars)).unwrap();
assert_eq!(
(connection.api_base(), connection.api_base_source()),
(expected, source)
);
}
#[rstest]
#[case::explicit("not a url", None)]
#[case::runtime_default(" ", Some("relative/path"))]
fn a_malformed_override_is_rejected(
#[case] explicit: &str,
#[case] runtime_default: Option<&str>,
) {
assert_eq!(
AnthropicConnection::resolve(Some(explicit), &runtime(runtime_default), &env(&[])),
Err(InvalidApiBase)
);
}
}

97
litellm-rust/crates/llms/src/anthropic/count_tokens/transformation.rs
View file

@ -1,13 +1,16 @@
use litellm_llms_types::{
formats::messages::{Message, SystemPrompt},
providers::anthropic::{
API_BASE, API_KEY_HEADER, API_VERSION, BETA_HEADER, COUNT_TOKENS_PATH, VERSION_HEADER,
API_KEY_HEADER, API_VERSION, BETA_HEADER, COUNT_TOKENS_PATH, VERSION_HEADER,
},
};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::{Error, anthropic::ANTHROPIC_OAUTH_TOKEN_PREFIX};
use crate::{
Error,
anthropic::{ANTHROPIC_OAUTH_TOKEN_PREFIX, connection::AnthropicConnection},
};
const TOKEN_COUNTING_BETA: &str = "token-counting-2024-11-01";
@ -27,7 +30,7 @@ pub struct AnthropicCountTokensResponse {
}
pub trait AnthropicCountTokensConfig {
fn endpoint(&self) -> String;
fn endpoint(&self, connection: &AnthropicConnection) -> Result<String, Error>;
fn validate_request(&self, model: &str, messages: &[Message]) -> Result<(), Error>;
@ -48,8 +51,8 @@ pub const ANTHROPIC_COUNT_TOKENS_TRANSFORMATION: AnthropicCountTokensTransformat
AnthropicCountTokensTransformation;
impl AnthropicCountTokensConfig for AnthropicCountTokensTransformation {
fn endpoint(&self) -> String {
format!("{API_BASE}{COUNT_TOKENS_PATH}")
fn endpoint(&self, connection: &AnthropicConnection) -> Result<String, Error> {
Ok(connection.endpoint(COUNT_TOKENS_PATH)?)
}
fn transform_request(
@ -129,9 +132,89 @@ mod tests {
"system": "system"
})
);
}
fn connection(
api_base: Option<&str>,
runtime_default: Option<&str>,
env: &'static [(&'static str, &'static str)],
) -> Result<AnthropicConnection, crate::base_llm::connection::InvalidApiBase> {
AnthropicConnection::resolve(
api_base,
&crate::base_llm::connection::RuntimeDefaults {
api_base: runtime_default.map(Into::into),
..Default::default()
},
&|name| {
env.iter()
.find(|(key, _)| *key == name)
.map(|(_, value)| value.to_string())
},
)
}
#[rstest::rstest]
#[case::explicit_base_beats_both_aliases(
Some("https://explicit.test"),
None,
&[("ANTHROPIC_API_BASE", "https://a.test"), ("ANTHROPIC_BASE_URL", "https://b.test")],
"https://explicit.test/v1/messages/count_tokens"
)]
#[case::runtime_default_beats_the_environment(
None,
Some("https://runtime.test"),
&[("ANTHROPIC_API_BASE", "https://a.test"), ("ANTHROPIC_BASE_URL", "https://b.test")],
"https://runtime.test/v1/messages/count_tokens"
)]
#[case::api_base_alias(
None,
None,
&[("ANTHROPIC_API_BASE", "https://a.test"), ("ANTHROPIC_BASE_URL", "https://b.test")],
"https://a.test/v1/messages/count_tokens"
)]
#[case::base_url_alias(
None,
None,
&[("ANTHROPIC_BASE_URL", "https://b.test")],
"https://b.test/v1/messages/count_tokens"
)]
#[case::gateway_prefix_survives(
Some("https://gateway.test/anthropic/"),
None,
&[],
"https://gateway.test/anthropic/v1/messages/count_tokens"
)]
#[case::chat_surface_base(
Some("https://gateway.test/anthropic/v1/messages"),
None,
&[],
"https://gateway.test/anthropic/v1/messages/count_tokens"
)]
#[case::public_api_without_overrides(None, None, &[], "https://api.anthropic.com/v1/messages/count_tokens")]
fn the_endpoint_follows_the_resolved_anthropic_base(
#[case] api_base: Option<&str>,
#[case] runtime_default: Option<&str>,
#[case] env: &'static [(&'static str, &'static str)],
#[case] expected: &str,
) {
let connection = connection(api_base, runtime_default, env).unwrap();
assert_eq!(
ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.endpoint(),
"https://api.anthropic.com/v1/messages/count_tokens"
ANTHROPIC_COUNT_TOKENS_TRANSFORMATION
.endpoint(&connection)
.unwrap(),
expected
);
}
#[test]
fn a_malformed_explicit_base_never_falls_back_to_the_public_api() {
assert!(
connection(
Some("not a url"),
None,
&[("ANTHROPIC_API_BASE", "https://a.test")]
)
.is_err()
);
}

28
litellm-rust/crates/llms/src/anthropic/messages/transformation.rs
View file

@ -31,6 +31,34 @@ pub struct AnthropicMessagesConfig;
pub const ANTHROPIC_MESSAGES_CONFIG: AnthropicMessagesConfig = AnthropicMessagesConfig;
impl BaseMessagesConfig for AnthropicMessagesConfig {
fn prepare_connection(
&self,
inputs: crate::base_llm::connection::ConnectionInputs<'_>,
headers: Headers,
model: &str,
_stream: bool,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<crate::base_llm::connection::PreparedConnection, Error> {
let validated = self.validate_environment(
headers,
inputs.api_key,
model,
inputs.litellm_params,
env_lookup,
)?;
let connection = crate::anthropic::connection::AnthropicConnection::resolve(
inputs.api_base,
inputs.runtime_defaults,
env_lookup,
)?;
Ok(crate::base_llm::connection::PreparedConnection {
url: connection.messages_url(),
headers: validated.headers,
auth: validated.auth,
api_base_source: connection.api_base_source(),
})
}
fn shape_request(
&self,
request: MessagesRequest,

1
litellm-rust/crates/llms/src/anthropic/mod.rs
View file

@ -3,6 +3,7 @@ pub mod common_utils;
pub mod batches;
pub mod chat;
pub mod connection;
pub mod count_tokens;
pub mod messages;

13
litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs
View file

@ -36,6 +36,19 @@ pub const AZURE_ANTHROPIC_MESSAGES_CONFIG: AzureAnthropicMessagesConfig =
AzureAnthropicMessagesConfig;
impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
fn prepare_connection(
&self,
inputs: crate::base_llm::connection::ConnectionInputs<'_>,
headers: Headers,
model: &str,
stream: bool,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<crate::base_llm::connection::PreparedConnection, Error> {
crate::base_llm::messages::transformation::prepare_from_parts(
self, inputs, headers, model, stream, env_lookup,
)
}
fn shape_request(
&self,
request: MessagesRequest,

60
litellm-rust/crates/llms/src/base_llm/connection.rs Normal file
View file

@ -0,0 +1,60 @@
use litellm_core_utils::{
settings::{Selected, SettingSource},
url_utils::ApiUrl,
};
use litellm_router_types::LitellmParams;
use serde::{Deserialize, Serialize};
use super::auth::{AuthScheme, Headers};
/// Defaults a host reads from its own runtime configuration. They are kept apart from a
/// call's explicit arguments so a provider resolver can rank them below every explicit
/// spelling and above the environment.
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct RuntimeDefaults {
pub api_base: Option<String>,
pub vertex_project: Option<String>,
pub vertex_location: Option<String>,
}
#[derive(Clone, Copy)]
pub struct ConnectionInputs<'a> {
pub api_key: Option<&'a str>,
pub api_base: Option<&'a str>,
pub litellm_params: &'a LitellmParams,
pub runtime_defaults: &'a RuntimeDefaults,
}
pub struct PreparedConnection {
pub url: String,
pub headers: Headers,
pub auth: AuthScheme,
pub api_base_source: SettingSource,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
#[error("api_base is not a valid base URL")]
pub struct InvalidApiBase;
impl From<InvalidApiBase> for crate::Error {
fn from(error: InvalidApiBase) -> Self {
Self::InvalidRequest(crate::ErrorDetail::invalid("api_base", error))
}
}
pub fn validated_api_base(selected: Selected<String>) -> Result<Selected<String>, InvalidApiBase> {
ApiUrl::parse(&selected.value)
.map(|_| selected)
.map_err(|_| InvalidApiBase)
}
pub fn complete_url(base: &str, path: &[&str]) -> Result<String, InvalidApiBase> {
ApiUrl::parse(base)
.and_then(|url| url.complete_path(path))
.map(ApiUrl::into_string)
.map_err(|_| InvalidApiBase)
}
pub fn path_segments(path: &str) -> impl Iterator<Item = &str> {
path.split('/').filter(|segment| !segment.is_empty())
}

67
litellm-rust/crates/llms/src/base_llm/messages/transformation.rs
View file

@ -5,7 +5,48 @@ use serde_json::Value;
use super::context::MessagesTransformContext;
pub use crate::base_llm::auth::{Headers, ValidatedEnvironment};
use crate::{Error, base_llm::messages::streaming::StreamDecoder};
use crate::{
Error,
base_llm::{
connection::{ConnectionInputs, PreparedConnection},
messages::streaming::StreamDecoder,
},
};
/// A connection from a config's own `validate_environment` and `get_complete_url`, for
/// providers whose settings have no shared resolver yet.
pub fn prepare_from_parts<C: BaseMessagesConfig + ?Sized>(
config: &C,
inputs: ConnectionInputs<'_>,
headers: Headers,
model: &str,
stream: bool,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<PreparedConnection, Error> {
let validated = config.validate_environment(
headers,
inputs.api_key,
model,
inputs.litellm_params,
env_lookup,
)?;
let url = config.get_complete_url(
inputs.api_base,
model,
inputs.litellm_params,
stream,
env_lookup,
)?;
Ok(PreparedConnection {
url,
headers: validated.headers,
auth: validated.auth,
api_base_source: match inputs.api_base {
Some(_) => litellm_core_utils::settings::SettingSource::Explicit,
None => litellm_core_utils::settings::SettingSource::Derived,
},
})
}
pub trait BaseMessagesConfig: Sync {
fn shape_request(
@ -55,6 +96,17 @@ pub trait BaseMessagesConfig: Sync {
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<ValidatedEnvironment, Error>;
/// The URL, headers and auth plan for one deployment, built from one selection of the
/// provider's connection settings.
fn prepare_connection(
&self,
inputs: ConnectionInputs<'_>,
headers: Headers,
model: &str,
stream: bool,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<PreparedConnection, Error>;
/// `None` relays the upstream bytes untouched, which is right for every host that already
/// speaks Anthropic SSE. A host on another wire returns the decoder that lifts its frames
/// into Anthropic stream events, and the route re-encodes those as Anthropic SSE.
@ -86,6 +138,19 @@ mod tests {
struct DefaultsConfig;
impl BaseMessagesConfig for DefaultsConfig {
fn prepare_connection(
&self,
inputs: crate::base_llm::connection::ConnectionInputs<'_>,
headers: Headers,
model: &str,
stream: bool,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<crate::base_llm::connection::PreparedConnection, Error> {
crate::base_llm::messages::transformation::prepare_from_parts(
self, inputs, headers, model, stream, env_lookup,
)
}
fn secret_names(&self) -> &'static [&'static str] {
&[]
}

1
litellm-rust/crates/llms/src/base_llm/mod.rs
View file

@ -2,6 +2,7 @@ pub mod audio_transcription;
pub mod auth;
pub mod base_model_iterator;
pub mod chat;
pub mod connection;
pub mod messages;
pub mod ocr;
pub mod responses;

1
litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs
View file

@ -168,6 +168,7 @@ pub struct PreparedOcrRequest {
pub optional_params: CallArguments,
pub input_sources: BTreeMap<String, InputSource>,
pub azure_ad_token_provider: Option<TokenProviderHandle>,
pub runtime_defaults: crate::base_llm::connection::RuntimeDefaults,
}
impl PreparedOcrRequest {

13
litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs
View file

@ -100,6 +100,19 @@ fn invoke_url(
}
impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig {
fn prepare_connection(
&self,
inputs: crate::base_llm::connection::ConnectionInputs<'_>,
headers: Headers,
model: &str,
stream: bool,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<crate::base_llm::connection::PreparedConnection, Error> {
crate::base_llm::messages::transformation::prepare_from_parts(
self, inputs, headers, model, stream, env_lookup,
)
}
fn shape_request(
&self,
request: MessagesRequest,

13
litellm-rust/crates/llms/src/deepseek/messages/transformation.rs
View file

@ -40,6 +40,19 @@ pub const DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG: DeepSeekAnthropicMessagesConfig =
DeepSeekAnthropicMessagesConfig;
impl BaseMessagesConfig for DeepSeekAnthropicMessagesConfig {
fn prepare_connection(
&self,
inputs: crate::base_llm::connection::ConnectionInputs<'_>,
headers: Headers,
model: &str,
stream: bool,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<crate::base_llm::connection::PreparedConnection, Error> {
crate::base_llm::messages::transformation::prepare_from_parts(
self, inputs, headers, model, stream, env_lookup,
)
}
fn shape_request(
&self,
request: MessagesRequest,

8
litellm-rust/crates/llms/src/vertex_ai/common_utils.rs
View file

@ -23,12 +23,16 @@ pub fn validate_vertex_location(location: &str) -> Result<&str, Error> {
if location == GLOBAL_LOCATION || is_location_token(location) {
return Ok(location);
}
Err(Error::Auth(litellm_auth::Error::InvalidConfiguration(
Err(invalid_location())
}
pub(super) fn invalid_location() -> Error {
Error::Auth(litellm_auth::Error::InvalidConfiguration(
litellm_auth::ErrorDetail::InvalidType {
field: "vertex_location".into(),
expected: "global or a lowercase alphanumeric token with hyphens",
},
)))
))
}
/// Python's `get_vertex_base_url`: the global control plane, a multi-region geography, or a

348
litellm-rust/crates/llms/src/vertex_ai/connection.rs Normal file
View file

@ -0,0 +1,348 @@
use litellm_auth::VertexParams;
use litellm_auth_gcp::{VertexConfig, get_vertex_ai_project_from_credentials};
use litellm_core_utils::settings::{Selected, SettingSource, select_non_blank};
use litellm_llms_types::providers::vertex_ai::{DEFAULT_LOCATION, GLOBAL_API_BASE};
use super::common_utils::{get_vertex_base_url, invalid_location, validate_vertex_location};
use crate::base_llm::{
connection::{InvalidApiBase, RuntimeDefaults, complete_url, validated_api_base},
ocr::error::Error as OcrError,
};
#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
pub enum VertexConnectionError {
#[error("vertex_location must be global or a lowercase alphanumeric token with hyphens")]
InvalidLocation,
#[error(transparent)]
InvalidApiBase(#[from] InvalidApiBase),
}
impl From<VertexConnectionError> for crate::Error {
fn from(error: VertexConnectionError) -> Self {
match error {
VertexConnectionError::InvalidLocation => invalid_location(),
VertexConnectionError::InvalidApiBase(error) => error.into(),
}
}
}
impl From<VertexConnectionError> for OcrError {
fn from(error: VertexConnectionError) -> Self {
let path = match error {
VertexConnectionError::InvalidLocation => "vertex_location",
VertexConnectionError::InvalidApiBase(_) => "api_base",
};
Self::RequestField { path: path.into() }
}
}
/// The project, location and host every Vertex AI operation sends to, chosen once per
/// deployment. Each comes from the explicit params, the host's runtime defaults, the
/// environment, then a derived or provider default: the project from the credentials, the
/// location `us-central1`, and the host from the location. `config` carries the same
/// project and location to authentication.
#[derive(Clone, Debug)]
pub struct VertexConnection {
pub project: Option<Selected<String>>,
pub location: Selected<String>,
pub api_base: Selected<String>,
pub config: VertexConfig,
}
impl VertexConnection {
pub fn resolve(
config: VertexConfig,
api_base: Option<&str>,
runtime_defaults: &RuntimeDefaults,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<Self, VertexConnectionError> {
let location = select_non_blank(
config.location(),
runtime_defaults.vertex_location.as_deref(),
env_lookup,
VertexParams::LOCATION.env,
)
.unwrap_or_else(|| {
Selected::new(DEFAULT_LOCATION.to_string(), SettingSource::ProviderDefault)
});
let derived_base = get_vertex_base_url(&location.value)
.map_err(|_| VertexConnectionError::InvalidLocation)?;
let project = select_non_blank(
config.project_id(),
runtime_defaults.vertex_project.as_deref(),
env_lookup,
VertexParams::PROJECT.env,
)
.or_else(|| {
get_vertex_ai_project_from_credentials(&config, env_lookup)
.map(|project| Selected::new(project, SettingSource::Derived))
});
let api_base = match select_non_blank(api_base, None, &|_| None, &[]) {
Some(custom) => validated_api_base(custom)?,
None => Selected::new(derived_base, SettingSource::Derived),
};
let config = config.with_project_and_location(
project.as_ref().map(|project| project.value.clone()),
location.value.clone(),
);
Ok(Self {
project,
location,
api_base,
config,
})
}
pub fn location(&self) -> &str {
&self.location.value
}
pub fn custom_api_base(&self) -> Option<&str> {
(self.api_base.source != SettingSource::Derived).then_some(self.api_base.value.as_str())
}
/// `{api_base}/v1/projects/{project}/locations/{location}/{resource...}`.
pub fn resource_url(
&self,
project: &str,
resource: &[&str],
) -> Result<String, VertexConnectionError> {
self.resource_url_on(&self.api_base.value, project, resource)
}
/// [`Self::resource_url`] for a publisher model Vertex serves only from the global host,
/// whatever location the path names; a custom base still wins.
pub fn global_resource_url(
&self,
project: &str,
resource: &[&str],
) -> Result<String, VertexConnectionError> {
let base = self.custom_api_base().unwrap_or(GLOBAL_API_BASE);
self.resource_url_on(base, project, resource)
}
fn resource_url_on(
&self,
base: &str,
project: &str,
resource: &[&str],
) -> Result<String, VertexConnectionError> {
validate_vertex_location(self.location())
.map_err(|_| VertexConnectionError::InvalidLocation)?;
let path = ["v1", "projects", project, "locations", self.location()]
.into_iter()
.chain(resource.iter().copied())
.collect::<Vec<_>>();
Ok(complete_url(base, &path)?)
}
}
#[cfg(test)]
mod tests {
use litellm_auth::{InputSource, SecretValue, Sourced};
use rstest::rstest;
use super::*;
fn env(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
move |name| {
values
.iter()
.find(|(key, _)| *key == name)
.map(|(_, value)| value.to_string())
}
}
fn explicit(project: Option<&str>, location: Option<&str>) -> VertexConfig {
VertexConfig::new(None, project.map(Into::into), location.map(Into::into))
}
fn runtime(project: Option<&str>, location: Option<&str>) -> RuntimeDefaults {
RuntimeDefaults {
vertex_project: project.map(Into::into),
vertex_location: location.map(Into::into),
..RuntimeDefaults::default()
}
}
#[rstest]
#[case::explicit_beats_runtime_and_environment(
explicit(Some("arg"), Some("europe-west4")),
runtime(Some("default"), Some("us-east5")),
&[("VERTEXAI_PROJECT", "env"), ("VERTEXAI_LOCATION", "asia-east1")],
("arg", SettingSource::Explicit),
("europe-west4", SettingSource::Explicit)
)]
#[case::runtime_defaults_beat_the_environment(
explicit(None, None),
runtime(Some("default"), Some("us-east5")),
&[("VERTEXAI_PROJECT", "env"), ("VERTEXAI_LOCATION", "asia-east1")],
("default", SettingSource::RuntimeDefault),
("us-east5", SettingSource::RuntimeDefault)
)]
#[case::environment_then_the_default_location(
explicit(None, None),
runtime(None, None),
&[("VERTEXAI_PROJECT", "env")],
("env", SettingSource::Environment("VERTEXAI_PROJECT")),
("us-central1", SettingSource::ProviderDefault)
)]
#[case::second_location_alias(
explicit(Some("arg"), None),
runtime(None, None),
&[("VERTEX_LOCATION", "europe-west1")],
("arg", SettingSource::Explicit),
("europe-west1", SettingSource::Environment("VERTEX_LOCATION"))
)]
fn project_and_location_follow_the_layer_precedence(
#[case] config: VertexConfig,
#[case] defaults: RuntimeDefaults,
#[case] vars: &'static [(&'static str, &'static str)],
#[case] project: (&str, SettingSource),
#[case] location: (&str, SettingSource),
) {
let connection = VertexConnection::resolve(config, None, &defaults, &env(vars)).unwrap();
assert_eq!(
connection.project,
Some(Selected::new(project.0.to_string(), project.1))
);
assert_eq!(
connection.location,
Selected::new(location.0.to_string(), location.1)
);
assert_eq!(connection.config.project_id(), Some(project.0));
assert_eq!(connection.config.location(), Some(location.0));
}
#[test]
fn the_project_falls_back_to_the_credentials() {
let config = VertexConfig::new(
Some(Sourced::new(
SecretValue::new(r#"{"type": "service_account", "project_id": "from-creds"}"#),
InputSource::Deployment,
)),
None,
Some("us-east5".into()),
);
let connection =
VertexConnection::resolve(config, None, &RuntimeDefaults::default(), &env(&[]))
.unwrap();
assert_eq!(
connection.project,
Some(Selected::new("from-creds".into(), SettingSource::Derived))
);
assert_eq!(connection.config.project_id(), Some("from-creds"));
}
#[rstest]
#[case::regional(
None,
"us-east5",
"https://us-east5-aiplatform.googleapis.com/v1/projects/proj/locations/us-east5/endpoints/openapi"
)]
#[case::global(
None,
"global",
"https://aiplatform.googleapis.com/v1/projects/proj/locations/global/endpoints/openapi"
)]
#[case::multi_region_geography(
None,
"eu",
"https://aiplatform.eu.rep.googleapis.com/v1/projects/proj/locations/eu/endpoints/openapi"
)]
#[case::custom_base_keeps_project_and_location(
Some("https://gateway.test/vertex"),
"europe-west4",
"https://gateway.test/vertex/v1/projects/proj/locations/europe-west4/endpoints/openapi"
)]
fn the_host_and_path_share_one_location(
#[case] api_base: Option<&str>,
#[case] location: &str,
#[case] expected: &str,
) {
let connection = VertexConnection::resolve(
explicit(Some("proj"), Some(location)),
api_base,
&RuntimeDefaults::default(),
&env(&[]),
)
.unwrap();
assert_eq!(
connection
.resource_url("proj", &["endpoints", "openapi"])
.unwrap(),
expected
);
}
#[rstest]
#[case::host_injection(None, "attacker.example/", VertexConnectionError::InvalidLocation)]
#[case::uppercase(None, "US-EAST5", VertexConnectionError::InvalidLocation)]
#[case::malformed_custom_base(
Some("not a url"),
"us-east5",
VertexConnectionError::InvalidApiBase(InvalidApiBase)
)]
fn an_unusable_connection_fails_before_sending(
#[case] api_base: Option<&str>,
#[case] location: &str,
#[case] expected: VertexConnectionError,
) {
assert_eq!(
VertexConnection::resolve(
explicit(Some("proj"), Some(location)),
api_base,
&RuntimeDefaults::default(),
&env(&[]),
)
.unwrap_err(),
expected
);
}
#[rstest]
#[case::regional_location_keeps_the_global_host(
None,
"us-central1",
"https://aiplatform.googleapis.com/v1/projects/p/locations/us-central1/endpoints/openapi"
)]
#[case::global_location(
None,
"global",
"https://aiplatform.googleapis.com/v1/projects/p/locations/global/endpoints/openapi"
)]
#[case::custom_base_wins(
Some("https://gateway.test/vertex"),
"us-central1",
"https://gateway.test/vertex/v1/projects/p/locations/us-central1/endpoints/openapi"
)]
fn global_only_models_use_the_global_host_with_the_resolved_location(
#[case] api_base: Option<&str>,
#[case] location: &str,
#[case] expected: &str,
) {
let connection = VertexConnection::resolve(
explicit(Some("p"), Some(location)),
api_base,
&RuntimeDefaults::default(),
&env(&[]),
)
.unwrap();
assert_eq!(
connection
.global_resource_url("p", &["endpoints", "openapi"])
.unwrap(),
expected
);
}
#[test]
fn global_only_models_still_reject_an_invalid_location() {
let config = explicit(Some("p"), Some("evil.com#"));
assert_eq!(
VertexConnection::resolve(config, None, &RuntimeDefaults::default(), &env(&[]))
.unwrap_err(),
VertexConnectionError::InvalidLocation
);
}
}

41
litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs
View file

@ -45,6 +45,45 @@ pub const VERTEX_ANTHROPIC_MESSAGES_CONFIG: VertexAiPartnerModelsAnthropicMessag
VertexAiPartnerModelsAnthropicMessagesConfig;
impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
fn prepare_connection(
&self,
inputs: crate::base_llm::connection::ConnectionInputs<'_>,
headers: Headers,
model: &str,
stream: bool,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<crate::base_llm::connection::PreparedConnection, Error> {
let connection = crate::vertex_ai::connection::VertexConnection::resolve(
VertexConfig::from_params(&inputs.litellm_params.vertex),
inputs.api_base,
inputs.runtime_defaults,
env_lookup,
)?;
let auth = match has_header(&headers, "authorization") {
true => AuthScheme::Forwarded,
false => AuthScheme::GcpAccessToken {
config: Box::new(connection.config.clone()),
},
};
let project = connection
.project
.as_ref()
.ok_or_else(|| Error::Auth(VertexParams::PROJECT.missing("Vertex AI")))?;
let url = complete_vertex_anthropic_url(
connection.custom_api_base(),
&project.value,
connection.location(),
model,
stream,
)?;
Ok(crate::base_llm::connection::PreparedConnection {
url,
headers,
auth,
api_base_source: connection.api_base.source,
})
}
fn shape_request(
&self,
request: MessagesRequest,
@ -435,7 +474,7 @@ mod tests {
));
assert_eq!(
error.to_string(),
"Missing Vertex AI project - pass vertex_project or vertex_ai_project, set litellm.vertex_project, or set VERTEXAI_PROJECT"
"Missing Vertex AI project - pass vertex_project or vertex_ai_project or set VERTEXAI_PROJECT"
);
}

1
litellm-rust/crates/llms/src/vertex_ai/mod.rs
View file

@ -1,3 +1,4 @@
pub mod common_utils;
pub mod connection;
pub mod messages;
pub mod ocr;

11
litellm-rust/crates/llms/src/vertex_ai/ocr/common_utils.rs
View file

@ -5,11 +5,18 @@ use crate::base_llm::ocr::{
error::Error,
transformation::{OcrConnection, PreparedOcrRequest},
};
use crate::vertex_ai::connection::VertexConnection;
pub(super) fn vertex_config(request: &PreparedOcrRequest) -> Result<VertexConfig, Error> {
Ok(VertexConfig::from_sourced_optional_params(
pub(super) fn vertex_connection(request: &PreparedOcrRequest) -> Result<VertexConnection, Error> {
let config = VertexConfig::from_sourced_optional_params(
&request.optional_params,
&request.input_sources,
)?;
Ok(VertexConnection::resolve(
config,
request.connection.api_base.as_deref(),
&request.runtime_defaults,
&|name: &str| request.connection.secret(name),
)?)
}

54
litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs
View file

@ -1,9 +1,9 @@
use litellm_auth_gcp as vertex;
use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl};
use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use super::{common_utils::vertex_config, transformation::VertexAiOcrConfig};
use super::{common_utils::vertex_connection, transformation::VertexAiOcrConfig};
use crate::base_llm::ocr::{
error::Error,
handler::OcrClient,
@ -16,7 +16,6 @@ use litellm_llms_types::formats::ocr::{
LiteLLMOcrResponse, OcrDocument, OcrPage, OcrPageDimensions, OcrPageImage, OcrResponseFormat,
OcrUsageInfo,
};
use litellm_llms_types::providers::vertex_ai::{DEFAULT_LOCATION, GLOBAL_API_BASE};
const MODEL_PREFIX: &str = "deepseek-ai/";
const DEEPSEEK_OCR_PARAMS: &[&str] = &["stream", "temperature", "max_tokens", "top_p", "n", "stop"];
@ -135,15 +134,10 @@ impl BaseOcrConfig for VertexAIDeepSeekOCRConfig {
_params: &Self::OcrParams,
environment: &Self::Environment,
) -> Result<String, Error> {
let config = vertex_config(request)?;
let location =
vertex::get_vertex_ai_location(&config, &|name: &str| request.connection.secret(name))
.unwrap_or_else(|| DEFAULT_LOCATION.to_string());
self.get_complete_url(
request.connection.api_base.as_deref(),
Ok(vertex_connection(request)?.global_resource_url(
&environment.project_id,
&location,
)
&["endpoints", "openapi", "chat", "completions"],
)?)
}
async fn async_transform_ocr_request(
@ -385,38 +379,6 @@ pub fn provider_model(model: &str) -> Result<String, Error> {
Ok(format!("{MODEL_PREFIX}{local_model}"))
}
impl VertexAIDeepSeekOCRConfig {
fn get_complete_url(
&self,
api_base: Option<&str>,
project: &str,
location: &str,
) -> Result<String, Error> {
let base = api_base
.map(str::trim)
.filter(|base| !base.is_empty())
.unwrap_or(GLOBAL_API_BASE);
ApiUrl::parse(base)
.and_then(|url| {
url.complete_path(&[
"v1",
"projects",
project,
"locations",
location,
"endpoints",
"openapi",
"chat",
"completions",
])
})
.map(|url| url.into_string())
.map_err(|_| Error::RequestField {
path: "api_base".into(),
})
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
@ -471,12 +433,6 @@ mod tests {
provider_model("deepseek-ai/deepseek-ocr-maas").unwrap(),
"deepseek-ai/deepseek-ocr-maas"
);
assert_eq!(
VertexAIDeepSeekOCRConfig
.get_complete_url(None, "proj-1", "europe-west4")
.unwrap(),
"https://aiplatform.googleapis.com/v1/projects/proj-1/locations/europe-west4/endpoints/openapi/chat/completions"
);
}
#[rstest]

103
litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs
View file

@ -1,8 +1,8 @@
use litellm_auth_gcp::{self as vertex, VertexConfig};
use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl};
use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams};
use serde_json::Value;
use super::common_utils::{validate_destination, vertex_config};
use super::common_utils::{validate_destination, vertex_connection};
use crate::{
base_llm::ocr::{
document::{inline_remote_document, validate_inline_document},
@ -16,8 +16,6 @@ use crate::{
};
use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrDocument, OcrResponseFormat};
use litellm_llms_types::providers::vertex_ai::DEFAULT_LOCATION;
#[derive(Clone, Debug, Default)]
pub struct VertexAiOcrConfig;
@ -51,8 +49,8 @@ impl BaseOcrConfig for VertexAiOcrConfig {
request: &PreparedOcrRequest,
client: &OcrClient,
) -> Result<Self::Environment, Error> {
let config = vertex_config(request)?;
self.resolve_environment(&request.connection, &config, client)
let connection = vertex_connection(request)?;
self.resolve_environment(&request.connection, &connection.config, client)
.await
}
@ -62,16 +60,11 @@ impl BaseOcrConfig for VertexAiOcrConfig {
_optional_params: &Self::OcrParams,
environment: &Self::Environment,
) -> Result<String, Error> {
let config = vertex_config(request)?;
let location =
vertex::get_vertex_ai_location(&config, &|name: &str| request.connection.secret(name))
.unwrap_or_else(|| DEFAULT_LOCATION.to_string());
self.build_ocr_url(
request.connection.api_base.as_deref(),
let prediction = format!("{}:rawPredict", request.model);
Ok(vertex_connection(request)?.resource_url(
&environment.project_id,
&location,
&request.model,
)
&["publishers", "mistralai", "models", &prediction],
)?)
}
fn transform_ocr_request(
@ -144,84 +137,4 @@ impl VertexAiOcrConfig {
.await
.map_err(Error::from)
}
fn build_ocr_url(
&self,
api_base: Option<&str>,
project: &str,
location: &str,
model: &str,
) -> Result<String, Error> {
validate_location(location)?;
let default_base = format!("https://{location}-aiplatform.googleapis.com");
let base = api_base
.map(str::trim)
.filter(|base| !base.is_empty())
.unwrap_or(&default_base);
let prediction = format!("{model}:rawPredict");
ApiUrl::parse(base)
.and_then(|url| {
url.complete_path(&[
"v1",
"projects",
project,
"locations",
location,
"publishers",
"mistralai",
"models",
&prediction,
])
})
.map(|url| url.into_string())
.map_err(|_| Error::RequestField {
path: "api_base".into(),
})
}
}
fn validate_location(location: &str) -> Result<(), Error> {
let valid = !location.is_empty()
&& location
.bytes()
.all(|value| value.is_ascii_lowercase() || value.is_ascii_digit() || value == b'-')
&& location
.as_bytes()
.first()
.is_some_and(u8::is_ascii_alphanumeric)
&& location
.as_bytes()
.last()
.is_some_and(u8::is_ascii_alphanumeric);
if valid {
return Ok(());
}
Err(Error::RequestField {
path: "vertex_location".into(),
})
}
#[cfg(test)]
mod tests {
use super::VertexAiOcrConfig;
#[test]
fn endpoint_uses_location_project_and_model() {
assert_eq!(
VertexAiOcrConfig
.build_ocr_url(None, "proj-1", "europe-west4", "mistral-ocr-maas")
.unwrap(),
"https://europe-west4-aiplatform.googleapis.com/v1/projects/proj-1/locations/europe-west4/publishers/mistralai/models/mistral-ocr-maas:rawPredict"
);
}
#[test]
fn endpoint_rejects_invalid_location() {
assert!(
VertexAiOcrConfig
.build_ocr_url(None, "proj-1", "attacker.example/path", "model")
.is_err()
);
}
}

1
litellm-rust/crates/python-bridge/src/lib.rs
View file

@ -12,6 +12,7 @@ mod marshal;
mod preflight;
mod python_settings;
mod routes;
mod runtime_defaults;
mod secrets;
mod tokenizer;

101
litellm-rust/crates/python-bridge/src/routes/messages/host.rs
View file

@ -2,6 +2,7 @@ use crate::cache::{CacheCall, Cached, PythonCache, Selection};
use litellm_host_python::{PythonHostCalls, PythonOwned};
use bytes::Bytes;
use litellm_auth::VertexParams;
use litellm_host_python::{InvokeError, PythonBinding, from_py, to_py};
use litellm_http::transport::Error as TransportError;
use litellm_inference_messages::{
@ -22,8 +23,8 @@ use serde_json::{Map, Value};
use crate::{
errors::{RustUpstreamError, route_error_to_pyerr},
marshal::{optional_timeout, project_optional_fields, public_response, python_timeout_seconds},
python_settings::missing_module,
routes::RequestView,
runtime_defaults::RuntimeDefault,
};
const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.messages.route_host";
@ -66,31 +67,13 @@ fn merge_headers(
(!merged.is_empty()).then_some(merged)
}
/// Reads `litellm.<name>`, the module global a param spec names, the way
/// `VertexBase.get_vertex_ai_project` reads `litellm.vertex_project`.
fn module_global<'py>(py: Python<'py>, name: &str) -> PyResult<Option<Bound<'py, PyAny>>> {
let module = match py.import("litellm") {
Ok(module) => module,
Err(error) if missing_module(py, &error, "litellm")? => return Ok(None),
Err(error) => return Err(error),
};
let value = module.getattr(name)?;
Ok((!value.is_none()).then_some(value))
}
/// The caller's litellm params, read from the kwargs by the names the typed params declare,
/// so a key the configs do not read is never converted; a spec whose spellings the call all
/// leaves out falls back to the module global it names, which stays the host's concern.
/// so a key the configs do not read is never converted.
fn project_litellm_params<'py>(
argument: impl Fn(&str) -> PyResult<Option<Bound<'py, PyAny>>>,
global: impl Fn(&str) -> PyResult<Option<Bound<'py, PyAny>>>,
) -> PyResult<Result<LitellmParams, Error>> {
let fields = project_optional_fields(LitellmParams::fields(), &argument)?;
let globals = LitellmParams::specs()
.filter(|spec| !spec.wire.iter().any(|name| fields.contains_key(*name)))
.filter_map(|spec| spec.module_global);
let folded = project_optional_fields(globals, &global)?;
Ok(litellm_params(fields.into_iter().chain(folded).collect()))
Ok(litellm_params(fields))
}
fn native_error(py: Python<'_>, error: Error) -> PyResult<PyErr> {
@ -161,7 +144,21 @@ impl MessagesPythonHost {
let api_base = string("api_base")?;
let extra_headers = self.merged_headers(py, arguments)?;
let provider_specific_header = self.provider_specific_header(py, arguments)?;
let litellm_params = project_litellm_params(argument, |name| module_global(py, name))?;
let litellm_params = project_litellm_params(argument)?;
let supplied = [VertexParams::PROJECT, VertexParams::LOCATION]
.iter()
.flat_map(|spec| spec.wire.iter().copied())
.filter_map(|name| argument(name).map(|value| value.map(|_| name)).transpose())
.collect::<PyResult<Vec<_>>>()?;
let wanted = api_base
.is_none()
.then_some(RuntimeDefault::ApiBase)
.into_iter()
.chain(RuntimeDefault::vertex_fallbacks(|spec| {
!spec.wire.iter().any(|name| supplied.contains(name))
}))
.collect::<Vec<_>>();
let runtime_defaults = crate::runtime_defaults::read(py, &wanted)?;
Ok(messages_body(body).and_then(|body| {
Ok(MessagesCall {
body,
@ -171,6 +168,7 @@ impl MessagesPythonHost {
provider_specific_header,
custom_llm_provider,
litellm_params: litellm_params?,
runtime_defaults,
timeout: optional_timeout(timeout),
shaping,
})
@ -410,58 +408,10 @@ mod tests {
#[case] kwargs: Value,
#[case] expected: Result<Value, ()>,
) {
assert_projection(kwargs, json!({}), expected);
assert_projection(kwargs, expected);
}
#[rstest]
#[case::global_fills_a_missing_vertex_project(
json!({"model": "m"}),
json!({"vertex_project": "from-global", "vertex_location": "us-east5"}),
Ok(json!({"model": "m", "vertex_project": "from-global", "vertex_location": "us-east5"})),
)]
#[case::the_call_wins_over_the_global(
json!({"model": "m", "vertex_project": "from-call"}),
json!({"vertex_project": "from-global"}),
Ok(json!({"model": "m", "vertex_project": "from-call"})),
)]
#[case::an_explicit_none_in_the_call_still_falls_back(
json!({"model": "m", "vertex_project": null}),
json!({"vertex_project": "from-global"}),
Ok(json!({"model": "m", "vertex_project": "from-global"})),
)]
#[case::a_global_of_the_wrong_type_is_a_request_error(
json!({"model": "m"}),
json!({"vertex_location": 5}),
Err(()),
)]
fn module_globals_fill_the_litellm_params_the_call_leaves_out(
#[case] kwargs: Value,
#[case] globals: Value,
#[case] expected: Result<Value, ()>,
) {
assert_projection(kwargs, globals, expected);
}
#[rstest]
#[case::a_global_without_a_spec_is_never_read(
json!({"model": "m"}),
json!({"aws_region_name": "eu-central-1", "api_key": "k"}),
Ok(json!({"model": "m"})),
)]
#[case::only_the_spec_named_global_is_read(
json!({"model": "m"}),
json!({"vertex_ai_project": "legacy-global", "vertex_project": "from-global"}),
Ok(json!({"model": "m", "vertex_project": "from-global"})),
)]
fn only_the_globals_the_specs_name_are_consulted(
#[case] kwargs: Value,
#[case] globals: Value,
#[case] expected: Result<Value, ()>,
) {
assert_projection(kwargs, globals, expected);
}
fn assert_projection(kwargs: Value, globals: Value, expected: Result<Value, ()>) {
fn assert_projection(kwargs: Value, expected: Result<Value, ()>) {
Python::initialize();
Python::attach(|py| {
let dict = |value: &Value| {
@ -472,13 +422,8 @@ mod tests {
.unwrap()
};
let kwargs = dict(&kwargs);
let globals = dict(&globals);
let bound = PyDict::new(py);
let projected = project_litellm_params(
|name| present(&kwargs, &bound, name),
|name| present(&globals, &bound, name),
)
.unwrap();
let projected = project_litellm_params(|name| present(&kwargs, &bound, name)).unwrap();
match (projected, expected) {
(Ok(projected), Ok(fields)) => assert_eq!(
projected,

171
litellm-rust/crates/python-bridge/src/routes/ocr/project.rs
View file

@ -1,5 +1,4 @@
use litellm_auth::SecretValue;
use litellm_auth::VertexParams;
use litellm_host_python::{from_py, present};
use litellm_inference_ocr::{
types::{LiteLLMOcrRequest, OcrDocumentInput},
@ -11,10 +10,9 @@ use serde_json::{Map, Value};
use super::{document::FileDocumentInput, errors::to_pyerr as ocr_error_to_pyerr};
use crate::{
coercion::FieldSpec,
credentials::{self, CallerTokenProvider},
marshal::{project_optional_fields, python_timeout_seconds, request_input_sources},
python_settings::{PythonSettings, Snapshot},
runtime_defaults::{self, RuntimeDefault},
};
/// What the host keeps after projection: the caller's token callable that answers the
@ -126,16 +124,16 @@ pub(super) fn project_request(
let names = specs.iter().map(|spec| spec.name).collect::<Vec<_>>();
let optional_params =
project_optional_fields(names.iter().copied(), |name| present(kwargs, bound, name))?;
let globals = module_globals_to_read(&optional_params, &names);
let defaults = if globals.is_empty() {
None
} else {
PythonSettings::ProviderDefaults.read_or_unset(bound.py())?
};
let optional_params = match defaults {
Some(defaults) => fold_module_globals(bound.py(), optional_params, &globals, &defaults)?,
None => optional_params,
};
let wanted = RuntimeDefault::vertex_fallbacks(|spec| {
spec.wire.iter().all(|name| names.contains(name))
&& !spec
.wire
.iter()
.any(|name| optional_params.contains_key(*name))
})
.collect::<Vec<_>>();
let runtime_defaults = runtime_defaults::read(bound.py(), &wanted)
.map_err(|error| settings_error(bound.py(), error))?;
let input_sources = request_input_sources(
kwargs,
names
@ -158,7 +156,9 @@ pub(super) fn project_request(
input_sources,
timeout_seconds,
};
let request = decode_request_input(wire).map_err(ocr_error_to_pyerr)?;
let request = decode_request_input(wire)
.map_err(ocr_error_to_pyerr)?
.with_runtime_defaults(runtime_defaults);
let provider = request.provider_name();
Ok((
request,
@ -169,50 +169,6 @@ pub(super) fn project_request(
))
}
/// Python's `VertexBase.get_vertex_ai_project` and `get_vertex_ai_location` fall back to a
/// `litellm.<name>` module global when a call names neither spelling: the globals of the
/// specs this route consumes whose wire names the call left out.
fn module_globals_to_read(
optional_params: &Map<String, Value>,
consumed: &[&str],
) -> Vec<&'static str> {
VertexParams::SPECS
.iter()
.filter(|spec| spec.wire.iter().all(|name| consumed.contains(name)))
.filter(|spec| {
!spec
.wire
.iter()
.any(|name| optional_params.contains_key(*name))
})
.filter_map(|spec| spec.module_global)
.collect()
}
/// Folds each module global in under the name its spec declares, skipping falsy values the
/// way Python's `or` chain does.
fn fold_module_globals(
py: Python<'_>,
optional_params: Map<String, Value>,
globals: &[&'static str],
defaults: &Snapshot<'_>,
) -> PyResult<Map<String, Value>> {
let read = globals
.iter()
.map(|name| {
let spec = FieldSpec::new(name, |field| field.falsy_optional_string());
Ok(defaults
.read(&spec)
.map_err(|error| settings_error(py, error.into()))?
.map(|value| (name.to_string(), Value::String(value))))
})
.collect::<PyResult<Vec<_>>>()?;
Ok(optional_params
.into_iter()
.chain(read.into_iter().flatten())
.collect())
}
/// A misconfigured litellm setting is the operator's error, not the request's or the
/// provider's, so the host raises it as is instead of mapping it onto a provider failure.
pub(super) const SETTINGS_ERROR_MARKER: &str = "native_settings_error";
@ -229,105 +185,6 @@ mod tests {
use super::*;
#[rstest::rstest]
#[case::global_fills_a_missing_project(
serde_json::json!({"vertex_location": "us-east5"}),
"vertex_project='from-global', vertex_location='ignored'",
serde_json::json!({"vertex_location": "us-east5", "vertex_project": "from-global"}),
)]
#[case::either_spelling_on_the_call_wins(
serde_json::json!({"vertex_ai_project": "from-call"}),
"vertex_project='from-global', vertex_location=None",
serde_json::json!({"vertex_ai_project": "from-call"}),
)]
#[case::falsy_globals_are_absent(
serde_json::json!({}),
"vertex_project=[], vertex_location=''",
serde_json::json!({}),
)]
fn module_globals_fill_the_vertex_params_the_call_leaves_out(
#[case] params: Value,
#[case] defaults: &str,
#[case] expected: Value,
) {
Python::initialize();
Python::attach(|py| {
let namespace = py
.eval(
&std::ffi::CString::new(format!(
"__import__('types').SimpleNamespace({defaults})"
))
.unwrap(),
None,
None,
)
.unwrap();
let snapshot = PythonSettings::ProviderDefaults.snapshot(namespace);
let consumed = VertexParams::fields().collect::<Vec<_>>();
let params = params.as_object().cloned().unwrap();
let globals = module_globals_to_read(&params, &consumed);
let folded = fold_module_globals(py, params, &globals, &snapshot).unwrap();
assert_eq!(Value::Object(folded), expected);
});
}
#[rstest::rstest]
fn a_bad_global_is_a_marked_settings_error() {
Python::initialize();
Python::attach(|py| {
let namespace = py
.eval(
c"__import__('types').SimpleNamespace(vertex_project=1)",
None,
None,
)
.unwrap();
let snapshot = PythonSettings::ProviderDefaults.snapshot(namespace);
let error =
fold_module_globals(py, Map::new(), &["vertex_project"], &snapshot).unwrap_err();
assert!(error.is_instance_of::<PyValueError>(py));
assert!(
error
.to_string()
.contains("provider_defaults.vertex_project")
);
assert!(
error
.value(py)
.getattr_opt(SETTINGS_ERROR_MARKER)
.unwrap()
.is_some()
);
});
}
#[rstest::rstest]
#[case::not_consumed(&["pages"], serde_json::json!({}), &[])]
#[case::consumed_and_absent(
&["vertex_project", "vertex_ai_project", "vertex_location", "vertex_ai_location"],
serde_json::json!({}),
&["vertex_project", "vertex_location"]
)]
#[case::consumed_and_present_in_either_spelling(
&["vertex_project", "vertex_ai_project", "vertex_location", "vertex_ai_location"],
serde_json::json!({"vertex_ai_project": "p"}),
&["vertex_location"]
)]
fn only_the_globals_of_consumed_absent_params_are_read(
#[case] consumed: &[&str],
#[case] params: Value,
#[case] expected: &[&str],
) {
assert_eq!(
module_globals_to_read(params.as_object().unwrap(), consumed),
expected
);
}
fn eval<'py>(py: Python<'py>, source: &std::ffi::CStr) -> Bound<'py, PyDict> {
let locals = PyDict::new(py);
py.run(source, Some(&locals), Some(&locals)).unwrap();

184
litellm-rust/crates/python-bridge/src/runtime_defaults.rs Normal file
View file

@ -0,0 +1,184 @@
use litellm_auth::{ParamSpec, VertexParams};
use litellm_llms::base_llm::connection::RuntimeDefaults;
use pyo3::prelude::*;
use crate::{
coercion::{FieldSpec, ProjectionError},
python_settings::{PythonSettings, Snapshot},
};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum RuntimeDefault {
ApiBase,
VertexProject,
VertexLocation,
}
impl RuntimeDefault {
const fn name(self) -> &'static str {
match self {
Self::ApiBase => "api_base",
Self::VertexProject => "vertex_project",
Self::VertexLocation => "vertex_location",
}
}
/// The Vertex defaults whose spec the call leaves `unset`.
pub(crate) fn vertex_fallbacks(
unset: impl Fn(&ParamSpec) -> bool,
) -> impl Iterator<Item = Self> {
[
(Self::VertexProject, VertexParams::PROJECT),
(Self::VertexLocation, VertexParams::LOCATION),
]
.into_iter()
.filter(move |(_, spec)| unset(spec))
.map(|(default, _)| default)
}
}
/// The `litellm.<name>` module defaults the call falls back to, read once per call into the
/// typed runtime-defaults layer that provider resolvers rank between explicit arguments and
/// the environment. A default the call does not fall back to is never read, so a malformed
/// one cannot fail an unrelated call.
pub(crate) fn read(py: Python<'_>, wanted: &[RuntimeDefault]) -> PyResult<RuntimeDefaults> {
if wanted.is_empty() {
return Ok(RuntimeDefaults::default());
}
match PythonSettings::ProviderDefaults.read_or_unset(py)? {
Some(snapshot) => project(&snapshot, wanted).map_err(PyErr::from),
None => Ok(RuntimeDefaults::default()),
}
}
fn project(
snapshot: &Snapshot<'_>,
wanted: &[RuntimeDefault],
) -> Result<RuntimeDefaults, ProjectionError> {
let read = |default: RuntimeDefault| match wanted.contains(&default) {
true => snapshot.read(&FieldSpec::new(default.name(), |field| {
field.falsy_optional_string()
})),
false => Ok(None),
};
Ok(RuntimeDefaults {
api_base: read(RuntimeDefault::ApiBase)?,
vertex_project: read(RuntimeDefault::VertexProject)?,
vertex_location: read(RuntimeDefault::VertexLocation)?,
})
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
const EVERY: [RuntimeDefault; 3] = [
RuntimeDefault::ApiBase,
RuntimeDefault::VertexProject,
RuntimeDefault::VertexLocation,
];
fn projected(
source: &std::ffi::CStr,
wanted: &[RuntimeDefault],
) -> Result<RuntimeDefaults, String> {
Python::initialize();
Python::attach(|py| {
let value = py.eval(source, None, None).unwrap();
project(&PythonSettings::ProviderDefaults.snapshot(value), wanted)
.map_err(|error| PyErr::from(error).value(py).to_string())
})
}
#[rstest]
#[case::every_default(
c"__import__('types').SimpleNamespace(api_base='https://runtime.test', vertex_project='p', vertex_location='us-east5')",
RuntimeDefaults {
api_base: Some("https://runtime.test".into()),
vertex_project: Some("p".into()),
vertex_location: Some("us-east5".into()),
}
)]
#[case::falsy_values_are_unset(
c"__import__('types').SimpleNamespace(api_base='', vertex_project=None, vertex_location=[])",
RuntimeDefaults::default()
)]
fn module_defaults_project_into_their_own_layer(
#[case] source: &std::ffi::CStr,
#[case] expected: RuntimeDefaults,
) {
assert_eq!(projected(source, &EVERY).unwrap(), expected);
}
#[rstest]
#[case::nothing_wanted(&[], RuntimeDefaults::default())]
#[case::only_the_api_base(
&[RuntimeDefault::ApiBase],
RuntimeDefaults { api_base: Some("https://runtime.test".into()), ..Default::default() }
)]
fn a_malformed_default_the_call_does_not_use_is_never_read(
#[case] wanted: &[RuntimeDefault],
#[case] expected: RuntimeDefaults,
) {
let source = c"__import__('types').SimpleNamespace(api_base='https://runtime.test', vertex_project=5, vertex_location=5)";
assert_eq!(projected(source, wanted).unwrap(), expected);
}
#[rstest]
#[case::both_unset(&[], vec![RuntimeDefault::VertexProject, RuntimeDefault::VertexLocation])]
#[case::project_on_the_call(&["vertex_ai_project"], vec![RuntimeDefault::VertexLocation])]
#[case::both_on_the_call(&["vertex_project", "vertex_location"], vec![])]
fn vertex_fallbacks_skip_the_settings_the_call_supplies(
#[case] supplied: &[&str],
#[case] expected: Vec<RuntimeDefault>,
) {
let fallbacks = RuntimeDefault::vertex_fallbacks(|spec| {
!spec.wire.iter().any(|name| supplied.contains(name))
})
.collect::<Vec<_>>();
assert_eq!(fallbacks, expected);
}
#[test]
fn a_misconfigured_module_default_names_the_setting() {
let error = projected(
c"__import__('types').SimpleNamespace(api_base=1, vertex_project=None, vertex_location=None)",
&EVERY,
)
.unwrap_err();
assert!(error.contains("provider_defaults.api_base"), "{error}");
}
#[test]
fn each_call_reads_the_current_module_defaults() {
Python::initialize();
Python::attach(|py| {
let namespace = py
.eval(
c"__import__('types').SimpleNamespace(api_base='https://first.test', vertex_project=None, vertex_location=None)",
None,
None,
)
.unwrap();
let first = project(
&PythonSettings::ProviderDefaults.snapshot(namespace.clone()),
&EVERY,
)
.unwrap();
namespace
.setattr("api_base", "https://second.test")
.unwrap();
let second = project(
&PythonSettings::ProviderDefaults.snapshot(namespace),
&EVERY,
)
.unwrap();
assert_eq!(
(first.api_base.as_deref(), second.api_base.as_deref()),
(Some("https://first.test"), Some("https://second.test"))
);
});
}
}

2
litellm/rust_bridge/settings.py
View file

@ -26,6 +26,7 @@ class UrlPolicy:
@dataclass(frozen=True, slots=True)
class ProviderDefaults:
api_base: object
vertex_project: object
vertex_location: object
enable_azure_ad_token_refresh: object
@ -112,6 +113,7 @@ def provider_defaults() -> ProviderDefaults:
import litellm
return ProviderDefaults(
api_base=litellm.api_base,
vertex_project=litellm.vertex_project,
vertex_location=litellm.vertex_location,
enable_azure_ad_token_refresh=litellm.enable_azure_ad_token_refresh,

2
tests/unit/rust_bridge/test_settings.py
View file

@ -195,11 +195,13 @@ def test_secret_manager_uses_key_management_defaults(monkeypatch: pytest.MonkeyP
def test_provider_defaults_read_the_litellm_globals(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "api_base", "https://runtime.example")
monkeypatch.setattr(litellm, "vertex_project", "configured-project")
monkeypatch.setattr(litellm, "vertex_location", "europe-west4")
monkeypatch.setattr(litellm, "enable_azure_ad_token_refresh", True)
assert settings.provider_defaults() == settings.ProviderDefaults(
api_base="https://runtime.example",
vertex_project="configured-project",
vertex_location="europe-west4",
enable_azure_ad_token_refresh=True,