From 27008cc355c25a46c2868801ef7b023caf5718d5 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Sun, 11 Oct 2026 00:31:51 +0000 Subject: [PATCH 1/3] fix(rust): resolve Anthropic and Vertex connections from one typed contract Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/crates/auth-gcp/src/config.rs | 8 + .../crates/auth-types/src/params/aws.rs | 7 +- .../crates/auth-types/src/params/mod.rs | 15 +- .../crates/auth-types/src/params/vertex.rs | 3 - .../crates/auth-types/tests/params.rs | 9 +- .../crates/core-utils/src/settings.rs | 72 +++++ .../crates/gateway-inference/src/messages.rs | 1 + .../crates/inference-messages/src/prepare.rs | 31 +- .../crates/inference-messages/src/types.rs | 1 + .../inference-messages/tests/caching.rs | 2 +- .../inference-messages/tests/messages/main.rs | 1 + .../tests/messages/request.rs | 11 + .../tests/messages/response.rs | 1 + .../tests/messages/secrets.rs | 2 + .../tests/messages/stream.rs | 2 + .../crates/inference-ocr/src/prepare.rs | 2 + .../crates/inference-ocr/src/types.rs | 12 + .../crates/llms/src/anthropic/common_utils.rs | 4 +- .../crates/llms/src/anthropic/connection.rs | 140 +++++++++ .../anthropic/count_tokens/transformation.rs | 97 +++++- .../src/anthropic/messages/transformation.rs | 28 ++ litellm-rust/crates/llms/src/anthropic/mod.rs | 1 + .../src/azure_ai/messages/transformation.rs | 13 + .../crates/llms/src/base_llm/connection.rs | 60 ++++ .../src/base_llm/messages/transformation.rs | 67 ++++- litellm-rust/crates/llms/src/base_llm/mod.rs | 1 + .../llms/src/base_llm/ocr/transformation.rs | 1 + .../anthropic_claude3_transformation.rs | 13 + .../src/deepseek/messages/transformation.rs | 13 + .../crates/llms/src/vertex_ai/common_utils.rs | 8 +- .../crates/llms/src/vertex_ai/connection.rs | 282 ++++++++++++++++++ .../src/vertex_ai/messages/transformation.rs | 41 ++- litellm-rust/crates/llms/src/vertex_ai/mod.rs | 1 + .../llms/src/vertex_ai/ocr/common_utils.rs | 11 +- .../vertex_ai/ocr/deepseek_transformation.rs | 54 +--- .../llms/src/vertex_ai/ocr/transformation.rs | 103 +------ litellm-rust/crates/python-bridge/src/lib.rs | 1 + .../python-bridge/src/routes/messages/host.rs | 86 +----- .../python-bridge/src/routes/ocr/project.rs | 163 +--------- .../python-bridge/src/runtime_defaults.rs | 96 ++++++ litellm/rust_bridge/settings.py | 2 + tests/unit/rust_bridge/test_settings.py | 2 + 42 files changed, 1037 insertions(+), 431 deletions(-) create mode 100644 litellm-rust/crates/llms/src/anthropic/connection.rs create mode 100644 litellm-rust/crates/llms/src/base_llm/connection.rs create mode 100644 litellm-rust/crates/llms/src/vertex_ai/connection.rs create mode 100644 litellm-rust/crates/python-bridge/src/runtime_defaults.rs diff --git a/litellm-rust/crates/auth-gcp/src/config.rs b/litellm-rust/crates/auth-gcp/src/config.rs index 0a76bf93cf0..4789196833a 100644 --- a/litellm-rust/crates/auth-gcp/src/config.rs +++ b/litellm-rust/crates/auth-gcp/src/config.rs @@ -50,6 +50,14 @@ impl VertexConfig { ) } + pub fn with_project_and_location(self, project_id: Option, location: String) -> Self { + Self::new( + self.credentials, + project_id.or(self.project_id), + Some(location), + ) + } + pub fn project_id(&self) -> Option<&str> { self.project_id.as_deref() } diff --git a/litellm-rust/crates/auth-types/src/params/aws.rs b/litellm-rust/crates/auth-types/src/params/aws.rs index 25fe52b2ce1..e36c5a8d9d7 100644 --- a/litellm-rust/crates/auth-types/src/params/aws.rs +++ b/litellm-rust/crates/auth-types/src/params/aws.rs @@ -11,12 +11,7 @@ const fn spec( wire: &'static [&'static str], env: &'static [&'static str], ) -> ParamSpec { - ParamSpec { - setting, - wire, - module_global: None, - env, - } + ParamSpec { setting, wire, env } } /// The `aws_*` fields of Python's `GenericLiteLLMParams`. diff --git a/litellm-rust/crates/auth-types/src/params/mod.rs b/litellm-rust/crates/auth-types/src/params/mod.rs index 88bc64cc2fc..f4b80928cce 100644 --- a/litellm-rust/crates/auth-types/src/params/mod.rs +++ b/litellm-rust/crates/auth-types/src/params/mod.rs @@ -5,13 +5,11 @@ pub use aws::AwsParams; pub use vertex::VertexParams; /// One connection param of Python's `CredentialLiteLLMParams` and every place Python reads -/// it from, in order: the wire names on a call or a deployment, the `litellm.` module -/// global a host folds in, then the environment names. +/// it from, in order: the wire names on a call or a deployment, then the environment names. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ParamSpec { pub setting: &'static str, pub wire: &'static [&'static str], - pub module_global: Option<&'static str>, pub env: &'static [&'static str], } @@ -38,11 +36,10 @@ impl ParamSpec { } pub(crate) fn guidance(&self) -> String { - let wire = self.wire.join(" or "); - let env = self.env.join(" or "); - match self.module_global { - Some(global) => format!("pass {wire}, set litellm.{global}, or set {env}"), - None => format!("pass {wire} or set {env}"), - } + format!( + "pass {} or set {}", + self.wire.join(" or "), + self.env.join(" or ") + ) } } diff --git a/litellm-rust/crates/auth-types/src/params/vertex.rs b/litellm-rust/crates/auth-types/src/params/vertex.rs index 06ec3008dfb..217441ee4fe 100644 --- a/litellm-rust/crates/auth-types/src/params/vertex.rs +++ b/litellm-rust/crates/auth-types/src/params/vertex.rs @@ -28,19 +28,16 @@ impl VertexParams { pub const CREDENTIALS: ParamSpec = ParamSpec { setting: "credentials", wire: &["vertex_credentials", "vertex_ai_credentials"], - module_global: None, env: &["VERTEXAI_CREDENTIALS"], }; pub const PROJECT: ParamSpec = ParamSpec { setting: "project", wire: &["vertex_project", "vertex_ai_project"], - module_global: Some("vertex_project"), env: &["VERTEXAI_PROJECT"], }; pub const LOCATION: ParamSpec = ParamSpec { setting: "location", wire: &["vertex_location", "vertex_ai_location"], - module_global: Some("vertex_location"), env: &["VERTEXAI_LOCATION", "VERTEX_LOCATION"], }; diff --git a/litellm-rust/crates/auth-types/tests/params.rs b/litellm-rust/crates/auth-types/tests/params.rs index 9d488b170e1..35482591d81 100644 --- a/litellm-rust/crates/auth-types/tests/params.rs +++ b/litellm-rust/crates/auth-types/tests/params.rs @@ -4,7 +4,6 @@ use rstest::rstest; const LOCATION: ParamSpec = ParamSpec { setting: "location", wire: &["vertex_location", "vertex_ai_location"], - module_global: Some("vertex_location"), env: &["VERTEXAI_LOCATION", "VERTEX_LOCATION"], }; @@ -51,13 +50,13 @@ fn resolution_walks_wire_names_then_environment_names( } #[rstest] -#[case::without_a_module_global( +#[case::aws( &AwsParams::REGION, "Missing AWS region - pass aws_region_name or set AWS_REGION_NAME or AWS_REGION" )] -#[case::with_a_module_global( +#[case::several_wire_names( &LOCATION, - "Missing AWS location - pass vertex_location or vertex_ai_location, set litellm.vertex_location, or set VERTEXAI_LOCATION or VERTEX_LOCATION" + "Missing AWS location - pass vertex_location or vertex_ai_location or set VERTEXAI_LOCATION or VERTEX_LOCATION" )] fn a_missing_param_names_every_place_the_spec_reads( #[case] spec: &'static ParamSpec, @@ -68,5 +67,5 @@ fn a_missing_param_names_every_place_the_spec_reads( assert_eq!(message, expected); assert!(spec.wire.iter().all(|name| message.contains(name))); assert!(spec.env.iter().all(|name| message.contains(name))); - assert_eq!(message.contains("litellm."), spec.module_global.is_some()); + assert!(!message.contains("litellm.")); } diff --git a/litellm-rust/crates/core-utils/src/settings.rs b/litellm-rust/crates/core-utils/src/settings.rs index 6b472295508..290460203f4 100644 --- a/litellm-rust/crates/core-utils/src/settings.rs +++ b/litellm-rust/crates/core-utils/src/settings.rs @@ -53,6 +53,55 @@ pub fn resolve_non_empty( }) } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SettingSource { + Explicit, + RuntimeDefault, + Environment(&'static str), + Derived, + ProviderDefault, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Selected { + pub value: T, + pub source: SettingSource, +} + +impl Selected { + pub fn new(value: T, source: SettingSource) -> Self { + Self { value, source } + } +} + +/// The first nonblank value of an explicit argument, a runtime default, then each +/// environment name in order, keeping which layer supplied it. +pub fn select_non_blank( + explicit: Option<&str>, + runtime_default: Option<&str>, + env_lookup: &dyn Fn(&str) -> Option, + env_names: &[&'static str], +) -> Option> { + [ + (SettingSource::Explicit, explicit), + (SettingSource::RuntimeDefault, runtime_default), + ] + .into_iter() + .find_map(|(source, value)| { + value + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(|value| Selected::new(value.to_string(), source)) + }) + .or_else(|| { + env_names.iter().find_map(|name| { + env_lookup(name) + .filter(|value| !value.trim().is_empty()) + .map(|value| Selected::new(value, SettingSource::Environment(name))) + }) + }) +} + pub trait Layer: Default { fn or(self, lower: Self) -> Self; } @@ -159,4 +208,27 @@ mod tests { fn merging_no_layers_yields_the_empty_layer() { assert_eq!(merge(Vec::::new()), Pair::default()); } + + #[rstest::rstest] + #[case::explicit_beats_everything(Some("arg"), Some("default"), &[("FIRST", "env")], Some(("arg", SettingSource::Explicit)))] + #[case::runtime_default_beats_the_environment(None, Some("default"), &[("FIRST", "env")], Some(("default", SettingSource::RuntimeDefault)))] + #[case::blank_layers_are_skipped(Some(" "), Some(""), &[("FIRST", " "), ("SECOND", "env")], Some(("env", SettingSource::Environment("SECOND"))))] + #[case::environment_names_in_order(None, None, &[("SECOND", "b"), ("FIRST", "a")], Some(("a", SettingSource::Environment("FIRST"))))] + #[case::nothing_set(None, None, &[], None)] + fn selection_keeps_the_layer_that_supplied_the_value( + #[case] explicit: Option<&str>, + #[case] runtime_default: Option<&str>, + #[case] env: &'static [(&'static str, &'static str)], + #[case] expected: Option<(&str, SettingSource)>, + ) { + assert_eq!( + select_non_blank( + explicit, + runtime_default, + &env_of(env), + &["FIRST", "SECOND"] + ), + expected.map(|(value, source)| Selected::new(value.to_string(), source)) + ); + } } diff --git a/litellm-rust/crates/gateway-inference/src/messages.rs b/litellm-rust/crates/gateway-inference/src/messages.rs index e56308b7ae1..98f5dd3f128 100644 --- a/litellm-rust/crates/gateway-inference/src/messages.rs +++ b/litellm-rust/crates/gateway-inference/src/messages.rs @@ -80,6 +80,7 @@ fn project( api_base: deployment.api_base.clone(), custom_llm_provider: deployment.custom_llm_provider.clone(), litellm_params: deployment.litellm_params.clone(), + runtime_defaults: Default::default(), extra_headers: None, provider_specific_header: anthropic_api_headers(headers), timeout: deployment.timeout, diff --git a/litellm-rust/crates/inference-messages/src/prepare.rs b/litellm-rust/crates/inference-messages/src/prepare.rs index 50a079d5eb3..b830f4ced0c 100644 --- a/litellm-rust/crates/inference-messages/src/prepare.rs +++ b/litellm-rust/crates/inference-messages/src/prepare.rs @@ -7,7 +7,8 @@ use litellm_core_utils::{ }; use litellm_http::request::with_default_headers; use litellm_llms::base_llm::{ - auth::ValidatedEnvironment, messages::context::MessagesTransformContext, + auth::ValidatedEnvironment, connection::ConnectionInputs, + messages::context::MessagesTransformContext, }; use litellm_llms_types::formats::messages::MessagesRequest; use litellm_secrets::source::SecretSource; @@ -70,6 +71,7 @@ fn prepare_provider_request( api_key, api_base, litellm_params, + runtime_defaults, extra_headers, provider_specific_header, timeout, @@ -95,28 +97,26 @@ fn prepare_provider_request( let forwarded = string_headers(Some( extra_headers.into_iter().flatten().chain(scoped).collect(), ))?; - let validated = config.validate_environment( + let connection = config.prepare_connection( + ConnectionInputs { + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + litellm_params: &litellm_params, + runtime_defaults: &runtime_defaults, + }, forwarded, - api_key.as_deref(), &transformed.model, - &litellm_params, + transformed.params.stream == Some(true), &env_lookup, )?; let environment = ValidatedEnvironment { headers: config.request_headers( - with_default_headers(validated.headers, config.default_headers()), + with_default_headers(connection.headers, config.default_headers()), &transformed, ), - auth: validated.auth, + auth: connection.auth, }; - - let url = config.get_complete_url( - api_base.as_deref(), - &transformed.model, - &litellm_params, - transformed.params.stream == Some(true), - &env_lookup, - )?; + let url = connection.url; Ok(ProviderMessagesRequest { provider, @@ -232,6 +232,7 @@ mod tests { api_base: None, custom_llm_provider: Some("anthropic".into()), litellm_params: Default::default(), + runtime_defaults: Default::default(), extra_headers: None, provider_specific_header: None, timeout: None, @@ -259,6 +260,7 @@ mod tests { api_base: Some("https://anthropic.test".into()), custom_llm_provider: Some("anthropic".into()), litellm_params: Default::default(), + runtime_defaults: Default::default(), extra_headers: None, provider_specific_header: None, timeout: None, @@ -367,6 +369,7 @@ mod tests { api_base: Some("https://resource.services.ai.azure.com".into()), custom_llm_provider: custom_llm_provider.map(Into::into), litellm_params: Default::default(), + runtime_defaults: Default::default(), extra_headers: Some(Map::from_iter([("x-priority".into(), json!("extra"))])), provider_specific_header: Some(configured), timeout: None, diff --git a/litellm-rust/crates/inference-messages/src/types.rs b/litellm-rust/crates/inference-messages/src/types.rs index 669f7b283a1..f926f51c480 100644 --- a/litellm-rust/crates/inference-messages/src/types.rs +++ b/litellm-rust/crates/inference-messages/src/types.rs @@ -19,6 +19,7 @@ pub struct MessagesCall { pub api_base: Option, pub custom_llm_provider: Option, pub litellm_params: LitellmParams, + pub runtime_defaults: litellm_llms::base_llm::connection::RuntimeDefaults, pub extra_headers: Option>, pub provider_specific_header: Option, pub timeout: Option, diff --git a/litellm-rust/crates/inference-messages/tests/caching.rs b/litellm-rust/crates/inference-messages/tests/caching.rs index e9005123119..3feb59d9061 100644 --- a/litellm-rust/crates/inference-messages/tests/caching.rs +++ b/litellm-rust/crates/inference-messages/tests/caching.rs @@ -206,7 +206,7 @@ async fn messages_cache_identity_follows_resolved_configuration_and_request_call let cache = ScopedCache::new(cache.clone(), CacheScope::Shared); support::messages_route(secrets.clone()).with_cache(cache).execute(MessagesCall { body: serde_json::from_value(json!({"model":"anthropic/cache-test-model","messages":[{"role":"user","content":"hello"}],"max_tokens":32})).unwrap(), - api_key:None,api_base:None,custom_llm_provider:None,litellm_params:Default::default(),extra_headers:None,provider_specific_header:None,timeout:None,shaping:Default::default(), + api_key:None,api_base:None,custom_llm_provider:None,litellm_params:Default::default(),runtime_defaults:Default::default(),extra_headers:None,provider_specific_header:None,timeout:None,shaping:Default::default(), }, &hooks, None).await.unwrap(); } assert_eq!(hooks.calls.load(Ordering::SeqCst), 4); diff --git a/litellm-rust/crates/inference-messages/tests/messages/main.rs b/litellm-rust/crates/inference-messages/tests/messages/main.rs index 8f92b31c999..436e436603c 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/main.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/main.rs @@ -82,6 +82,7 @@ fn call() -> MessagesCall { api_base: None, custom_llm_provider: Some("anthropic".into()), litellm_params: Default::default(), + runtime_defaults: Default::default(), extra_headers: None, provider_specific_header: None, timeout: Some(Duration::from_secs(5)), diff --git a/litellm-rust/crates/inference-messages/tests/messages/request.rs b/litellm-rust/crates/inference-messages/tests/messages/request.rs index 25a318ecad5..44a5d639cc9 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/request.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/request.rs @@ -69,6 +69,7 @@ async fn credentials_become_exactly_one_auth_header( run_message(MessagesCall { custom_llm_provider: Some(provider.into()), litellm_params: Default::default(), + runtime_defaults: Default::default(), api_key: api_key.map(Into::into), api_base: Some(upstream.uri()), extra_headers: headers(extra_headers.iter().copied()), @@ -98,6 +99,7 @@ async fn a_call_without_credentials_fails_before_sending( let error = run(MessagesCall { custom_llm_provider: Some(provider.into()), litellm_params: Default::default(), + runtime_defaults: Default::default(), api_base: Some(upstream.uri()), ..call }) @@ -140,6 +142,7 @@ async fn each_provider_posts_to_its_messages_endpoint( run_message(MessagesCall { custom_llm_provider: provider.map(Into::into), litellm_params: Default::default(), + runtime_defaults: Default::default(), api_key: Some("sk".into()), api_base: Some(format!("{}{base_suffix}", upstream.uri())), ..with_model(call, model) @@ -171,6 +174,7 @@ async fn unsupported_providers_are_rejected_before_sending( let error = run(MessagesCall { custom_llm_provider: provider.map(Into::into), litellm_params: Default::default(), + runtime_defaults: Default::default(), api_key: Some("sk".into()), api_base: Some(UNREACHABLE_BASE.into()), ..with_model(call, model) @@ -224,6 +228,7 @@ async fn cache_scope_removal_is_selected_by_the_provider( run_message(MessagesCall { custom_llm_provider: Some(provider.into()), litellm_params: Default::default(), + runtime_defaults: Default::default(), api_key: Some("sk-azure".into()), api_base: Some(upstream.uri()), body: body(json!({ @@ -369,6 +374,7 @@ async fn caller_protocol_headers_win_over_the_defaults(call: MessagesCall, #[cas run_message(MessagesCall { custom_llm_provider: Some(provider.into()), litellm_params: Default::default(), + runtime_defaults: Default::default(), api_key: Some("sk".into()), api_base: Some(upstream.uri()), extra_headers: headers([ @@ -421,6 +427,7 @@ async fn unsupported_params_are_dropped_under_drop_params_and_rejected_without_i body: call.body.clone(), custom_llm_provider: call.custom_llm_provider.clone(), litellm_params: Default::default(), + runtime_defaults: Default::default(), extra_headers: None, provider_specific_header: None, timeout: call.timeout, @@ -600,6 +607,7 @@ async fn metadata_is_reduced_to_the_user_id(call: MessagesCall, #[case] provider MessagesCall { custom_llm_provider: Some(provider.into()), litellm_params: Default::default(), + runtime_defaults: Default::default(), api_key: Some("sk".into()), api_base: Some(upstream.uri()), ..call @@ -625,6 +633,7 @@ async fn deepseek_sends_neither_billing_blocks_nor_the_custom_tool_discriminator MessagesCall { custom_llm_provider: Some("deepseek".into()), litellm_params: Default::default(), + runtime_defaults: Default::default(), api_key: Some("sk".into()), api_base: Some(upstream.uri()), ..call @@ -747,6 +756,7 @@ async fn system_message_folding_is_selected_by_the_provider( MessagesCall { custom_llm_provider: Some(provider.into()), litellm_params: Default::default(), + runtime_defaults: Default::default(), api_key: Some("sk-azure".into()), api_base: Some(upstream.uri()), shaping: MessagesShaping { @@ -812,6 +822,7 @@ async fn provider_validation_runs_before_caller_parameter_removal( MessagesCall { custom_llm_provider: Some(provider.into()), litellm_params: Default::default(), + runtime_defaults: Default::default(), api_key: Some("sk-test".into()), api_base: Some(upstream.uri()), shaping: MessagesShaping { diff --git a/litellm-rust/crates/inference-messages/tests/messages/response.rs b/litellm-rust/crates/inference-messages/tests/messages/response.rs index b5d623a5582..f8ce8ec7a83 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/response.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/response.rs @@ -105,6 +105,7 @@ async fn the_provider_message_is_returned(call: MessagesCall, #[case] provider: let message = run_message(MessagesCall { custom_llm_provider: Some(provider.into()), litellm_params: Default::default(), + runtime_defaults: Default::default(), api_key: Some("sk".into()), api_base: Some(upstream.uri()), ..call diff --git a/litellm-rust/crates/inference-messages/tests/messages/secrets.rs b/litellm-rust/crates/inference-messages/tests/messages/secrets.rs index 203cc620d35..eeef10dd972 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/secrets.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/secrets.rs @@ -46,6 +46,7 @@ async fn the_credential_and_base_come_from_the_secret_source( MessagesCall { custom_llm_provider: Some(provider.into()), litellm_params: Default::default(), + runtime_defaults: Default::default(), ..call }, ) @@ -198,6 +199,7 @@ async fn azure_without_a_base_anywhere_fails_before_sending(call: MessagesCall) MessagesCall { custom_llm_provider: Some("azure_ai".into()), litellm_params: Default::default(), + runtime_defaults: Default::default(), ..call }, ) diff --git a/litellm-rust/crates/inference-messages/tests/messages/stream.rs b/litellm-rust/crates/inference-messages/tests/messages/stream.rs index 049a620e454..a718f776e17 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/stream.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/stream.rs @@ -347,6 +347,7 @@ async fn the_sdk_returns_stream_headers_and_every_sse_byte( MessagesCall { custom_llm_provider: Some(provider.into()), litellm_params: Default::default(), + runtime_defaults: Default::default(), ..streaming(call, upstream.uri()) }, &(), @@ -472,6 +473,7 @@ async fn a_host_on_anthropic_sse_is_relayed_byte_for_byte(call: MessagesCall) { MessagesCall { custom_llm_provider: Some("azure_ai".into()), litellm_params: Default::default(), + runtime_defaults: Default::default(), ..streaming(call, upstream.uri()) }, usize::MAX, diff --git a/litellm-rust/crates/inference-ocr/src/prepare.rs b/litellm-rust/crates/inference-ocr/src/prepare.rs index 14120606864..d6d770d0e16 100644 --- a/litellm-rust/crates/inference-ocr/src/prepare.rs +++ b/litellm-rust/crates/inference-ocr/src/prepare.rs @@ -54,6 +54,7 @@ pub(crate) fn prepare_request( optional_params, input_sources, azure_ad_token_provider, + runtime_defaults, .. } = request; PreparedOcrRequest { @@ -64,6 +65,7 @@ pub(crate) fn prepare_request( optional_params, input_sources, azure_ad_token_provider, + runtime_defaults, } } diff --git a/litellm-rust/crates/inference-ocr/src/types.rs b/litellm-rust/crates/inference-ocr/src/types.rs index fda65e3284e..c04c1a83886 100644 --- a/litellm-rust/crates/inference-ocr/src/types.rs +++ b/litellm-rust/crates/inference-ocr/src/types.rs @@ -1,3 +1,4 @@ +use litellm_llms::base_llm::connection::RuntimeDefaults; use std::{collections::BTreeMap, path::PathBuf, time::Duration}; use bytes::Bytes; @@ -81,6 +82,7 @@ pub struct LiteLLMOcrRequest { pub optional_params: CallArguments, pub input_sources: BTreeMap, pub azure_ad_token_provider: Option, + pub runtime_defaults: RuntimeDefaults, pub(crate) config: OcrConfigKind, } @@ -123,12 +125,20 @@ impl LiteLLMOcrRequest { optional_params, input_sources: BTreeMap::new(), azure_ad_token_provider: None, + runtime_defaults: RuntimeDefaults::default(), config, }) } } impl LiteLLMOcrRequest { + pub fn with_runtime_defaults(self, runtime_defaults: RuntimeDefaults) -> Self { + Self { + runtime_defaults, + ..self + } + } + pub fn map_document( self, map: impl FnOnce(D) -> Result, @@ -141,6 +151,7 @@ impl LiteLLMOcrRequest { optional_params: self.optional_params, input_sources: self.input_sources, azure_ad_token_provider: self.azure_ad_token_provider, + runtime_defaults: self.runtime_defaults, config: self.config, }) } @@ -154,6 +165,7 @@ impl LiteLLMOcrRequest { optional_params: self.optional_params, input_sources: self.input_sources, azure_ad_token_provider: self.azure_ad_token_provider, + runtime_defaults: self.runtime_defaults, config: self.config, } } diff --git a/litellm-rust/crates/llms/src/anthropic/common_utils.rs b/litellm-rust/crates/llms/src/anthropic/common_utils.rs index 7991a57353e..cf41e4ea985 100644 --- a/litellm-rust/crates/llms/src/anthropic/common_utils.rs +++ b/litellm-rust/crates/llms/src/anthropic/common_utils.rs @@ -150,8 +150,10 @@ pub fn complete_anthropic_url( api_base: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, ) -> String { - let api_base = resolve_anthropic_api_base(api_base, env_lookup); + messages_url(&resolve_anthropic_api_base(api_base, env_lookup)) +} +pub fn messages_url(api_base: &str) -> String { let api_base = api_base.trim_end_matches('/'); if api_base.ends_with(MESSAGES_PATH) { return api_base.to_string(); diff --git a/litellm-rust/crates/llms/src/anthropic/connection.rs b/litellm-rust/crates/llms/src/anthropic/connection.rs new file mode 100644 index 00000000000..78ebdcc085e --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/connection.rs @@ -0,0 +1,140 @@ +use litellm_core_utils::settings::{Selected, SettingSource, select_non_blank}; +use litellm_llms_types::providers::anthropic::API_BASE; + +use super::common_utils::{ANTHROPIC_API_BASE_ENV, ANTHROPIC_BASE_URL_ENV, messages_url}; +use crate::base_llm::connection::{ + InvalidApiBase, RuntimeDefaults, complete_url, path_segments, validated_api_base, +}; + +/// The Anthropic host every Anthropic operation sends to, chosen once from the explicit +/// `api_base`, the host's runtime default, `ANTHROPIC_API_BASE`, `ANTHROPIC_BASE_URL`, then +/// the public API. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AnthropicConnection { + api_base: Selected, +} + +impl AnthropicConnection { + pub fn resolve( + api_base: Option<&str>, + runtime_defaults: &RuntimeDefaults, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + let api_base = select_non_blank( + api_base, + runtime_defaults.api_base.as_deref(), + env_lookup, + &[ANTHROPIC_API_BASE_ENV, ANTHROPIC_BASE_URL_ENV], + ) + .unwrap_or_else(|| Selected::new(API_BASE.to_string(), SettingSource::ProviderDefault)); + Ok(Self { + api_base: validated_api_base(api_base)?, + }) + } + + pub fn api_base(&self) -> &str { + &self.api_base.value + } + + pub fn api_base_source(&self) -> SettingSource { + self.api_base.source + } + + pub fn messages_url(&self) -> String { + messages_url(self.api_base()) + } + + pub fn endpoint(&self, path: &str) -> Result { + complete_url( + self.api_base().trim_end_matches('/'), + &path_segments(path).collect::>(), + ) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn env(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option { + move |name| { + values + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.to_string()) + } + } + + fn runtime(api_base: Option<&str>) -> RuntimeDefaults { + RuntimeDefaults { + api_base: api_base.map(Into::into), + ..RuntimeDefaults::default() + } + } + + #[rstest] + #[case::explicit_beats_both_aliases( + Some("https://explicit.test"), + None, + &[("ANTHROPIC_API_BASE", "https://a.test"), ("ANTHROPIC_BASE_URL", "https://b.test")], + "https://explicit.test", + SettingSource::Explicit + )] + #[case::runtime_default_beats_the_environment( + None, + Some("https://runtime.test"), + &[("ANTHROPIC_API_BASE", "https://a.test")], + "https://runtime.test", + SettingSource::RuntimeDefault + )] + #[case::api_base_alias_before_base_url_alias( + None, + None, + &[("ANTHROPIC_BASE_URL", "https://b.test"), ("ANTHROPIC_API_BASE", "https://a.test")], + "https://a.test", + SettingSource::Environment("ANTHROPIC_API_BASE") + )] + #[case::base_url_alias_alone( + None, + None, + &[("ANTHROPIC_BASE_URL", "https://b.test")], + "https://b.test", + SettingSource::Environment("ANTHROPIC_BASE_URL") + )] + #[case::blank_overrides_fall_through( + Some(" "), + Some(""), + &[], + "https://api.anthropic.com", + SettingSource::ProviderDefault + )] + fn the_api_base_follows_the_layer_precedence( + #[case] explicit: Option<&str>, + #[case] runtime_default: Option<&str>, + #[case] vars: &'static [(&'static str, &'static str)], + #[case] expected: &str, + #[case] source: SettingSource, + ) { + let connection = + AnthropicConnection::resolve(explicit, &runtime(runtime_default), &env(vars)).unwrap(); + assert_eq!( + (connection.api_base(), connection.api_base_source()), + (expected, source) + ); + } + + #[rstest] + #[case::explicit("not a url", None)] + #[case::runtime_default(" ", Some("relative/path"))] + fn a_malformed_override_is_rejected( + #[case] explicit: &str, + #[case] runtime_default: Option<&str>, + ) { + assert_eq!( + AnthropicConnection::resolve(Some(explicit), &runtime(runtime_default), &env(&[])), + Err(InvalidApiBase) + ); + } +} diff --git a/litellm-rust/crates/llms/src/anthropic/count_tokens/transformation.rs b/litellm-rust/crates/llms/src/anthropic/count_tokens/transformation.rs index 9b2ee44516c..c21b04cc112 100644 --- a/litellm-rust/crates/llms/src/anthropic/count_tokens/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/count_tokens/transformation.rs @@ -1,13 +1,16 @@ use litellm_llms_types::{ formats::messages::{Message, SystemPrompt}, providers::anthropic::{ - API_BASE, API_KEY_HEADER, API_VERSION, BETA_HEADER, COUNT_TOKENS_PATH, VERSION_HEADER, + API_KEY_HEADER, API_VERSION, BETA_HEADER, COUNT_TOKENS_PATH, VERSION_HEADER, }, }; use serde::{Deserialize, Serialize}; use serde_json::Value; -use crate::{Error, anthropic::ANTHROPIC_OAUTH_TOKEN_PREFIX}; +use crate::{ + Error, + anthropic::{ANTHROPIC_OAUTH_TOKEN_PREFIX, connection::AnthropicConnection}, +}; const TOKEN_COUNTING_BETA: &str = "token-counting-2024-11-01"; @@ -27,7 +30,7 @@ pub struct AnthropicCountTokensResponse { } pub trait AnthropicCountTokensConfig { - fn endpoint(&self) -> String; + fn endpoint(&self, connection: &AnthropicConnection) -> Result; fn validate_request(&self, model: &str, messages: &[Message]) -> Result<(), Error>; @@ -48,8 +51,8 @@ pub const ANTHROPIC_COUNT_TOKENS_TRANSFORMATION: AnthropicCountTokensTransformat AnthropicCountTokensTransformation; impl AnthropicCountTokensConfig for AnthropicCountTokensTransformation { - fn endpoint(&self) -> String { - format!("{API_BASE}{COUNT_TOKENS_PATH}") + fn endpoint(&self, connection: &AnthropicConnection) -> Result { + Ok(connection.endpoint(COUNT_TOKENS_PATH)?) } fn transform_request( @@ -129,9 +132,89 @@ mod tests { "system": "system" }) ); + } + + fn connection( + api_base: Option<&str>, + runtime_default: Option<&str>, + env: &'static [(&'static str, &'static str)], + ) -> Result { + AnthropicConnection::resolve( + api_base, + &crate::base_llm::connection::RuntimeDefaults { + api_base: runtime_default.map(Into::into), + ..Default::default() + }, + &|name| { + env.iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.to_string()) + }, + ) + } + + #[rstest::rstest] + #[case::explicit_base_beats_both_aliases( + Some("https://explicit.test"), + None, + &[("ANTHROPIC_API_BASE", "https://a.test"), ("ANTHROPIC_BASE_URL", "https://b.test")], + "https://explicit.test/v1/messages/count_tokens" + )] + #[case::runtime_default_beats_the_environment( + None, + Some("https://runtime.test"), + &[("ANTHROPIC_API_BASE", "https://a.test"), ("ANTHROPIC_BASE_URL", "https://b.test")], + "https://runtime.test/v1/messages/count_tokens" + )] + #[case::api_base_alias( + None, + None, + &[("ANTHROPIC_API_BASE", "https://a.test"), ("ANTHROPIC_BASE_URL", "https://b.test")], + "https://a.test/v1/messages/count_tokens" + )] + #[case::base_url_alias( + None, + None, + &[("ANTHROPIC_BASE_URL", "https://b.test")], + "https://b.test/v1/messages/count_tokens" + )] + #[case::gateway_prefix_survives( + Some("https://gateway.test/anthropic/"), + None, + &[], + "https://gateway.test/anthropic/v1/messages/count_tokens" + )] + #[case::chat_surface_base( + Some("https://gateway.test/anthropic/v1/messages"), + None, + &[], + "https://gateway.test/anthropic/v1/messages/count_tokens" + )] + #[case::public_api_without_overrides(None, None, &[], "https://api.anthropic.com/v1/messages/count_tokens")] + fn the_endpoint_follows_the_resolved_anthropic_base( + #[case] api_base: Option<&str>, + #[case] runtime_default: Option<&str>, + #[case] env: &'static [(&'static str, &'static str)], + #[case] expected: &str, + ) { + let connection = connection(api_base, runtime_default, env).unwrap(); assert_eq!( - ANTHROPIC_COUNT_TOKENS_TRANSFORMATION.endpoint(), - "https://api.anthropic.com/v1/messages/count_tokens" + ANTHROPIC_COUNT_TOKENS_TRANSFORMATION + .endpoint(&connection) + .unwrap(), + expected + ); + } + + #[test] + fn a_malformed_explicit_base_never_falls_back_to_the_public_api() { + assert!( + connection( + Some("not a url"), + None, + &[("ANTHROPIC_API_BASE", "https://a.test")] + ) + .is_err() ); } diff --git a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs index aa926662b7a..62e9ee5fb6e 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs @@ -31,6 +31,34 @@ pub struct AnthropicMessagesConfig; pub const ANTHROPIC_MESSAGES_CONFIG: AnthropicMessagesConfig = AnthropicMessagesConfig; impl BaseMessagesConfig for AnthropicMessagesConfig { + fn prepare_connection( + &self, + inputs: crate::base_llm::connection::ConnectionInputs<'_>, + headers: Headers, + model: &str, + _stream: bool, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + let validated = self.validate_environment( + headers, + inputs.api_key, + model, + inputs.litellm_params, + env_lookup, + )?; + let connection = crate::anthropic::connection::AnthropicConnection::resolve( + inputs.api_base, + inputs.runtime_defaults, + env_lookup, + )?; + Ok(crate::base_llm::connection::PreparedConnection { + url: connection.messages_url(), + headers: validated.headers, + auth: validated.auth, + api_base_source: connection.api_base_source(), + }) + } + fn shape_request( &self, request: MessagesRequest, diff --git a/litellm-rust/crates/llms/src/anthropic/mod.rs b/litellm-rust/crates/llms/src/anthropic/mod.rs index 72c390cfe21..dbcc4d33cf0 100644 --- a/litellm-rust/crates/llms/src/anthropic/mod.rs +++ b/litellm-rust/crates/llms/src/anthropic/mod.rs @@ -3,6 +3,7 @@ pub mod common_utils; pub mod batches; pub mod chat; +pub mod connection; pub mod count_tokens; pub mod messages; diff --git a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs index 9eaaa657f6f..609087b9bce 100644 --- a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs @@ -36,6 +36,19 @@ pub const AZURE_ANTHROPIC_MESSAGES_CONFIG: AzureAnthropicMessagesConfig = AzureAnthropicMessagesConfig; impl BaseMessagesConfig for AzureAnthropicMessagesConfig { + fn prepare_connection( + &self, + inputs: crate::base_llm::connection::ConnectionInputs<'_>, + headers: Headers, + model: &str, + stream: bool, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + crate::base_llm::messages::transformation::prepare_from_parts( + self, inputs, headers, model, stream, env_lookup, + ) + } + fn shape_request( &self, request: MessagesRequest, diff --git a/litellm-rust/crates/llms/src/base_llm/connection.rs b/litellm-rust/crates/llms/src/base_llm/connection.rs new file mode 100644 index 00000000000..c79f0a92e9b --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/connection.rs @@ -0,0 +1,60 @@ +use litellm_core_utils::{ + settings::{Selected, SettingSource}, + url_utils::ApiUrl, +}; +use litellm_router_types::LitellmParams; +use serde::{Deserialize, Serialize}; + +use super::auth::{AuthScheme, Headers}; + +/// Defaults a host reads from its own runtime configuration. They are kept apart from a +/// call's explicit arguments so a provider resolver can rank them below every explicit +/// spelling and above the environment. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct RuntimeDefaults { + pub api_base: Option, + pub vertex_project: Option, + pub vertex_location: Option, +} + +#[derive(Clone, Copy)] +pub struct ConnectionInputs<'a> { + pub api_key: Option<&'a str>, + pub api_base: Option<&'a str>, + pub litellm_params: &'a LitellmParams, + pub runtime_defaults: &'a RuntimeDefaults, +} + +pub struct PreparedConnection { + pub url: String, + pub headers: Headers, + pub auth: AuthScheme, + pub api_base_source: SettingSource, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +#[error("api_base is not a valid base URL")] +pub struct InvalidApiBase; + +impl From for crate::Error { + fn from(error: InvalidApiBase) -> Self { + Self::InvalidRequest(crate::ErrorDetail::invalid("api_base", error)) + } +} + +pub fn validated_api_base(selected: Selected) -> Result, InvalidApiBase> { + ApiUrl::parse(&selected.value) + .map(|_| selected) + .map_err(|_| InvalidApiBase) +} + +pub fn complete_url(base: &str, path: &[&str]) -> Result { + ApiUrl::parse(base) + .and_then(|url| url.complete_path(path)) + .map(ApiUrl::into_string) + .map_err(|_| InvalidApiBase) +} + +pub fn path_segments(path: &str) -> impl Iterator { + path.split('/').filter(|segment| !segment.is_empty()) +} diff --git a/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs b/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs index 61398fe4ea9..5ee18fa050a 100644 --- a/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs @@ -5,7 +5,48 @@ use serde_json::Value; use super::context::MessagesTransformContext; pub use crate::base_llm::auth::{Headers, ValidatedEnvironment}; -use crate::{Error, base_llm::messages::streaming::StreamDecoder}; +use crate::{ + Error, + base_llm::{ + connection::{ConnectionInputs, PreparedConnection}, + messages::streaming::StreamDecoder, + }, +}; + +/// A connection from a config's own `validate_environment` and `get_complete_url`, for +/// providers whose settings have no shared resolver yet. +pub fn prepare_from_parts( + config: &C, + inputs: ConnectionInputs<'_>, + headers: Headers, + model: &str, + stream: bool, + env_lookup: &dyn Fn(&str) -> Option, +) -> Result { + let validated = config.validate_environment( + headers, + inputs.api_key, + model, + inputs.litellm_params, + env_lookup, + )?; + let url = config.get_complete_url( + inputs.api_base, + model, + inputs.litellm_params, + stream, + env_lookup, + )?; + Ok(PreparedConnection { + url, + headers: validated.headers, + auth: validated.auth, + api_base_source: match inputs.api_base { + Some(_) => litellm_core_utils::settings::SettingSource::Explicit, + None => litellm_core_utils::settings::SettingSource::Derived, + }, + }) +} pub trait BaseMessagesConfig: Sync { fn shape_request( @@ -55,6 +96,17 @@ pub trait BaseMessagesConfig: Sync { env_lookup: &dyn Fn(&str) -> Option, ) -> Result; + /// The URL, headers and auth plan for one deployment, built from one selection of the + /// provider's connection settings. + fn prepare_connection( + &self, + inputs: ConnectionInputs<'_>, + headers: Headers, + model: &str, + stream: bool, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result; + /// `None` relays the upstream bytes untouched, which is right for every host that already /// speaks Anthropic SSE. A host on another wire returns the decoder that lifts its frames /// into Anthropic stream events, and the route re-encodes those as Anthropic SSE. @@ -86,6 +138,19 @@ mod tests { struct DefaultsConfig; impl BaseMessagesConfig for DefaultsConfig { + fn prepare_connection( + &self, + inputs: crate::base_llm::connection::ConnectionInputs<'_>, + headers: Headers, + model: &str, + stream: bool, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + crate::base_llm::messages::transformation::prepare_from_parts( + self, inputs, headers, model, stream, env_lookup, + ) + } + fn secret_names(&self) -> &'static [&'static str] { &[] } diff --git a/litellm-rust/crates/llms/src/base_llm/mod.rs b/litellm-rust/crates/llms/src/base_llm/mod.rs index 9a6a0659d7d..2e1e9425372 100644 --- a/litellm-rust/crates/llms/src/base_llm/mod.rs +++ b/litellm-rust/crates/llms/src/base_llm/mod.rs @@ -2,6 +2,7 @@ pub mod audio_transcription; pub mod auth; pub mod base_model_iterator; pub mod chat; +pub mod connection; pub mod messages; pub mod ocr; pub mod responses; diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs b/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs index 2da0e1abbfc..f8e15401d89 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs @@ -168,6 +168,7 @@ pub struct PreparedOcrRequest { pub optional_params: CallArguments, pub input_sources: BTreeMap, pub azure_ad_token_provider: Option, + pub runtime_defaults: crate::base_llm::connection::RuntimeDefaults, } impl PreparedOcrRequest { diff --git a/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs b/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs index 4f8d4f01697..73c6defbdd3 100644 --- a/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs @@ -100,6 +100,19 @@ fn invoke_url( } impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig { + fn prepare_connection( + &self, + inputs: crate::base_llm::connection::ConnectionInputs<'_>, + headers: Headers, + model: &str, + stream: bool, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + crate::base_llm::messages::transformation::prepare_from_parts( + self, inputs, headers, model, stream, env_lookup, + ) + } + fn shape_request( &self, request: MessagesRequest, diff --git a/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs b/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs index 9359184f40e..d470985007c 100644 --- a/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs @@ -40,6 +40,19 @@ pub const DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG: DeepSeekAnthropicMessagesConfig = DeepSeekAnthropicMessagesConfig; impl BaseMessagesConfig for DeepSeekAnthropicMessagesConfig { + fn prepare_connection( + &self, + inputs: crate::base_llm::connection::ConnectionInputs<'_>, + headers: Headers, + model: &str, + stream: bool, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + crate::base_llm::messages::transformation::prepare_from_parts( + self, inputs, headers, model, stream, env_lookup, + ) + } + fn shape_request( &self, request: MessagesRequest, diff --git a/litellm-rust/crates/llms/src/vertex_ai/common_utils.rs b/litellm-rust/crates/llms/src/vertex_ai/common_utils.rs index 43b0f3f0de3..5bc8cf6e234 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/common_utils.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/common_utils.rs @@ -23,12 +23,16 @@ pub fn validate_vertex_location(location: &str) -> Result<&str, Error> { if location == GLOBAL_LOCATION || is_location_token(location) { return Ok(location); } - Err(Error::Auth(litellm_auth::Error::InvalidConfiguration( + Err(invalid_location()) +} + +pub(super) fn invalid_location() -> Error { + Error::Auth(litellm_auth::Error::InvalidConfiguration( litellm_auth::ErrorDetail::InvalidType { field: "vertex_location".into(), expected: "global or a lowercase alphanumeric token with hyphens", }, - ))) + )) } /// Python's `get_vertex_base_url`: the global control plane, a multi-region geography, or a diff --git a/litellm-rust/crates/llms/src/vertex_ai/connection.rs b/litellm-rust/crates/llms/src/vertex_ai/connection.rs new file mode 100644 index 00000000000..ba2123b059d --- /dev/null +++ b/litellm-rust/crates/llms/src/vertex_ai/connection.rs @@ -0,0 +1,282 @@ +use litellm_auth::VertexParams; +use litellm_auth_gcp::{VertexConfig, get_vertex_ai_project_from_credentials}; +use litellm_core_utils::settings::{Selected, SettingSource, select_non_blank}; +use litellm_llms_types::providers::vertex_ai::DEFAULT_LOCATION; + +use super::common_utils::{get_vertex_base_url, invalid_location, validate_vertex_location}; +use crate::base_llm::{ + connection::{InvalidApiBase, RuntimeDefaults, complete_url, validated_api_base}, + ocr::error::Error as OcrError, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum VertexConnectionError { + #[error("vertex_location must be global or a lowercase alphanumeric token with hyphens")] + InvalidLocation, + #[error(transparent)] + InvalidApiBase(#[from] InvalidApiBase), +} + +impl From for crate::Error { + fn from(error: VertexConnectionError) -> Self { + match error { + VertexConnectionError::InvalidLocation => invalid_location(), + VertexConnectionError::InvalidApiBase(error) => error.into(), + } + } +} + +impl From for OcrError { + fn from(error: VertexConnectionError) -> Self { + let path = match error { + VertexConnectionError::InvalidLocation => "vertex_location", + VertexConnectionError::InvalidApiBase(_) => "api_base", + }; + Self::RequestField { path: path.into() } + } +} + +/// The project, location and host every Vertex AI operation sends to, chosen once per +/// deployment. Each comes from the explicit params, the host's runtime defaults, the +/// environment, then a derived or provider default: the project from the credentials, the +/// location `us-central1`, and the host from the location. `config` carries the same +/// project and location to authentication. +#[derive(Clone, Debug)] +pub struct VertexConnection { + pub project: Option>, + pub location: Selected, + pub api_base: Selected, + pub config: VertexConfig, +} + +impl VertexConnection { + pub fn resolve( + config: VertexConfig, + api_base: Option<&str>, + runtime_defaults: &RuntimeDefaults, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + let location = select_non_blank( + config.location(), + runtime_defaults.vertex_location.as_deref(), + env_lookup, + VertexParams::LOCATION.env, + ) + .unwrap_or_else(|| { + Selected::new(DEFAULT_LOCATION.to_string(), SettingSource::ProviderDefault) + }); + let derived_base = get_vertex_base_url(&location.value) + .map_err(|_| VertexConnectionError::InvalidLocation)?; + let project = select_non_blank( + config.project_id(), + runtime_defaults.vertex_project.as_deref(), + env_lookup, + VertexParams::PROJECT.env, + ) + .or_else(|| { + get_vertex_ai_project_from_credentials(&config, env_lookup) + .map(|project| Selected::new(project, SettingSource::Derived)) + }); + let api_base = match select_non_blank(api_base, None, &|_| None, &[]) { + Some(custom) => validated_api_base(custom)?, + None => Selected::new(derived_base, SettingSource::Derived), + }; + let config = config.with_project_and_location( + project.as_ref().map(|project| project.value.clone()), + location.value.clone(), + ); + Ok(Self { + project, + location, + api_base, + config, + }) + } + + pub fn location(&self) -> &str { + &self.location.value + } + + pub fn custom_api_base(&self) -> Option<&str> { + (self.api_base.source != SettingSource::Derived).then_some(self.api_base.value.as_str()) + } + + /// `{api_base}/v1/projects/{project}/locations/{location}/{resource...}`. + pub fn resource_url( + &self, + project: &str, + resource: &[&str], + ) -> Result { + validate_vertex_location(self.location()) + .map_err(|_| VertexConnectionError::InvalidLocation)?; + let path = ["v1", "projects", project, "locations", self.location()] + .into_iter() + .chain(resource.iter().copied()) + .collect::>(); + Ok(complete_url(&self.api_base.value, &path)?) + } +} + +#[cfg(test)] +mod tests { + use litellm_auth::{InputSource, SecretValue, Sourced}; + use rstest::rstest; + + use super::*; + + fn env(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option { + move |name| { + values + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.to_string()) + } + } + + fn explicit(project: Option<&str>, location: Option<&str>) -> VertexConfig { + VertexConfig::new(None, project.map(Into::into), location.map(Into::into)) + } + + fn runtime(project: Option<&str>, location: Option<&str>) -> RuntimeDefaults { + RuntimeDefaults { + vertex_project: project.map(Into::into), + vertex_location: location.map(Into::into), + ..RuntimeDefaults::default() + } + } + + #[rstest] + #[case::explicit_beats_runtime_and_environment( + explicit(Some("arg"), Some("europe-west4")), + runtime(Some("default"), Some("us-east5")), + &[("VERTEXAI_PROJECT", "env"), ("VERTEXAI_LOCATION", "asia-east1")], + ("arg", SettingSource::Explicit), + ("europe-west4", SettingSource::Explicit) + )] + #[case::runtime_defaults_beat_the_environment( + explicit(None, None), + runtime(Some("default"), Some("us-east5")), + &[("VERTEXAI_PROJECT", "env"), ("VERTEXAI_LOCATION", "asia-east1")], + ("default", SettingSource::RuntimeDefault), + ("us-east5", SettingSource::RuntimeDefault) + )] + #[case::environment_then_the_default_location( + explicit(None, None), + runtime(None, None), + &[("VERTEXAI_PROJECT", "env")], + ("env", SettingSource::Environment("VERTEXAI_PROJECT")), + ("us-central1", SettingSource::ProviderDefault) + )] + #[case::second_location_alias( + explicit(Some("arg"), None), + runtime(None, None), + &[("VERTEX_LOCATION", "europe-west1")], + ("arg", SettingSource::Explicit), + ("europe-west1", SettingSource::Environment("VERTEX_LOCATION")) + )] + fn project_and_location_follow_the_layer_precedence( + #[case] config: VertexConfig, + #[case] defaults: RuntimeDefaults, + #[case] vars: &'static [(&'static str, &'static str)], + #[case] project: (&str, SettingSource), + #[case] location: (&str, SettingSource), + ) { + let connection = VertexConnection::resolve(config, None, &defaults, &env(vars)).unwrap(); + assert_eq!( + connection.project, + Some(Selected::new(project.0.to_string(), project.1)) + ); + assert_eq!( + connection.location, + Selected::new(location.0.to_string(), location.1) + ); + assert_eq!(connection.config.project_id(), Some(project.0)); + assert_eq!(connection.config.location(), Some(location.0)); + } + + #[test] + fn the_project_falls_back_to_the_credentials() { + let config = VertexConfig::new( + Some(Sourced::new( + SecretValue::new(r#"{"type": "service_account", "project_id": "from-creds"}"#), + InputSource::Deployment, + )), + None, + Some("us-east5".into()), + ); + let connection = + VertexConnection::resolve(config, None, &RuntimeDefaults::default(), &env(&[])) + .unwrap(); + assert_eq!( + connection.project, + Some(Selected::new("from-creds".into(), SettingSource::Derived)) + ); + assert_eq!(connection.config.project_id(), Some("from-creds")); + } + + #[rstest] + #[case::regional( + None, + "us-east5", + "https://us-east5-aiplatform.googleapis.com/v1/projects/proj/locations/us-east5/endpoints/openapi" + )] + #[case::global( + None, + "global", + "https://aiplatform.googleapis.com/v1/projects/proj/locations/global/endpoints/openapi" + )] + #[case::multi_region_geography( + None, + "eu", + "https://aiplatform.eu.rep.googleapis.com/v1/projects/proj/locations/eu/endpoints/openapi" + )] + #[case::custom_base_keeps_project_and_location( + Some("https://gateway.test/vertex"), + "europe-west4", + "https://gateway.test/vertex/v1/projects/proj/locations/europe-west4/endpoints/openapi" + )] + fn the_host_and_path_share_one_location( + #[case] api_base: Option<&str>, + #[case] location: &str, + #[case] expected: &str, + ) { + let connection = VertexConnection::resolve( + explicit(Some("proj"), Some(location)), + api_base, + &RuntimeDefaults::default(), + &env(&[]), + ) + .unwrap(); + assert_eq!( + connection + .resource_url("proj", &["endpoints", "openapi"]) + .unwrap(), + expected + ); + } + + #[rstest] + #[case::host_injection(None, "attacker.example/", VertexConnectionError::InvalidLocation)] + #[case::uppercase(None, "US-EAST5", VertexConnectionError::InvalidLocation)] + #[case::malformed_custom_base( + Some("not a url"), + "us-east5", + VertexConnectionError::InvalidApiBase(InvalidApiBase) + )] + fn an_unusable_connection_fails_before_sending( + #[case] api_base: Option<&str>, + #[case] location: &str, + #[case] expected: VertexConnectionError, + ) { + assert_eq!( + VertexConnection::resolve( + explicit(Some("proj"), Some(location)), + api_base, + &RuntimeDefaults::default(), + &env(&[]), + ) + .unwrap_err(), + expected + ); + } +} diff --git a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs index e6a3661a81f..6318526a56f 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs @@ -45,6 +45,45 @@ pub const VERTEX_ANTHROPIC_MESSAGES_CONFIG: VertexAiPartnerModelsAnthropicMessag VertexAiPartnerModelsAnthropicMessagesConfig; impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig { + fn prepare_connection( + &self, + inputs: crate::base_llm::connection::ConnectionInputs<'_>, + headers: Headers, + model: &str, + stream: bool, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + let connection = crate::vertex_ai::connection::VertexConnection::resolve( + VertexConfig::from_params(&inputs.litellm_params.vertex), + inputs.api_base, + inputs.runtime_defaults, + env_lookup, + )?; + let auth = match has_header(&headers, "authorization") { + true => AuthScheme::Forwarded, + false => AuthScheme::GcpAccessToken { + config: Box::new(connection.config.clone()), + }, + }; + let project = connection + .project + .as_ref() + .ok_or_else(|| Error::Auth(VertexParams::PROJECT.missing("Vertex AI")))?; + let url = complete_vertex_anthropic_url( + connection.custom_api_base(), + &project.value, + connection.location(), + model, + stream, + )?; + Ok(crate::base_llm::connection::PreparedConnection { + url, + headers, + auth, + api_base_source: connection.api_base.source, + }) + } + fn shape_request( &self, request: MessagesRequest, @@ -435,7 +474,7 @@ mod tests { )); assert_eq!( error.to_string(), - "Missing Vertex AI project - pass vertex_project or vertex_ai_project, set litellm.vertex_project, or set VERTEXAI_PROJECT" + "Missing Vertex AI project - pass vertex_project or vertex_ai_project or set VERTEXAI_PROJECT" ); } diff --git a/litellm-rust/crates/llms/src/vertex_ai/mod.rs b/litellm-rust/crates/llms/src/vertex_ai/mod.rs index 138b893e4ca..df8e55f2329 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/mod.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/mod.rs @@ -1,3 +1,4 @@ pub mod common_utils; +pub mod connection; pub mod messages; pub mod ocr; diff --git a/litellm-rust/crates/llms/src/vertex_ai/ocr/common_utils.rs b/litellm-rust/crates/llms/src/vertex_ai/ocr/common_utils.rs index 810462c57f8..eb15ec8bac5 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/ocr/common_utils.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/ocr/common_utils.rs @@ -5,11 +5,18 @@ use crate::base_llm::ocr::{ error::Error, transformation::{OcrConnection, PreparedOcrRequest}, }; +use crate::vertex_ai::connection::VertexConnection; -pub(super) fn vertex_config(request: &PreparedOcrRequest) -> Result { - Ok(VertexConfig::from_sourced_optional_params( +pub(super) fn vertex_connection(request: &PreparedOcrRequest) -> Result { + let config = VertexConfig::from_sourced_optional_params( &request.optional_params, &request.input_sources, + )?; + Ok(VertexConnection::resolve( + config, + request.connection.api_base.as_deref(), + &request.runtime_defaults, + &|name: &str| request.connection.secret(name), )?) } diff --git a/litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs index 67f9122ed67..3f57b5ce8e8 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs @@ -1,9 +1,9 @@ use litellm_auth_gcp as vertex; -use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl}; +use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; -use super::{common_utils::vertex_config, transformation::VertexAiOcrConfig}; +use super::{common_utils::vertex_connection, transformation::VertexAiOcrConfig}; use crate::base_llm::ocr::{ error::Error, handler::OcrClient, @@ -16,7 +16,6 @@ use litellm_llms_types::formats::ocr::{ LiteLLMOcrResponse, OcrDocument, OcrPage, OcrPageDimensions, OcrPageImage, OcrResponseFormat, OcrUsageInfo, }; -use litellm_llms_types::providers::vertex_ai::{DEFAULT_LOCATION, GLOBAL_API_BASE}; const MODEL_PREFIX: &str = "deepseek-ai/"; const DEEPSEEK_OCR_PARAMS: &[&str] = &["stream", "temperature", "max_tokens", "top_p", "n", "stop"]; @@ -135,15 +134,10 @@ impl BaseOcrConfig for VertexAIDeepSeekOCRConfig { _params: &Self::OcrParams, environment: &Self::Environment, ) -> Result { - let config = vertex_config(request)?; - let location = - vertex::get_vertex_ai_location(&config, &|name: &str| request.connection.secret(name)) - .unwrap_or_else(|| DEFAULT_LOCATION.to_string()); - self.get_complete_url( - request.connection.api_base.as_deref(), + Ok(vertex_connection(request)?.resource_url( &environment.project_id, - &location, - ) + &["endpoints", "openapi", "chat", "completions"], + )?) } async fn async_transform_ocr_request( @@ -385,38 +379,6 @@ pub fn provider_model(model: &str) -> Result { Ok(format!("{MODEL_PREFIX}{local_model}")) } -impl VertexAIDeepSeekOCRConfig { - fn get_complete_url( - &self, - api_base: Option<&str>, - project: &str, - location: &str, - ) -> Result { - let base = api_base - .map(str::trim) - .filter(|base| !base.is_empty()) - .unwrap_or(GLOBAL_API_BASE); - ApiUrl::parse(base) - .and_then(|url| { - url.complete_path(&[ - "v1", - "projects", - project, - "locations", - location, - "endpoints", - "openapi", - "chat", - "completions", - ]) - }) - .map(|url| url.into_string()) - .map_err(|_| Error::RequestField { - path: "api_base".into(), - }) - } -} - #[cfg(test)] mod tests { use rstest::rstest; @@ -471,12 +433,6 @@ mod tests { provider_model("deepseek-ai/deepseek-ocr-maas").unwrap(), "deepseek-ai/deepseek-ocr-maas" ); - assert_eq!( - VertexAIDeepSeekOCRConfig - .get_complete_url(None, "proj-1", "europe-west4") - .unwrap(), - "https://aiplatform.googleapis.com/v1/projects/proj-1/locations/europe-west4/endpoints/openapi/chat/completions" - ); } #[rstest] diff --git a/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs index ab28ee8bff4..de576cf8630 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs @@ -1,8 +1,8 @@ use litellm_auth_gcp::{self as vertex, VertexConfig}; -use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl}; +use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams}; use serde_json::Value; -use super::common_utils::{validate_destination, vertex_config}; +use super::common_utils::{validate_destination, vertex_connection}; use crate::{ base_llm::ocr::{ document::{inline_remote_document, validate_inline_document}, @@ -16,8 +16,6 @@ use crate::{ }; use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrDocument, OcrResponseFormat}; -use litellm_llms_types::providers::vertex_ai::DEFAULT_LOCATION; - #[derive(Clone, Debug, Default)] pub struct VertexAiOcrConfig; @@ -51,8 +49,8 @@ impl BaseOcrConfig for VertexAiOcrConfig { request: &PreparedOcrRequest, client: &OcrClient, ) -> Result { - let config = vertex_config(request)?; - self.resolve_environment(&request.connection, &config, client) + let connection = vertex_connection(request)?; + self.resolve_environment(&request.connection, &connection.config, client) .await } @@ -62,16 +60,11 @@ impl BaseOcrConfig for VertexAiOcrConfig { _optional_params: &Self::OcrParams, environment: &Self::Environment, ) -> Result { - let config = vertex_config(request)?; - let location = - vertex::get_vertex_ai_location(&config, &|name: &str| request.connection.secret(name)) - .unwrap_or_else(|| DEFAULT_LOCATION.to_string()); - self.build_ocr_url( - request.connection.api_base.as_deref(), + let prediction = format!("{}:rawPredict", request.model); + Ok(vertex_connection(request)?.resource_url( &environment.project_id, - &location, - &request.model, - ) + &["publishers", "mistralai", "models", &prediction], + )?) } fn transform_ocr_request( @@ -144,84 +137,4 @@ impl VertexAiOcrConfig { .await .map_err(Error::from) } - - fn build_ocr_url( - &self, - api_base: Option<&str>, - project: &str, - location: &str, - model: &str, - ) -> Result { - validate_location(location)?; - let default_base = format!("https://{location}-aiplatform.googleapis.com"); - let base = api_base - .map(str::trim) - .filter(|base| !base.is_empty()) - .unwrap_or(&default_base); - let prediction = format!("{model}:rawPredict"); - ApiUrl::parse(base) - .and_then(|url| { - url.complete_path(&[ - "v1", - "projects", - project, - "locations", - location, - "publishers", - "mistralai", - "models", - &prediction, - ]) - }) - .map(|url| url.into_string()) - .map_err(|_| Error::RequestField { - path: "api_base".into(), - }) - } -} - -fn validate_location(location: &str) -> Result<(), Error> { - let valid = !location.is_empty() - && location - .bytes() - .all(|value| value.is_ascii_lowercase() || value.is_ascii_digit() || value == b'-') - && location - .as_bytes() - .first() - .is_some_and(u8::is_ascii_alphanumeric) - && location - .as_bytes() - .last() - .is_some_and(u8::is_ascii_alphanumeric); - if valid { - return Ok(()); - } - Err(Error::RequestField { - path: "vertex_location".into(), - }) -} - -#[cfg(test)] -mod tests { - - use super::VertexAiOcrConfig; - - #[test] - fn endpoint_uses_location_project_and_model() { - assert_eq!( - VertexAiOcrConfig - .build_ocr_url(None, "proj-1", "europe-west4", "mistral-ocr-maas") - .unwrap(), - "https://europe-west4-aiplatform.googleapis.com/v1/projects/proj-1/locations/europe-west4/publishers/mistralai/models/mistral-ocr-maas:rawPredict" - ); - } - - #[test] - fn endpoint_rejects_invalid_location() { - assert!( - VertexAiOcrConfig - .build_ocr_url(None, "proj-1", "attacker.example/path", "model") - .is_err() - ); - } } diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 9fed9422240..00985a021a2 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -12,6 +12,7 @@ mod marshal; mod preflight; mod python_settings; mod routes; +mod runtime_defaults; mod secrets; mod tokenizer; diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs index adeba397b26..af08e685f94 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs @@ -22,7 +22,6 @@ use serde_json::{Map, Value}; use crate::{ errors::{RustUpstreamError, route_error_to_pyerr}, marshal::{optional_timeout, project_optional_fields, public_response, python_timeout_seconds}, - python_settings::missing_module, routes::RequestView, }; @@ -66,31 +65,13 @@ fn merge_headers( (!merged.is_empty()).then_some(merged) } -/// Reads `litellm.`, the module global a param spec names, the way -/// `VertexBase.get_vertex_ai_project` reads `litellm.vertex_project`. -fn module_global<'py>(py: Python<'py>, name: &str) -> PyResult>> { - let module = match py.import("litellm") { - Ok(module) => module, - Err(error) if missing_module(py, &error, "litellm")? => return Ok(None), - Err(error) => return Err(error), - }; - let value = module.getattr(name)?; - Ok((!value.is_none()).then_some(value)) -} - /// The caller's litellm params, read from the kwargs by the names the typed params declare, -/// so a key the configs do not read is never converted; a spec whose spellings the call all -/// leaves out falls back to the module global it names, which stays the host's concern. +/// so a key the configs do not read is never converted. fn project_litellm_params<'py>( argument: impl Fn(&str) -> PyResult>>, - global: impl Fn(&str) -> PyResult>>, ) -> PyResult> { let fields = project_optional_fields(LitellmParams::fields(), &argument)?; - let globals = LitellmParams::specs() - .filter(|spec| !spec.wire.iter().any(|name| fields.contains_key(*name))) - .filter_map(|spec| spec.module_global); - let folded = project_optional_fields(globals, &global)?; - Ok(litellm_params(fields.into_iter().chain(folded).collect())) + Ok(litellm_params(fields)) } fn native_error(py: Python<'_>, error: Error) -> PyResult { @@ -161,7 +142,8 @@ impl MessagesPythonHost { let api_base = string("api_base")?; let extra_headers = self.merged_headers(py, arguments)?; let provider_specific_header = self.provider_specific_header(py, arguments)?; - let litellm_params = project_litellm_params(argument, |name| module_global(py, name))?; + let litellm_params = project_litellm_params(argument)?; + let runtime_defaults = crate::runtime_defaults::read(py)?; Ok(messages_body(body).and_then(|body| { Ok(MessagesCall { body, @@ -171,6 +153,7 @@ impl MessagesPythonHost { provider_specific_header, custom_llm_provider, litellm_params: litellm_params?, + runtime_defaults, timeout: optional_timeout(timeout), shaping, }) @@ -410,58 +393,10 @@ mod tests { #[case] kwargs: Value, #[case] expected: Result, ) { - assert_projection(kwargs, json!({}), expected); + assert_projection(kwargs, expected); } - #[rstest] - #[case::global_fills_a_missing_vertex_project( - json!({"model": "m"}), - json!({"vertex_project": "from-global", "vertex_location": "us-east5"}), - Ok(json!({"model": "m", "vertex_project": "from-global", "vertex_location": "us-east5"})), - )] - #[case::the_call_wins_over_the_global( - json!({"model": "m", "vertex_project": "from-call"}), - json!({"vertex_project": "from-global"}), - Ok(json!({"model": "m", "vertex_project": "from-call"})), - )] - #[case::an_explicit_none_in_the_call_still_falls_back( - json!({"model": "m", "vertex_project": null}), - json!({"vertex_project": "from-global"}), - Ok(json!({"model": "m", "vertex_project": "from-global"})), - )] - #[case::a_global_of_the_wrong_type_is_a_request_error( - json!({"model": "m"}), - json!({"vertex_location": 5}), - Err(()), - )] - fn module_globals_fill_the_litellm_params_the_call_leaves_out( - #[case] kwargs: Value, - #[case] globals: Value, - #[case] expected: Result, - ) { - assert_projection(kwargs, globals, expected); - } - - #[rstest] - #[case::a_global_without_a_spec_is_never_read( - json!({"model": "m"}), - json!({"aws_region_name": "eu-central-1", "api_key": "k"}), - Ok(json!({"model": "m"})), - )] - #[case::only_the_spec_named_global_is_read( - json!({"model": "m"}), - json!({"vertex_ai_project": "legacy-global", "vertex_project": "from-global"}), - Ok(json!({"model": "m", "vertex_project": "from-global"})), - )] - fn only_the_globals_the_specs_name_are_consulted( - #[case] kwargs: Value, - #[case] globals: Value, - #[case] expected: Result, - ) { - assert_projection(kwargs, globals, expected); - } - - fn assert_projection(kwargs: Value, globals: Value, expected: Result) { + fn assert_projection(kwargs: Value, expected: Result) { Python::initialize(); Python::attach(|py| { let dict = |value: &Value| { @@ -472,13 +407,8 @@ mod tests { .unwrap() }; let kwargs = dict(&kwargs); - let globals = dict(&globals); let bound = PyDict::new(py); - let projected = project_litellm_params( - |name| present(&kwargs, &bound, name), - |name| present(&globals, &bound, name), - ) - .unwrap(); + let projected = project_litellm_params(|name| present(&kwargs, &bound, name)).unwrap(); match (projected, expected) { (Ok(projected), Ok(fields)) => assert_eq!( projected, diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs index 93f04966f08..10b937cb7b1 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs @@ -1,5 +1,4 @@ use litellm_auth::SecretValue; -use litellm_auth::VertexParams; use litellm_host_python::{from_py, present}; use litellm_inference_ocr::{ types::{LiteLLMOcrRequest, OcrDocumentInput}, @@ -11,10 +10,9 @@ use serde_json::{Map, Value}; use super::{document::FileDocumentInput, errors::to_pyerr as ocr_error_to_pyerr}; use crate::{ - coercion::FieldSpec, credentials::{self, CallerTokenProvider}, marshal::{project_optional_fields, python_timeout_seconds, request_input_sources}, - python_settings::{PythonSettings, Snapshot}, + runtime_defaults, }; /// What the host keeps after projection: the caller's token callable that answers the @@ -126,16 +124,8 @@ pub(super) fn project_request( let names = specs.iter().map(|spec| spec.name).collect::>(); let optional_params = project_optional_fields(names.iter().copied(), |name| present(kwargs, bound, name))?; - let globals = module_globals_to_read(&optional_params, &names); - let defaults = if globals.is_empty() { - None - } else { - PythonSettings::ProviderDefaults.read_or_unset(bound.py())? - }; - let optional_params = match defaults { - Some(defaults) => fold_module_globals(bound.py(), optional_params, &globals, &defaults)?, - None => optional_params, - }; + let runtime_defaults = + runtime_defaults::read(bound.py()).map_err(|error| settings_error(bound.py(), error))?; let input_sources = request_input_sources( kwargs, names @@ -158,7 +148,9 @@ pub(super) fn project_request( input_sources, timeout_seconds, }; - let request = decode_request_input(wire).map_err(ocr_error_to_pyerr)?; + let request = decode_request_input(wire) + .map_err(ocr_error_to_pyerr)? + .with_runtime_defaults(runtime_defaults); let provider = request.provider_name(); Ok(( request, @@ -169,50 +161,6 @@ pub(super) fn project_request( )) } -/// Python's `VertexBase.get_vertex_ai_project` and `get_vertex_ai_location` fall back to a -/// `litellm.` module global when a call names neither spelling: the globals of the -/// specs this route consumes whose wire names the call left out. -fn module_globals_to_read( - optional_params: &Map, - consumed: &[&str], -) -> Vec<&'static str> { - VertexParams::SPECS - .iter() - .filter(|spec| spec.wire.iter().all(|name| consumed.contains(name))) - .filter(|spec| { - !spec - .wire - .iter() - .any(|name| optional_params.contains_key(*name)) - }) - .filter_map(|spec| spec.module_global) - .collect() -} - -/// Folds each module global in under the name its spec declares, skipping falsy values the -/// way Python's `or` chain does. -fn fold_module_globals( - py: Python<'_>, - optional_params: Map, - globals: &[&'static str], - defaults: &Snapshot<'_>, -) -> PyResult> { - let read = globals - .iter() - .map(|name| { - let spec = FieldSpec::new(name, |field| field.falsy_optional_string()); - Ok(defaults - .read(&spec) - .map_err(|error| settings_error(py, error.into()))? - .map(|value| (name.to_string(), Value::String(value)))) - }) - .collect::>>()?; - Ok(optional_params - .into_iter() - .chain(read.into_iter().flatten()) - .collect()) -} - /// A misconfigured litellm setting is the operator's error, not the request's or the /// provider's, so the host raises it as is instead of mapping it onto a provider failure. pub(super) const SETTINGS_ERROR_MARKER: &str = "native_settings_error"; @@ -229,105 +177,6 @@ mod tests { use super::*; - #[rstest::rstest] - #[case::global_fills_a_missing_project( - serde_json::json!({"vertex_location": "us-east5"}), - "vertex_project='from-global', vertex_location='ignored'", - serde_json::json!({"vertex_location": "us-east5", "vertex_project": "from-global"}), - )] - #[case::either_spelling_on_the_call_wins( - serde_json::json!({"vertex_ai_project": "from-call"}), - "vertex_project='from-global', vertex_location=None", - serde_json::json!({"vertex_ai_project": "from-call"}), - )] - #[case::falsy_globals_are_absent( - serde_json::json!({}), - "vertex_project=[], vertex_location=''", - serde_json::json!({}), - )] - fn module_globals_fill_the_vertex_params_the_call_leaves_out( - #[case] params: Value, - #[case] defaults: &str, - #[case] expected: Value, - ) { - Python::initialize(); - Python::attach(|py| { - let namespace = py - .eval( - &std::ffi::CString::new(format!( - "__import__('types').SimpleNamespace({defaults})" - )) - .unwrap(), - None, - None, - ) - .unwrap(); - let snapshot = PythonSettings::ProviderDefaults.snapshot(namespace); - let consumed = VertexParams::fields().collect::>(); - let params = params.as_object().cloned().unwrap(); - let globals = module_globals_to_read(¶ms, &consumed); - - let folded = fold_module_globals(py, params, &globals, &snapshot).unwrap(); - - assert_eq!(Value::Object(folded), expected); - }); - } - - #[rstest::rstest] - fn a_bad_global_is_a_marked_settings_error() { - Python::initialize(); - Python::attach(|py| { - let namespace = py - .eval( - c"__import__('types').SimpleNamespace(vertex_project=1)", - None, - None, - ) - .unwrap(); - let snapshot = PythonSettings::ProviderDefaults.snapshot(namespace); - - let error = - fold_module_globals(py, Map::new(), &["vertex_project"], &snapshot).unwrap_err(); - - assert!(error.is_instance_of::(py)); - assert!( - error - .to_string() - .contains("provider_defaults.vertex_project") - ); - assert!( - error - .value(py) - .getattr_opt(SETTINGS_ERROR_MARKER) - .unwrap() - .is_some() - ); - }); - } - - #[rstest::rstest] - #[case::not_consumed(&["pages"], serde_json::json!({}), &[])] - #[case::consumed_and_absent( - &["vertex_project", "vertex_ai_project", "vertex_location", "vertex_ai_location"], - serde_json::json!({}), - &["vertex_project", "vertex_location"] - )] - #[case::consumed_and_present_in_either_spelling( - &["vertex_project", "vertex_ai_project", "vertex_location", "vertex_ai_location"], - serde_json::json!({"vertex_ai_project": "p"}), - &["vertex_location"] - )] - fn only_the_globals_of_consumed_absent_params_are_read( - #[case] consumed: &[&str], - #[case] params: Value, - #[case] expected: &[&str], - ) { - assert_eq!( - module_globals_to_read(params.as_object().unwrap(), consumed), - expected - ); - } - fn eval<'py>(py: Python<'py>, source: &std::ffi::CStr) -> Bound<'py, PyDict> { let locals = PyDict::new(py); py.run(source, Some(&locals), Some(&locals)).unwrap(); diff --git a/litellm-rust/crates/python-bridge/src/runtime_defaults.rs b/litellm-rust/crates/python-bridge/src/runtime_defaults.rs new file mode 100644 index 00000000000..c4a9f254500 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/runtime_defaults.rs @@ -0,0 +1,96 @@ +use litellm_llms::base_llm::connection::RuntimeDefaults; +use pyo3::prelude::*; + +use crate::{ + coercion::{FieldSpec, ProjectionError}, + python_settings::{PythonSettings, Snapshot}, +}; + +/// The `litellm.` module defaults, read once per call into the typed runtime-defaults +/// layer that provider resolvers rank between explicit arguments and the environment. +pub(crate) fn read(py: Python<'_>) -> PyResult { + match PythonSettings::ProviderDefaults.read_or_unset(py)? { + Some(snapshot) => project(&snapshot).map_err(PyErr::from), + None => Ok(RuntimeDefaults::default()), + } +} + +fn project(snapshot: &Snapshot<'_>) -> Result { + let read = |name: &'static str| { + snapshot.read(&FieldSpec::new(name, |field| field.falsy_optional_string())) + }; + Ok(RuntimeDefaults { + api_base: read("api_base")?, + vertex_project: read("vertex_project")?, + vertex_location: read("vertex_location")?, + }) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn projected(source: &std::ffi::CStr) -> Result { + Python::initialize(); + Python::attach(|py| { + let value = py.eval(source, None, None).unwrap(); + project(&PythonSettings::ProviderDefaults.snapshot(value)) + .map_err(|error| PyErr::from(error).value(py).to_string()) + }) + } + + #[rstest] + #[case::every_default( + c"__import__('types').SimpleNamespace(api_base='https://runtime.test', vertex_project='p', vertex_location='us-east5')", + RuntimeDefaults { + api_base: Some("https://runtime.test".into()), + vertex_project: Some("p".into()), + vertex_location: Some("us-east5".into()), + } + )] + #[case::falsy_values_are_unset( + c"__import__('types').SimpleNamespace(api_base='', vertex_project=None, vertex_location=[])", + RuntimeDefaults::default() + )] + fn module_defaults_project_into_their_own_layer( + #[case] source: &std::ffi::CStr, + #[case] expected: RuntimeDefaults, + ) { + assert_eq!(projected(source).unwrap(), expected); + } + + #[test] + fn a_misconfigured_module_default_names_the_setting() { + let error = projected( + c"__import__('types').SimpleNamespace(api_base=1, vertex_project=None, vertex_location=None)", + ) + .unwrap_err(); + assert!(error.contains("provider_defaults.api_base"), "{error}"); + } + + #[test] + fn each_call_reads_the_current_module_defaults() { + Python::initialize(); + Python::attach(|py| { + let namespace = py + .eval( + c"__import__('types').SimpleNamespace(api_base='https://first.test', vertex_project=None, vertex_location=None)", + None, + None, + ) + .unwrap(); + let first = + project(&PythonSettings::ProviderDefaults.snapshot(namespace.clone())).unwrap(); + namespace + .setattr("api_base", "https://second.test") + .unwrap(); + let second = project(&PythonSettings::ProviderDefaults.snapshot(namespace)).unwrap(); + assert_eq!( + (first.api_base.as_deref(), second.api_base.as_deref()), + (Some("https://first.test"), Some("https://second.test")) + ); + }); + } +} diff --git a/litellm/rust_bridge/settings.py b/litellm/rust_bridge/settings.py index 60a58f1aaba..df29c3f4110 100644 --- a/litellm/rust_bridge/settings.py +++ b/litellm/rust_bridge/settings.py @@ -26,6 +26,7 @@ class UrlPolicy: @dataclass(frozen=True, slots=True) class ProviderDefaults: + api_base: object vertex_project: object vertex_location: object enable_azure_ad_token_refresh: object @@ -112,6 +113,7 @@ def provider_defaults() -> ProviderDefaults: import litellm return ProviderDefaults( + api_base=litellm.api_base, vertex_project=litellm.vertex_project, vertex_location=litellm.vertex_location, enable_azure_ad_token_refresh=litellm.enable_azure_ad_token_refresh, diff --git a/tests/unit/rust_bridge/test_settings.py b/tests/unit/rust_bridge/test_settings.py index ea943afe970..bb443d6bcc1 100644 --- a/tests/unit/rust_bridge/test_settings.py +++ b/tests/unit/rust_bridge/test_settings.py @@ -195,11 +195,13 @@ def test_secret_manager_uses_key_management_defaults(monkeypatch: pytest.MonkeyP def test_provider_defaults_read_the_litellm_globals(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "api_base", "https://runtime.example") monkeypatch.setattr(litellm, "vertex_project", "configured-project") monkeypatch.setattr(litellm, "vertex_location", "europe-west4") monkeypatch.setattr(litellm, "enable_azure_ad_token_refresh", True) assert settings.provider_defaults() == settings.ProviderDefaults( + api_base="https://runtime.example", vertex_project="configured-project", vertex_location="europe-west4", enable_azure_ad_token_refresh=True, From c3d8b6aa7d01cb42ce0b654ade10df5ca641d232 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Sun, 11 Oct 2026 00:38:25 +0000 Subject: [PATCH 2/3] fix(rust): read only the runtime defaults a call falls back to Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../python-bridge/src/routes/messages/host.rs | 17 ++- .../python-bridge/src/routes/ocr/project.rs | 14 +- .../python-bridge/src/runtime_defaults.rs | 120 +++++++++++++++--- 3 files changed, 131 insertions(+), 20 deletions(-) diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs index af08e685f94..3b85697dc8e 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs @@ -2,6 +2,7 @@ use crate::cache::{CacheCall, Cached, PythonCache, Selection}; use litellm_host_python::{PythonHostCalls, PythonOwned}; use bytes::Bytes; +use litellm_auth::VertexParams; use litellm_host_python::{InvokeError, PythonBinding, from_py, to_py}; use litellm_http::transport::Error as TransportError; use litellm_inference_messages::{ @@ -23,6 +24,7 @@ use crate::{ errors::{RustUpstreamError, route_error_to_pyerr}, marshal::{optional_timeout, project_optional_fields, public_response, python_timeout_seconds}, routes::RequestView, + runtime_defaults::RuntimeDefault, }; const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.messages.route_host"; @@ -143,7 +145,20 @@ impl MessagesPythonHost { let extra_headers = self.merged_headers(py, arguments)?; let provider_specific_header = self.provider_specific_header(py, arguments)?; let litellm_params = project_litellm_params(argument)?; - let runtime_defaults = crate::runtime_defaults::read(py)?; + let supplied = [VertexParams::PROJECT, VertexParams::LOCATION] + .iter() + .flat_map(|spec| spec.wire.iter().copied()) + .filter_map(|name| argument(name).map(|value| value.map(|_| name)).transpose()) + .collect::>>()?; + let wanted = api_base + .is_none() + .then_some(RuntimeDefault::ApiBase) + .into_iter() + .chain(RuntimeDefault::vertex_fallbacks(|spec| { + !spec.wire.iter().any(|name| supplied.contains(name)) + })) + .collect::>(); + let runtime_defaults = crate::runtime_defaults::read(py, &wanted)?; Ok(messages_body(body).and_then(|body| { Ok(MessagesCall { body, diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs index 10b937cb7b1..f36d8d2a68f 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs @@ -12,7 +12,7 @@ use super::{document::FileDocumentInput, errors::to_pyerr as ocr_error_to_pyerr} use crate::{ credentials::{self, CallerTokenProvider}, marshal::{project_optional_fields, python_timeout_seconds, request_input_sources}, - runtime_defaults, + runtime_defaults::{self, RuntimeDefault}, }; /// What the host keeps after projection: the caller's token callable that answers the @@ -124,8 +124,16 @@ pub(super) fn project_request( let names = specs.iter().map(|spec| spec.name).collect::>(); let optional_params = project_optional_fields(names.iter().copied(), |name| present(kwargs, bound, name))?; - let runtime_defaults = - runtime_defaults::read(bound.py()).map_err(|error| settings_error(bound.py(), error))?; + let wanted = RuntimeDefault::vertex_fallbacks(|spec| { + spec.wire.iter().all(|name| names.contains(name)) + && !spec + .wire + .iter() + .any(|name| optional_params.contains_key(*name)) + }) + .collect::>(); + let runtime_defaults = runtime_defaults::read(bound.py(), &wanted) + .map_err(|error| settings_error(bound.py(), error))?; let input_sources = request_input_sources( kwargs, names diff --git a/litellm-rust/crates/python-bridge/src/runtime_defaults.rs b/litellm-rust/crates/python-bridge/src/runtime_defaults.rs index c4a9f254500..75d13812c79 100644 --- a/litellm-rust/crates/python-bridge/src/runtime_defaults.rs +++ b/litellm-rust/crates/python-bridge/src/runtime_defaults.rs @@ -1,3 +1,4 @@ +use litellm_auth::{ParamSpec, VertexParams}; use litellm_llms::base_llm::connection::RuntimeDefaults; use pyo3::prelude::*; @@ -6,23 +7,64 @@ use crate::{ python_settings::{PythonSettings, Snapshot}, }; -/// The `litellm.` module defaults, read once per call into the typed runtime-defaults -/// layer that provider resolvers rank between explicit arguments and the environment. -pub(crate) fn read(py: Python<'_>) -> PyResult { +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum RuntimeDefault { + ApiBase, + VertexProject, + VertexLocation, +} + +impl RuntimeDefault { + const fn name(self) -> &'static str { + match self { + Self::ApiBase => "api_base", + Self::VertexProject => "vertex_project", + Self::VertexLocation => "vertex_location", + } + } + + /// The Vertex defaults whose spec the call leaves `unset`. + pub(crate) fn vertex_fallbacks( + unset: impl Fn(&ParamSpec) -> bool, + ) -> impl Iterator { + [ + (Self::VertexProject, VertexParams::PROJECT), + (Self::VertexLocation, VertexParams::LOCATION), + ] + .into_iter() + .filter(move |(_, spec)| unset(spec)) + .map(|(default, _)| default) + } +} + +/// The `litellm.` module defaults the call falls back to, read once per call into the +/// typed runtime-defaults layer that provider resolvers rank between explicit arguments and +/// the environment. A default the call does not fall back to is never read, so a malformed +/// one cannot fail an unrelated call. +pub(crate) fn read(py: Python<'_>, wanted: &[RuntimeDefault]) -> PyResult { + if wanted.is_empty() { + return Ok(RuntimeDefaults::default()); + } match PythonSettings::ProviderDefaults.read_or_unset(py)? { - Some(snapshot) => project(&snapshot).map_err(PyErr::from), + Some(snapshot) => project(&snapshot, wanted).map_err(PyErr::from), None => Ok(RuntimeDefaults::default()), } } -fn project(snapshot: &Snapshot<'_>) -> Result { - let read = |name: &'static str| { - snapshot.read(&FieldSpec::new(name, |field| field.falsy_optional_string())) +fn project( + snapshot: &Snapshot<'_>, + wanted: &[RuntimeDefault], +) -> Result { + let read = |default: RuntimeDefault| match wanted.contains(&default) { + true => snapshot.read(&FieldSpec::new(default.name(), |field| { + field.falsy_optional_string() + })), + false => Ok(None), }; Ok(RuntimeDefaults { - api_base: read("api_base")?, - vertex_project: read("vertex_project")?, - vertex_location: read("vertex_location")?, + api_base: read(RuntimeDefault::ApiBase)?, + vertex_project: read(RuntimeDefault::VertexProject)?, + vertex_location: read(RuntimeDefault::VertexLocation)?, }) } @@ -32,11 +74,20 @@ mod tests { use super::*; - fn projected(source: &std::ffi::CStr) -> Result { + const EVERY: [RuntimeDefault; 3] = [ + RuntimeDefault::ApiBase, + RuntimeDefault::VertexProject, + RuntimeDefault::VertexLocation, + ]; + + fn projected( + source: &std::ffi::CStr, + wanted: &[RuntimeDefault], + ) -> Result { Python::initialize(); Python::attach(|py| { let value = py.eval(source, None, None).unwrap(); - project(&PythonSettings::ProviderDefaults.snapshot(value)) + project(&PythonSettings::ProviderDefaults.snapshot(value), wanted) .map_err(|error| PyErr::from(error).value(py).to_string()) }) } @@ -58,13 +109,43 @@ mod tests { #[case] source: &std::ffi::CStr, #[case] expected: RuntimeDefaults, ) { - assert_eq!(projected(source).unwrap(), expected); + assert_eq!(projected(source, &EVERY).unwrap(), expected); + } + + #[rstest] + #[case::nothing_wanted(&[], RuntimeDefaults::default())] + #[case::only_the_api_base( + &[RuntimeDefault::ApiBase], + RuntimeDefaults { api_base: Some("https://runtime.test".into()), ..Default::default() } + )] + fn a_malformed_default_the_call_does_not_use_is_never_read( + #[case] wanted: &[RuntimeDefault], + #[case] expected: RuntimeDefaults, + ) { + let source = c"__import__('types').SimpleNamespace(api_base='https://runtime.test', vertex_project=5, vertex_location=5)"; + assert_eq!(projected(source, wanted).unwrap(), expected); + } + + #[rstest] + #[case::both_unset(&[], vec![RuntimeDefault::VertexProject, RuntimeDefault::VertexLocation])] + #[case::project_on_the_call(&["vertex_ai_project"], vec![RuntimeDefault::VertexLocation])] + #[case::both_on_the_call(&["vertex_project", "vertex_location"], vec![])] + fn vertex_fallbacks_skip_the_settings_the_call_supplies( + #[case] supplied: &[&str], + #[case] expected: Vec, + ) { + let fallbacks = RuntimeDefault::vertex_fallbacks(|spec| { + !spec.wire.iter().any(|name| supplied.contains(name)) + }) + .collect::>(); + assert_eq!(fallbacks, expected); } #[test] fn a_misconfigured_module_default_names_the_setting() { let error = projected( c"__import__('types').SimpleNamespace(api_base=1, vertex_project=None, vertex_location=None)", + &EVERY, ) .unwrap_err(); assert!(error.contains("provider_defaults.api_base"), "{error}"); @@ -81,12 +162,19 @@ mod tests { None, ) .unwrap(); - let first = - project(&PythonSettings::ProviderDefaults.snapshot(namespace.clone())).unwrap(); + let first = project( + &PythonSettings::ProviderDefaults.snapshot(namespace.clone()), + &EVERY, + ) + .unwrap(); namespace .setattr("api_base", "https://second.test") .unwrap(); - let second = project(&PythonSettings::ProviderDefaults.snapshot(namespace)).unwrap(); + let second = project( + &PythonSettings::ProviderDefaults.snapshot(namespace), + &EVERY, + ) + .unwrap(); assert_eq!( (first.api_base.as_deref(), second.api_base.as_deref()), (Some("https://first.test"), Some("https://second.test")) From 075c58a92a7475650cee7352f800c7b753c347de Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Sun, 11 Oct 2026 00:45:53 +0000 Subject: [PATCH 3/3] fix(rust): keep DeepSeek OCR on the global Vertex host Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../crates/llms/src/vertex_ai/connection.rs | 70 ++++++++++++++++++- .../vertex_ai/ocr/deepseek_transformation.rs | 2 +- 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/litellm-rust/crates/llms/src/vertex_ai/connection.rs b/litellm-rust/crates/llms/src/vertex_ai/connection.rs index ba2123b059d..7545183047c 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/connection.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/connection.rs @@ -1,7 +1,7 @@ use litellm_auth::VertexParams; use litellm_auth_gcp::{VertexConfig, get_vertex_ai_project_from_credentials}; use litellm_core_utils::settings::{Selected, SettingSource, select_non_blank}; -use litellm_llms_types::providers::vertex_ai::DEFAULT_LOCATION; +use litellm_llms_types::providers::vertex_ai::{DEFAULT_LOCATION, GLOBAL_API_BASE}; use super::common_utils::{get_vertex_base_url, invalid_location, validate_vertex_location}; use crate::base_llm::{ @@ -106,6 +106,26 @@ impl VertexConnection { &self, project: &str, resource: &[&str], + ) -> Result { + self.resource_url_on(&self.api_base.value, project, resource) + } + + /// [`Self::resource_url`] for a publisher model Vertex serves only from the global host, + /// whatever location the path names; a custom base still wins. + pub fn global_resource_url( + &self, + project: &str, + resource: &[&str], + ) -> Result { + let base = self.custom_api_base().unwrap_or(GLOBAL_API_BASE); + self.resource_url_on(base, project, resource) + } + + fn resource_url_on( + &self, + base: &str, + project: &str, + resource: &[&str], ) -> Result { validate_vertex_location(self.location()) .map_err(|_| VertexConnectionError::InvalidLocation)?; @@ -113,7 +133,7 @@ impl VertexConnection { .into_iter() .chain(resource.iter().copied()) .collect::>(); - Ok(complete_url(&self.api_base.value, &path)?) + Ok(complete_url(base, &path)?) } } @@ -279,4 +299,50 @@ mod tests { expected ); } + + #[rstest] + #[case::regional_location_keeps_the_global_host( + None, + "us-central1", + "https://aiplatform.googleapis.com/v1/projects/p/locations/us-central1/endpoints/openapi" + )] + #[case::global_location( + None, + "global", + "https://aiplatform.googleapis.com/v1/projects/p/locations/global/endpoints/openapi" + )] + #[case::custom_base_wins( + Some("https://gateway.test/vertex"), + "us-central1", + "https://gateway.test/vertex/v1/projects/p/locations/us-central1/endpoints/openapi" + )] + fn global_only_models_use_the_global_host_with_the_resolved_location( + #[case] api_base: Option<&str>, + #[case] location: &str, + #[case] expected: &str, + ) { + let connection = VertexConnection::resolve( + explicit(Some("p"), Some(location)), + api_base, + &RuntimeDefaults::default(), + &env(&[]), + ) + .unwrap(); + assert_eq!( + connection + .global_resource_url("p", &["endpoints", "openapi"]) + .unwrap(), + expected + ); + } + + #[test] + fn global_only_models_still_reject_an_invalid_location() { + let config = explicit(Some("p"), Some("evil.com#")); + assert_eq!( + VertexConnection::resolve(config, None, &RuntimeDefaults::default(), &env(&[])) + .unwrap_err(), + VertexConnectionError::InvalidLocation + ); + } } diff --git a/litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs index 3f57b5ce8e8..28dc3ba64e0 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs @@ -134,7 +134,7 @@ impl BaseOcrConfig for VertexAIDeepSeekOCRConfig { _params: &Self::OcrParams, environment: &Self::Environment, ) -> Result { - Ok(vertex_connection(request)?.resource_url( + Ok(vertex_connection(request)?.global_resource_url( &environment.project_id, &["endpoints", "openapi", "chat", "completions"], )?)