diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs index af08e685f94..3b85697dc8e 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs @@ -2,6 +2,7 @@ use crate::cache::{CacheCall, Cached, PythonCache, Selection}; use litellm_host_python::{PythonHostCalls, PythonOwned}; use bytes::Bytes; +use litellm_auth::VertexParams; use litellm_host_python::{InvokeError, PythonBinding, from_py, to_py}; use litellm_http::transport::Error as TransportError; use litellm_inference_messages::{ @@ -23,6 +24,7 @@ use crate::{ errors::{RustUpstreamError, route_error_to_pyerr}, marshal::{optional_timeout, project_optional_fields, public_response, python_timeout_seconds}, routes::RequestView, + runtime_defaults::RuntimeDefault, }; const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.messages.route_host"; @@ -143,7 +145,20 @@ impl MessagesPythonHost { let extra_headers = self.merged_headers(py, arguments)?; let provider_specific_header = self.provider_specific_header(py, arguments)?; let litellm_params = project_litellm_params(argument)?; - let runtime_defaults = crate::runtime_defaults::read(py)?; + let supplied = [VertexParams::PROJECT, VertexParams::LOCATION] + .iter() + .flat_map(|spec| spec.wire.iter().copied()) + .filter_map(|name| argument(name).map(|value| value.map(|_| name)).transpose()) + .collect::>>()?; + let wanted = api_base + .is_none() + .then_some(RuntimeDefault::ApiBase) + .into_iter() + .chain(RuntimeDefault::vertex_fallbacks(|spec| { + !spec.wire.iter().any(|name| supplied.contains(name)) + })) + .collect::>(); + let runtime_defaults = crate::runtime_defaults::read(py, &wanted)?; Ok(messages_body(body).and_then(|body| { Ok(MessagesCall { body, diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs index 10b937cb7b1..f36d8d2a68f 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs @@ -12,7 +12,7 @@ use super::{document::FileDocumentInput, errors::to_pyerr as ocr_error_to_pyerr} use crate::{ credentials::{self, CallerTokenProvider}, marshal::{project_optional_fields, python_timeout_seconds, request_input_sources}, - runtime_defaults, + runtime_defaults::{self, RuntimeDefault}, }; /// What the host keeps after projection: the caller's token callable that answers the @@ -124,8 +124,16 @@ pub(super) fn project_request( let names = specs.iter().map(|spec| spec.name).collect::>(); let optional_params = project_optional_fields(names.iter().copied(), |name| present(kwargs, bound, name))?; - let runtime_defaults = - runtime_defaults::read(bound.py()).map_err(|error| settings_error(bound.py(), error))?; + let wanted = RuntimeDefault::vertex_fallbacks(|spec| { + spec.wire.iter().all(|name| names.contains(name)) + && !spec + .wire + .iter() + .any(|name| optional_params.contains_key(*name)) + }) + .collect::>(); + let runtime_defaults = runtime_defaults::read(bound.py(), &wanted) + .map_err(|error| settings_error(bound.py(), error))?; let input_sources = request_input_sources( kwargs, names diff --git a/litellm-rust/crates/python-bridge/src/runtime_defaults.rs b/litellm-rust/crates/python-bridge/src/runtime_defaults.rs index c4a9f254500..75d13812c79 100644 --- a/litellm-rust/crates/python-bridge/src/runtime_defaults.rs +++ b/litellm-rust/crates/python-bridge/src/runtime_defaults.rs @@ -1,3 +1,4 @@ +use litellm_auth::{ParamSpec, VertexParams}; use litellm_llms::base_llm::connection::RuntimeDefaults; use pyo3::prelude::*; @@ -6,23 +7,64 @@ use crate::{ python_settings::{PythonSettings, Snapshot}, }; -/// The `litellm.` module defaults, read once per call into the typed runtime-defaults -/// layer that provider resolvers rank between explicit arguments and the environment. -pub(crate) fn read(py: Python<'_>) -> PyResult { +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum RuntimeDefault { + ApiBase, + VertexProject, + VertexLocation, +} + +impl RuntimeDefault { + const fn name(self) -> &'static str { + match self { + Self::ApiBase => "api_base", + Self::VertexProject => "vertex_project", + Self::VertexLocation => "vertex_location", + } + } + + /// The Vertex defaults whose spec the call leaves `unset`. + pub(crate) fn vertex_fallbacks( + unset: impl Fn(&ParamSpec) -> bool, + ) -> impl Iterator { + [ + (Self::VertexProject, VertexParams::PROJECT), + (Self::VertexLocation, VertexParams::LOCATION), + ] + .into_iter() + .filter(move |(_, spec)| unset(spec)) + .map(|(default, _)| default) + } +} + +/// The `litellm.` module defaults the call falls back to, read once per call into the +/// typed runtime-defaults layer that provider resolvers rank between explicit arguments and +/// the environment. A default the call does not fall back to is never read, so a malformed +/// one cannot fail an unrelated call. +pub(crate) fn read(py: Python<'_>, wanted: &[RuntimeDefault]) -> PyResult { + if wanted.is_empty() { + return Ok(RuntimeDefaults::default()); + } match PythonSettings::ProviderDefaults.read_or_unset(py)? { - Some(snapshot) => project(&snapshot).map_err(PyErr::from), + Some(snapshot) => project(&snapshot, wanted).map_err(PyErr::from), None => Ok(RuntimeDefaults::default()), } } -fn project(snapshot: &Snapshot<'_>) -> Result { - let read = |name: &'static str| { - snapshot.read(&FieldSpec::new(name, |field| field.falsy_optional_string())) +fn project( + snapshot: &Snapshot<'_>, + wanted: &[RuntimeDefault], +) -> Result { + let read = |default: RuntimeDefault| match wanted.contains(&default) { + true => snapshot.read(&FieldSpec::new(default.name(), |field| { + field.falsy_optional_string() + })), + false => Ok(None), }; Ok(RuntimeDefaults { - api_base: read("api_base")?, - vertex_project: read("vertex_project")?, - vertex_location: read("vertex_location")?, + api_base: read(RuntimeDefault::ApiBase)?, + vertex_project: read(RuntimeDefault::VertexProject)?, + vertex_location: read(RuntimeDefault::VertexLocation)?, }) } @@ -32,11 +74,20 @@ mod tests { use super::*; - fn projected(source: &std::ffi::CStr) -> Result { + const EVERY: [RuntimeDefault; 3] = [ + RuntimeDefault::ApiBase, + RuntimeDefault::VertexProject, + RuntimeDefault::VertexLocation, + ]; + + fn projected( + source: &std::ffi::CStr, + wanted: &[RuntimeDefault], + ) -> Result { Python::initialize(); Python::attach(|py| { let value = py.eval(source, None, None).unwrap(); - project(&PythonSettings::ProviderDefaults.snapshot(value)) + project(&PythonSettings::ProviderDefaults.snapshot(value), wanted) .map_err(|error| PyErr::from(error).value(py).to_string()) }) } @@ -58,13 +109,43 @@ mod tests { #[case] source: &std::ffi::CStr, #[case] expected: RuntimeDefaults, ) { - assert_eq!(projected(source).unwrap(), expected); + assert_eq!(projected(source, &EVERY).unwrap(), expected); + } + + #[rstest] + #[case::nothing_wanted(&[], RuntimeDefaults::default())] + #[case::only_the_api_base( + &[RuntimeDefault::ApiBase], + RuntimeDefaults { api_base: Some("https://runtime.test".into()), ..Default::default() } + )] + fn a_malformed_default_the_call_does_not_use_is_never_read( + #[case] wanted: &[RuntimeDefault], + #[case] expected: RuntimeDefaults, + ) { + let source = c"__import__('types').SimpleNamespace(api_base='https://runtime.test', vertex_project=5, vertex_location=5)"; + assert_eq!(projected(source, wanted).unwrap(), expected); + } + + #[rstest] + #[case::both_unset(&[], vec![RuntimeDefault::VertexProject, RuntimeDefault::VertexLocation])] + #[case::project_on_the_call(&["vertex_ai_project"], vec![RuntimeDefault::VertexLocation])] + #[case::both_on_the_call(&["vertex_project", "vertex_location"], vec![])] + fn vertex_fallbacks_skip_the_settings_the_call_supplies( + #[case] supplied: &[&str], + #[case] expected: Vec, + ) { + let fallbacks = RuntimeDefault::vertex_fallbacks(|spec| { + !spec.wire.iter().any(|name| supplied.contains(name)) + }) + .collect::>(); + assert_eq!(fallbacks, expected); } #[test] fn a_misconfigured_module_default_names_the_setting() { let error = projected( c"__import__('types').SimpleNamespace(api_base=1, vertex_project=None, vertex_location=None)", + &EVERY, ) .unwrap_err(); assert!(error.contains("provider_defaults.api_base"), "{error}"); @@ -81,12 +162,19 @@ mod tests { None, ) .unwrap(); - let first = - project(&PythonSettings::ProviderDefaults.snapshot(namespace.clone())).unwrap(); + let first = project( + &PythonSettings::ProviderDefaults.snapshot(namespace.clone()), + &EVERY, + ) + .unwrap(); namespace .setattr("api_base", "https://second.test") .unwrap(); - let second = project(&PythonSettings::ProviderDefaults.snapshot(namespace)).unwrap(); + let second = project( + &PythonSettings::ProviderDefaults.snapshot(namespace), + &EVERY, + ) + .unwrap(); assert_eq!( (first.api_base.as_deref(), second.api_base.as_deref()), (Some("https://first.test"), Some("https://second.test"))