From a3aceec2f865b30c800b8ea9587582e13d6398ef Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Fri, 18 Sep 2026 17:37:08 -0700 Subject: [PATCH] fix(rust): match Python proxy, ssl_verify and client expiry behavior in the http pool Honor environment proxies whenever Python would use httpx (sync calls, HTTP/2, aiohttp disabled), apply the per-call ssl_verify argument, ignore empty or missing SSL env values the way http_handler.py does, expire pooled clients after an hour so rotated certificates reload, keep the client certificate off media downloads, and decline instead of raising when a litellm global has an unexpected type --- litellm-rust/crates/http/src/config.rs | 21 ++-- litellm-rust/crates/http/src/lib.rs | 4 - litellm-rust/crates/http/src/pool.rs | 80 +++++++++--- litellm-rust/crates/http/src/settings.rs | 82 ++++++++++-- .../llms/src/custom_httpx/llm_http_handler.rs | 2 +- .../crates/llms/src/custom_httpx/transport.rs | 6 - .../crates/python-bridge/python_settings.json | 1 + litellm-rust/crates/python-bridge/src/http.rs | 118 +++++++++++++++--- .../python-bridge/src/python_settings.rs | 6 - .../python-bridge/src/routes/ocr/mod.rs | 2 +- litellm/rust_bridge/settings.py | 8 +- .../test_litellm/rust_bridge/test_settings.py | 4 +- 12 files changed, 262 insertions(+), 72 deletions(-) diff --git a/litellm-rust/crates/http/src/config.rs b/litellm-rust/crates/http/src/config.rs index f27092c1fb5..7772e6cce5b 100644 --- a/litellm-rust/crates/http/src/config.rs +++ b/litellm-rust/crates/http/src/config.rs @@ -16,8 +16,6 @@ pub enum Verify { BuiltInRoots, } -/// One fully resolved client configuration. Every field is a plain value so the pool can -/// key cached clients on it. #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct HttpClientConfig { pub verify: Verify, @@ -30,9 +28,6 @@ pub struct HttpClientConfig { } impl HttpClientConfig { - /// Port of `get_ssl_verify` + `get_ssl_configuration`: the configured (environment-overlaid) - /// `ssl_verify`, then `SSL_CERT_FILE`, then the built-in roots. Settings rustls has no - /// equivalent for are an error instead of a silent no-op. pub fn resolve(settings: &HttpSettings) -> Result { if let Some(level) = &settings.ssl_security_level { return Err(Error::Unsupported { @@ -60,12 +55,11 @@ impl HttpClientConfig { force_ipv4: settings.force_ipv4, http2: settings.http2, user_agent: settings.user_agent.clone(), - trust_proxy_env: settings.trust_proxy_env, + trust_proxy_env: settings.trust_proxy_env || settings.http2 || settings.httpx_transport, connect_timeout: settings.connect_timeout, }) } - /// A builder carrying every shared setting; variants add their own policy on top. pub fn client_builder(&self) -> Result { let base = reqwest::Client::builder().connect_timeout(self.connect_timeout); let with_roots = match &self.verify { @@ -242,6 +236,19 @@ mod tests { ); } + #[rstest] + #[case::aiohttp_default(HttpSettings::default(), false)] + #[case::aiohttp_trust_env(HttpSettings { trust_proxy_env: true, ..HttpSettings::default() }, true)] + #[case::http2_uses_httpx(HttpSettings { http2: true, ..HttpSettings::default() }, true)] + #[case::aiohttp_disabled(HttpSettings { httpx_transport: true, ..HttpSettings::default() }, true)] + fn environment_proxies_apply_whenever_python_would_use_httpx( + #[case] settings: HttpSettings, + #[case] expected: bool, + ) { + let config = HttpClientConfig::resolve(&settings).unwrap(); + assert_eq!(config.trust_proxy_env, expected); + } + #[test] fn missing_ca_bundle_is_a_read_error() { let path = std::env::temp_dir().join("litellm-http-missing-bundle.pem"); diff --git a/litellm-rust/crates/http/src/lib.rs b/litellm-rust/crates/http/src/lib.rs index 9c88e3101a7..c02a82539ff 100644 --- a/litellm-rust/crates/http/src/lib.rs +++ b/litellm-rust/crates/http/src/lib.rs @@ -1,7 +1,3 @@ -//! Rust counterpart of `litellm/llms/custom_httpx/http_handler.py`: the plain HTTP settings -//! LiteLLM exposes, their resolution into one typed client configuration, and a pool that -//! caches `reqwest::Client`s per resolved configuration. - mod config; mod error; mod pool; diff --git a/litellm-rust/crates/http/src/pool.rs b/litellm-rust/crates/http/src/pool.rs index 613ce9c2831..0d9b1abf504 100644 --- a/litellm-rust/crates/http/src/pool.rs +++ b/litellm-rust/crates/http/src/pool.rs @@ -1,33 +1,44 @@ use std::{ collections::HashMap, sync::{Arc, Mutex, MutexGuard, PoisonError}, + time::{Duration, Instant}, }; use reqwest::dns::Resolve; use crate::{config::HttpClientConfig, error::Error}; -/// The client shapes routes need; each is the shared base plus one policy. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum ClientVariant { Provider, NoRedirect, - /// Media downloads: no redirects (the fetcher validates each hop), never a proxy, and the - /// pool's media resolver. Media, } -/// Counterpart of `get_async_httpx_client`: one `reqwest::Client` per resolved configuration -/// and variant, built on first use and shared afterwards. +const CLIENT_TTL: Duration = Duration::from_secs(3600); + +struct PooledClient { + client: reqwest::Client, + built_at: Instant, +} + +type Clients = HashMap<(HttpClientConfig, ClientVariant), PooledClient>; + pub struct HttpClientPool { media_resolver: Arc, - clients: Mutex>, + ttl: Duration, + clients: Mutex, } impl HttpClientPool { pub fn new(media_resolver: Arc) -> Self { + Self::with_ttl(media_resolver, CLIENT_TTL) + } + + pub fn with_ttl(media_resolver: Arc, ttl: Duration) -> Self { Self { media_resolver, + ttl, clients: Mutex::default(), } } @@ -37,15 +48,31 @@ impl HttpClientPool { config: &HttpClientConfig, variant: ClientVariant, ) -> Result { - let key = (config.clone(), variant); - if let Some(client) = self.lock().get(&key) { - return Ok(client.clone()); + let effective = match variant { + ClientVariant::Media => HttpClientConfig { + client_certificate: None, + ..config.clone() + }, + ClientVariant::Provider | ClientVariant::NoRedirect => config.clone(), + }; + let key = (effective, variant); + if let Some(pooled) = self.lock().get(&key) + && pooled.built_at.elapsed() < self.ttl + { + return Ok(pooled.client.clone()); } - let client = self.apply(variant, config.client_builder()?).build()?; - Ok(self.lock().entry(key).or_insert(client).clone()) + let client = self.apply(variant, key.0.client_builder()?).build()?; + self.lock().insert( + key, + PooledClient { + client: client.clone(), + built_at: Instant::now(), + }, + ); + Ok(client) } - fn lock(&self) -> MutexGuard<'_, HashMap<(HttpClientConfig, ClientVariant), reqwest::Client>> { + fn lock(&self) -> MutexGuard<'_, Clients> { self.clients.lock().unwrap_or_else(PoisonError::into_inner) } @@ -102,8 +129,6 @@ mod tests { } } - /// Answers every request on every connection with `status_line` and counts connections, - /// so a reused client shows up as a reused keep-alive connection. async fn serve( status_line: &'static str, ) -> (SocketAddr, Arc, Arc>>) { @@ -168,6 +193,33 @@ mod tests { assert_eq!(connections.load(Ordering::SeqCst), 3); } + #[tokio::test] + async fn expired_clients_are_rebuilt() { + let (address, connections, _) = serve("HTTP/1.1 204 No Content").await; + let url = format!("http://{address}"); + let pool = HttpClientPool::with_ttl( + Arc::new(FixedResolver(([192, 0, 2, 1], 80).into())), + Duration::ZERO, + ); + get(&pool, &config("a"), ClientVariant::Provider, &url).await; + get(&pool, &config("a"), ClientVariant::Provider, &url).await; + assert_eq!(connections.load(Ordering::SeqCst), 2); + } + + #[test] + fn media_variant_never_loads_the_client_certificate() { + let pool = pool(); + let with_identity = HttpClientConfig { + client_certificate: Some(std::env::temp_dir().join("litellm-http-absent-client.pem")), + ..config("a") + }; + assert!( + pool.client(&with_identity, ClientVariant::Provider) + .is_err() + ); + assert!(pool.client(&with_identity, ClientVariant::Media).is_ok()); + } + #[test] fn build_failures_are_not_cached() { let pool = pool(); diff --git a/litellm-rust/crates/http/src/settings.rs b/litellm-rust/crates/http/src/settings.rs index 45aab0d6fa5..55ac471bba9 100644 --- a/litellm-rust/crates/http/src/settings.rs +++ b/litellm-rust/crates/http/src/settings.rs @@ -1,6 +1,8 @@ -use std::{path::PathBuf, time::Duration}; +use std::{ + path::{Path, PathBuf}, + time::Duration, +}; -/// `litellm.ssl_verify` / `SSL_VERIFY`: a bool or a CA bundle path. #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub enum SslVerify { Enabled, @@ -18,7 +20,6 @@ impl SslVerify { } } -/// The plain inputs `http_handler.py` reads from `litellm.*` globals and the environment. #[derive(Clone, Debug, PartialEq, Eq)] pub struct HttpSettings { pub ssl_verify: Option, @@ -28,6 +29,7 @@ pub struct HttpSettings { pub ssl_ecdh_curve: Option, pub force_ipv4: bool, pub http2: bool, + pub httpx_transport: bool, pub user_agent: Option, pub trust_proxy_env: bool, pub connect_timeout: Duration, @@ -43,6 +45,7 @@ impl Default for HttpSettings { ssl_ecdh_curve: None, force_ipv4: false, http2: false, + httpx_transport: false, user_agent: None, trust_proxy_env: false, connect_timeout: Duration::from_secs(5), @@ -51,11 +54,6 @@ impl Default for HttpSettings { } impl HttpSettings { - /// Overlay the environment variables `http_handler.py` consults, with the same precedence: - /// `SSL_VERIFY`, `SSL_CERTIFICATE`, `SSL_SECURITY_LEVEL`, `SSL_ECDH_CURVE` and - /// `LITELLM_USER_AGENT` win over the configured value; `SSL_CERT_FILE` only applies when - /// verification is on without an explicit bundle; `LITELLM_HTTP2` and `AIOHTTP_TRUST_ENV` - /// can only turn their switch on. pub fn with_environment(self, env: &(dyn Fn(&str) -> Option + Sync)) -> Self { let enabled = |name: &str| env(name).is_some_and(|value| value.trim().eq_ignore_ascii_case("true")); @@ -68,15 +66,32 @@ impl HttpSettings { .or(self.ssl_cert_file), ssl_certificate: env("SSL_CERTIFICATE") .map(PathBuf::from) - .or(self.ssl_certificate), - ssl_security_level: env("SSL_SECURITY_LEVEL").or(self.ssl_security_level), - ssl_ecdh_curve: env("SSL_ECDH_CURVE").or(self.ssl_ecdh_curve), + .or(self.ssl_certificate) + .filter(|path| !path.as_os_str().is_empty()), + ssl_security_level: env("SSL_SECURITY_LEVEL") + .or(self.ssl_security_level) + .filter(|level| !level.is_empty()), + ssl_ecdh_curve: env("SSL_ECDH_CURVE") + .or(self.ssl_ecdh_curve) + .filter(|curve| !curve.is_empty()), http2: self.http2 || enabled("LITELLM_HTTP2"), + httpx_transport: self.httpx_transport || enabled("DISABLE_AIOHTTP_TRANSPORT"), user_agent: env("LITELLM_USER_AGENT").or(self.user_agent), trust_proxy_env: self.trust_proxy_env || enabled("AIOHTTP_TRUST_ENV"), ..self } } + + pub fn without_missing_files(self, exists: &dyn Fn(&Path) -> bool) -> Self { + Self { + ssl_verify: match self.ssl_verify { + Some(SslVerify::CaBundle(path)) if !exists(&path) => Some(SslVerify::Enabled), + other => other, + }, + ssl_cert_file: self.ssl_cert_file.filter(|path| exists(path)), + ..self + } + } } #[cfg(test)] @@ -152,6 +167,46 @@ mod tests { assert_eq!(configured.clone().with_environment(&no_env), configured); } + #[test] + fn empty_environment_values_clear_the_setting_like_python_truthiness() { + let settings = HttpSettings { + ssl_certificate: Some("/configured/client.pem".into()), + ssl_security_level: Some("configured".into()), + ssl_ecdh_curve: Some("X25519".into()), + ..HttpSettings::default() + } + .with_environment(&env_of(&[ + ("SSL_CERTIFICATE", ""), + ("SSL_SECURITY_LEVEL", ""), + ("SSL_ECDH_CURVE", ""), + ])); + assert_eq!(settings.ssl_certificate, None); + assert_eq!(settings.ssl_security_level, None); + assert_eq!(settings.ssl_ecdh_curve, None); + } + + #[test] + fn missing_files_fall_back_to_default_verification() { + let settings = HttpSettings { + ssl_verify: Some(SslVerify::CaBundle("/absent/roots.pem".into())), + ssl_cert_file: Some("/absent/env.pem".into()), + ..HttpSettings::default() + } + .without_missing_files(&|_| false); + assert_eq!(settings.ssl_verify, Some(SslVerify::Enabled)); + assert_eq!(settings.ssl_cert_file, None); + } + + #[test] + fn existing_files_are_kept() { + let settings = HttpSettings { + ssl_verify: Some(SslVerify::CaBundle("/present/roots.pem".into())), + ssl_cert_file: Some("/present/env.pem".into()), + ..HttpSettings::default() + }; + assert_eq!(settings.clone().without_missing_files(&|_| true), settings); + } + #[rstest] #[case("true", true)] #[case("True", true)] @@ -159,11 +214,14 @@ mod tests { #[case("1", false)] fn boolean_switches_only_turn_on_for_true(#[case] value: &'static str, #[case] expected: bool) { let env = move |name: &str| match name { - "LITELLM_HTTP2" | "AIOHTTP_TRUST_ENV" => Some(value.to_string()), + "LITELLM_HTTP2" | "AIOHTTP_TRUST_ENV" | "DISABLE_AIOHTTP_TRANSPORT" => { + Some(value.to_string()) + } _ => None, }; let settings = HttpSettings::default().with_environment(&env); assert_eq!(settings.http2, expected); + assert_eq!(settings.httpx_transport, expected); assert_eq!(settings.trust_proxy_env, expected); } } diff --git a/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs b/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs index 425b71efc78..876fa0aae87 100644 --- a/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs +++ b/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs @@ -42,7 +42,7 @@ impl OcrClient { pool: &HttpClientPool, config: &HttpClientConfig, vertex_auth: VertexAuth, - ) -> Result { + ) -> Result { Ok(Self { provider_http: pool.client(config, ClientVariant::Provider)?, polling_http: pool.client(config, ClientVariant::NoRedirect)?, diff --git a/litellm-rust/crates/llms/src/custom_httpx/transport.rs b/litellm-rust/crates/llms/src/custom_httpx/transport.rs index 8e5e1a8832d..172dd96476a 100644 --- a/litellm-rust/crates/llms/src/custom_httpx/transport.rs +++ b/litellm-rust/crates/llms/src/custom_httpx/transport.rs @@ -26,12 +26,6 @@ impl From for Error { } } -impl From for Error { - fn from(error: litellm_http::Error) -> Self { - Self::Connect(error.to_string()) - } -} - #[cfg(test)] mod tests { #[tokio::test] diff --git a/litellm-rust/crates/python-bridge/python_settings.json b/litellm-rust/crates/python-bridge/python_settings.json index a6cd959c6de..64dd01a0a84 100644 --- a/litellm-rust/crates/python-bridge/python_settings.json +++ b/litellm-rust/crates/python-bridge/python_settings.json @@ -7,6 +7,7 @@ "force_ipv4", "http2", "aiohttp_trust_env", + "disable_aiohttp_transport", "user_agent" ] } diff --git a/litellm-rust/crates/python-bridge/src/http.rs b/litellm-rust/crates/python-bridge/src/http.rs index de6fb5bb96d..cf7ca05515e 100644 --- a/litellm-rust/crates/python-bridge/src/http.rs +++ b/litellm-rust/crates/python-bridge/src/http.rs @@ -1,5 +1,5 @@ use std::{ - path::PathBuf, + path::{Path, PathBuf}, sync::{Arc, LazyLock}, }; @@ -12,27 +12,46 @@ use crate::{errors::RustBridgeDeclined, python_settings::PythonSettings}; static POOL: LazyLock = LazyLock::new(|| HttpClientPool::new(Arc::new(PublicDnsResolver))); -/// Keyword arguments that carry a live Python HTTP client or session. They cannot cross into -/// Rust, so a call that supplies one stays on the Python path. const LIVE_CLIENT_ARGUMENTS: [&str; 3] = ["client", "shared_session", "aclient_session"]; pub(crate) fn pool() -> &'static HttpClientPool { &POOL } -/// The client configuration for one call: the `litellm.*` HTTP settings with the environment -/// overlaid, the same way `http_handler.py` combines them. pub(crate) fn call_config( py: Python<'_>, kwargs: &Bound<'_, PyDict>, + asynchronous: bool, ) -> PyResult { decline_live_clients(kwargs)?; - let settings = settings(&PythonSettings::Http.read(py)?)? + let configured = settings(&PythonSettings::Http.read(py)?)? .with_environment(&|name| std::env::var(name).ok()); + let settings = for_call(configured, call_ssl_verify(kwargs)?, asynchronous) + .without_missing_files(&|path: &Path| path.exists()); HttpClientConfig::resolve(&settings) .map_err(|error| RustBridgeDeclined::new_err(error.to_string())) } +fn call_ssl_verify(kwargs: &Bound<'_, PyDict>) -> PyResult> { + kwargs + .get_item("ssl_verify")? + .filter(|value| !value.is_none()) + .map(|value| ssl_verify(&value, "the ssl_verify argument")) + .transpose() +} + +fn for_call( + configured: HttpSettings, + call_ssl_verify: Option, + asynchronous: bool, +) -> HttpSettings { + HttpSettings { + ssl_verify: call_ssl_verify.or(configured.ssl_verify), + httpx_transport: configured.httpx_transport || !asynchronous, + ..configured + } +} + pub(crate) fn decline_live_clients(kwargs: &Bound<'_, PyDict>) -> PyResult<()> { for name in LIVE_CLIENT_ARGUMENTS { if kwargs.get_item(name)?.is_some_and(|value| !value.is_none()) { @@ -53,25 +72,31 @@ struct PythonHttpSettings<'py> { force_ipv4: bool, http2: bool, aiohttp_trust_env: bool, + disable_aiohttp_transport: bool, user_agent: String, } fn settings(value: &Bound<'_, PyAny>) -> PyResult { - let python: PythonHttpSettings = value.extract()?; + let python: PythonHttpSettings = value.extract().map_err(|error: PyErr| { + RustBridgeDeclined::new_err(format!( + "litellm HTTP settings cannot be used by the Rust route: {error}" + )) + })?; Ok(HttpSettings { - ssl_verify: Some(ssl_verify(&python.ssl_verify)?), + ssl_verify: Some(ssl_verify(&python.ssl_verify, "litellm.ssl_verify")?), ssl_certificate: python.ssl_certificate.map(PathBuf::from), ssl_security_level: python.ssl_security_level, ssl_ecdh_curve: python.ssl_ecdh_curve, force_ipv4: python.force_ipv4, http2: python.http2, + httpx_transport: python.disable_aiohttp_transport, user_agent: Some(python.user_agent), trust_proxy_env: python.aiohttp_trust_env, ..HttpSettings::default() }) } -fn ssl_verify(value: &Bound<'_, PyAny>) -> PyResult { +fn ssl_verify(value: &Bound<'_, PyAny>, source: &str) -> PyResult { if let Ok(enabled) = value.extract::() { return Ok(if enabled { SslVerify::Enabled @@ -82,9 +107,9 @@ fn ssl_verify(value: &Bound<'_, PyAny>) -> PyResult { if let Ok(path) = value.extract::() { return Ok(SslVerify::parse(&path)); } - Err(RustBridgeDeclined::new_err( - "litellm.ssl_verify is a live Python object and cannot be used by the Rust route", - )) + Err(RustBridgeDeclined::new_err(format!( + "{source} is a live Python object and cannot be used by the Rust route" + ))) } #[cfg(test)] @@ -95,8 +120,6 @@ mod tests { use super::*; use crate::python_settings::CONTRACT; - /// A stand-in for `http_settings()` carrying exactly the fields the contract declares, so a - /// field Rust reads but Python does not return fails here. fn python_settings<'py>(py: Python<'py>, overrides: &str) -> Bound<'py, PyAny> { let source = format!( " @@ -110,6 +133,7 @@ defaults = dict( force_ipv4=False, http2=False, aiohttp_trust_env=False, + disable_aiohttp_transport=False, user_agent='litellm/test', ) defaults.update(dict({overrides})) @@ -153,6 +177,7 @@ ssl_ecdh_curve='X25519', force_ipv4=True, http2=True, aiohttp_trust_env=True, +disable_aiohttp_transport=True, user_agent='litellm/9.9.9', ", )) @@ -166,6 +191,7 @@ user_agent='litellm/9.9.9', ssl_ecdh_curve: Some("X25519".into()), force_ipv4: true, http2: true, + httpx_transport: true, user_agent: Some("litellm/9.9.9".into()), trust_proxy_env: true, ..HttpSettings::default() @@ -214,6 +240,70 @@ user_agent='litellm/9.9.9', }); } + #[test] + fn mistyped_python_settings_decline_instead_of_raising() { + Python::initialize(); + Python::attach(|py| { + let error = settings(&python_settings(py, "force_ipv4='yes'")).unwrap_err(); + assert!(error.is_instance_of::(py)); + }); + } + + #[test] + fn call_ssl_verify_beats_the_configured_and_environment_value() { + Python::initialize(); + Python::attach(|py| { + let kwargs = PyDict::new(py); + kwargs.set_item("ssl_verify", false).unwrap(); + let configured = HttpSettings { + ssl_verify: Some(SslVerify::Enabled), + ..HttpSettings::default() + }; + let settings = for_call(configured, call_ssl_verify(&kwargs).unwrap(), true); + assert_eq!(settings.ssl_verify, Some(SslVerify::Disabled)); + }); + } + + #[test] + fn absent_call_ssl_verify_keeps_the_configured_value() { + Python::initialize(); + Python::attach(|py| { + let kwargs = PyDict::new(py); + kwargs.set_item("ssl_verify", py.None()).unwrap(); + let configured = HttpSettings { + ssl_verify: Some(SslVerify::Disabled), + ..HttpSettings::default() + }; + let settings = for_call(configured.clone(), call_ssl_verify(&kwargs).unwrap(), true); + assert_eq!(settings, configured); + }); + } + + #[test] + fn live_ssl_context_argument_declines() { + Python::initialize(); + Python::attach(|py| { + let kwargs = PyDict::new(py); + kwargs + .set_item("ssl_verify", py.eval(c"object()", None, None).unwrap()) + .unwrap(); + let error = call_ssl_verify(&kwargs).unwrap_err(); + assert!(error.is_instance_of::(py)); + }); + } + + #[rstest] + #[case::asynchronous(true, false)] + #[case::synchronous(false, true)] + fn synchronous_calls_honor_environment_proxies_like_httpx( + #[case] asynchronous: bool, + #[case] expected: bool, + ) { + let settings = for_call(HttpSettings::default(), None, asynchronous); + let config = HttpClientConfig::resolve(&settings).unwrap(); + assert_eq!(config.trust_proxy_env, expected); + } + #[rstest] #[case::client("client")] #[case::shared_session("shared_session")] diff --git a/litellm-rust/crates/python-bridge/src/python_settings.rs b/litellm-rust/crates/python-bridge/src/python_settings.rs index c5f9f309615..dcb46e7d2b5 100644 --- a/litellm-rust/crates/python-bridge/src/python_settings.rs +++ b/litellm-rust/crates/python-bridge/src/python_settings.rs @@ -2,12 +2,6 @@ use pyo3::prelude::*; const MODULE: &str = "litellm.rust_bridge.settings"; -/// Every group of `litellm.*` module globals the native routes read. Environment overrides are -/// applied on the Rust side, so each function returns only what the Python process configured. -/// A group is deleted once Rust owns loading that configuration, so this enum only shrinks. -/// -/// `litellm/rust_bridge/settings.py` is the only Python module behind it, and -/// `python_settings.json` pins the fields each function returns on both sides. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum PythonSettings { Http, diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs index d9aeeb234f7..174d0ff18c8 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs @@ -37,7 +37,7 @@ fn run_ocr( kwargs: Bound<'_, PyDict>, asynchronous: bool, ) -> PyResult> { - let config = http::call_config(py, &kwargs)?; + let config = http::call_config(py, &kwargs, asynchronous)?; let client = OcrClient::new(http::pool(), &config, VERTEX_AUTH.clone()) .map_err(|error| RustBridgeDeclined::new_err(error.to_string()))?; run_legacy_call( diff --git a/litellm/rust_bridge/settings.py b/litellm/rust_bridge/settings.py index a8229b12d13..ad478fb28b5 100644 --- a/litellm/rust_bridge/settings.py +++ b/litellm/rust_bridge/settings.py @@ -1,9 +1,3 @@ -"""The `litellm.*` module globals the native routes read. - -Environment variables that override these are applied in Rust, so nothing here reads `os.environ`. -`litellm-rust/crates/python-bridge/python_settings.json` pins the fields each function returns. -""" - from __future__ import annotations from dataclasses import dataclass @@ -18,6 +12,7 @@ class HttpSettings: force_ipv4: bool http2: bool aiohttp_trust_env: bool + disable_aiohttp_transport: bool user_agent: str @@ -33,5 +28,6 @@ def http_settings() -> HttpSettings: force_ipv4=litellm.force_ipv4, http2=litellm.http2, aiohttp_trust_env=litellm.aiohttp_trust_env, + disable_aiohttp_transport=litellm.disable_aiohttp_transport, user_agent=default_user_agent(), ) diff --git a/tests/test_litellm/rust_bridge/test_settings.py b/tests/test_litellm/rust_bridge/test_settings.py index 618fa400136..dce2324de08 100644 --- a/tests/test_litellm/rust_bridge/test_settings.py +++ b/tests/test_litellm/rust_bridge/test_settings.py @@ -26,6 +26,7 @@ def test_http_settings_reads_the_litellm_globals(monkeypatch: pytest.MonkeyPatch monkeypatch.setattr(litellm, "force_ipv4", True) monkeypatch.setattr(litellm, "http2", True) monkeypatch.setattr(litellm, "aiohttp_trust_env", True) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) assert settings.http_settings() == settings.HttpSettings( ssl_verify="/etc/ssl/corp.pem", @@ -35,7 +36,8 @@ def test_http_settings_reads_the_litellm_globals(monkeypatch: pytest.MonkeyPatch force_ipv4=True, http2=True, aiohttp_trust_env=True, - user_agent=settings.http_settings().user_agent, + disable_aiohttp_transport=True, + user_agent=default_user_agent(), )