From abb9618971e80649d3db5e1ee85eaec82384ede4 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Fri, 18 Sep 2026 23:28:11 +0000 Subject: [PATCH 01/15] feat(rust): add litellm-http client pool and inject it into the OCR route Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/Cargo.lock | 13 + litellm-rust/Cargo.toml | 1 + litellm-rust/crates/core/Cargo.toml | 1 + litellm-rust/crates/core/src/ocr/client.rs | 9 +- litellm-rust/crates/core/tests/ocr.rs | 46 ++- litellm-rust/crates/http/Cargo.toml | 14 + litellm-rust/crates/http/src/config.rs | 327 ++++++++++++++++++ litellm-rust/crates/http/src/lib.rs | 11 + litellm-rust/crates/http/src/pool.rs | 172 +++++++++ litellm-rust/crates/http/src/settings.rs | 171 +++++++++ litellm-rust/crates/llms/Cargo.toml | 1 + .../llms/src/base_llm/ocr/transformation.rs | 1 - .../llms/src/custom_httpx/llm_http_handler.rs | 46 +-- .../crates/llms/src/custom_httpx/media.rs | 38 +- .../crates/llms/src/custom_httpx/transport.rs | 6 + litellm-rust/crates/python-bridge/Cargo.toml | 1 + litellm-rust/crates/python-bridge/src/http.rs | 234 +++++++++++++ litellm-rust/crates/python-bridge/src/lib.rs | 1 + .../python-bridge/src/routes/ocr/mod.rs | 6 +- 19 files changed, 1037 insertions(+), 62 deletions(-) create mode 100644 litellm-rust/crates/http/Cargo.toml create mode 100644 litellm-rust/crates/http/src/config.rs create mode 100644 litellm-rust/crates/http/src/lib.rs create mode 100644 litellm-rust/crates/http/src/pool.rs create mode 100644 litellm-rust/crates/http/src/settings.rs create mode 100644 litellm-rust/crates/python-bridge/src/http.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index c359ca19986..ffbdc80efd3 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -2058,6 +2058,7 @@ dependencies = [ "litellm-auth-aws", "litellm-callbacks", "litellm-core-utils", + "litellm-http", "litellm-llms", "litellm-types", "mime_guess", @@ -2125,6 +2126,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "litellm-http" +version = "0.1.0" +dependencies = [ + "reqwest 0.12.28", + "rstest", + "thiserror 2.0.19", + "tokio", +] + [[package]] name = "litellm-llms" version = "0.1.0" @@ -2142,6 +2153,7 @@ dependencies = [ "litellm-callbacks", "litellm-core-utils", "litellm-framing", + "litellm-http", "litellm-types", "reqwest 0.12.28", "rstest", @@ -2166,6 +2178,7 @@ dependencies = [ "litellm-callbacks-legacy", "litellm-core", "litellm-host-python", + "litellm-http", "litellm-llms", "litellm-token-counter", "litellm-types", diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index ffdbf64bb49..eeb473cd2e9 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -17,6 +17,7 @@ litellm-auth = { path = "crates/auth" } litellm-auth-aws = { path = "crates/auth-aws" } litellm-auth-azure = { path = "crates/auth-azure" } litellm-auth-gcp = { path = "crates/auth-gcp" } +litellm-http = { path = "crates/http" } litellm-llms = { path = "crates/llms" } litellm-types = { path = "crates/types" } litellm-core-utils = { path = "crates/core-utils" } diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index db6cfc4b340..047188c74f9 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -15,6 +15,7 @@ futures-util.workspace = true base64.workspace = true litellm-auth.workspace = true litellm-auth-aws.workspace = true +litellm-http.workspace = true litellm-llms.workspace = true moka.workspace = true mime_guess = "2.0.5" diff --git a/litellm-rust/crates/core/src/ocr/client.rs b/litellm-rust/crates/core/src/ocr/client.rs index 03782d91f24..21c81505cff 100644 --- a/litellm-rust/crates/core/src/ocr/client.rs +++ b/litellm-rust/crates/core/src/ocr/client.rs @@ -1,3 +1,4 @@ +use litellm_http::{HttpClientConfig, HttpClientPool}; use litellm_llms::{ base_llm::ocr::{error::Error, transformation::LiteLLMOcrResponse}, custom_httpx::llm_http_handler::OcrClient, @@ -15,6 +16,10 @@ pub async fn perform( litellm_callbacks::run::run(ocr_machine(client.clone()), &LocalOcrHost::new(request)).await } -pub async fn ocr(request: LiteLLMOcrRequest) -> Result { - perform(&OcrClient::shared()?, request).await +pub async fn ocr( + pool: &HttpClientPool, + config: &HttpClientConfig, + request: LiteLLMOcrRequest, +) -> Result { + perform(&OcrClient::new(pool, config)?, request).await } diff --git a/litellm-rust/crates/core/tests/ocr.rs b/litellm-rust/crates/core/tests/ocr.rs index 1f591d74d5d..d59c6179aa5 100644 --- a/litellm-rust/crates/core/tests/ocr.rs +++ b/litellm-rust/crates/core/tests/ocr.rs @@ -5,6 +5,7 @@ use litellm_callbacks::{ host::{Host, HostOp, HostResult}, machine::{HostFailure, Machine, MachineStep}, }; +use litellm_http::{HttpClientConfig, HttpClientPool, HttpSettings, Verify}; use litellm_llms::{ base_llm::ocr::{ error::Error as OcrError, @@ -171,25 +172,44 @@ async fn facade_retains_native_response_when_requested() { } #[tokio::test] -async fn facade_uses_the_injected_http_client() { +async fn facade_uses_the_injected_http_pool_configuration() { let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; - let mut default_headers = reqwest::header::HeaderMap::new(); - default_headers.insert( - "x-transport-owner", - reqwest::header::HeaderValue::from_static("host"), - ); - let provider_http = reqwest::Client::builder() - .default_headers(default_headers) - .build() - .unwrap(); - crate::ocr::client::perform( - &OcrClient::new(provider_http).unwrap(), + let settings = HttpSettings { + user_agent: Some("host-owned/1".into()), + ..HttpSettings::default() + }; + let config = HttpClientConfig::resolve(&settings, None).unwrap(); + crate::ocr::client::ocr( + &HttpClientPool::new(), + &config, wire_request("mistral/model", &base, json!({})), ) .await .unwrap(); server.await.unwrap(); - assert!(seen.lock().unwrap()[0].contains("x-transport-owner: host")); + assert!(seen.lock().unwrap()[0].contains("user-agent: host-owned/1")); +} + +#[tokio::test] +async fn unbuildable_http_configuration_fails_before_dispatch() { + let (base, _seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; + let config = HttpClientConfig { + verify: Verify::CaBundle(std::env::temp_dir().join("litellm-ocr-missing-bundle.pem")), + ..HttpClientConfig::resolve(&HttpSettings::default(), None).unwrap() + }; + let error = crate::ocr::client::ocr( + &HttpClientPool::new(), + &config, + wire_request("mistral/model", &base, json!({})), + ) + .await + .unwrap_err(); + server.abort(); + assert!(matches!( + error, + OcrError::Transport(litellm_llms::custom_httpx::transport::Error::Connect(_)) + )); + assert!(error.to_string().contains("litellm-ocr-missing-bundle.pem")); } fn event_name(event: &CallEvent) -> &'static str { diff --git a/litellm-rust/crates/http/Cargo.toml b/litellm-rust/crates/http/Cargo.toml new file mode 100644 index 00000000000..48ea4e66cef --- /dev/null +++ b/litellm-rust/crates/http/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "litellm-http" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +reqwest.workspace = true +thiserror.workspace = true + +[dev-dependencies] +rstest.workspace = true +tokio.workspace = true diff --git a/litellm-rust/crates/http/src/config.rs b/litellm-rust/crates/http/src/config.rs new file mode 100644 index 00000000000..86f1a9b43b6 --- /dev/null +++ b/litellm-rust/crates/http/src/config.rs @@ -0,0 +1,327 @@ +use std::{ + net::{IpAddr, Ipv4Addr}, + path::{Path, PathBuf}, + time::Duration, +}; + +use crate::settings::{HttpSettings, SslVerify}; + +#[derive(Clone, Debug, thiserror::Error, PartialEq, Eq)] +pub enum Error { + #[error("{setting} cannot be expressed with rustls: {reason}")] + Unsupported { + setting: &'static str, + reason: String, + }, + #[error("could not read {}: {message}", path.display())] + Read { path: PathBuf, message: String }, + #[error("{} is not a PEM file: {message}", path.display())] + InvalidPem { path: PathBuf, message: String }, + #[error("could not build the HTTP client: {0}")] + Client(String), +} + +impl From for Error { + fn from(error: reqwest::Error) -> Self { + Self::Client(error.without_url().to_string()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum Verify { + Disabled, + CaBundle(PathBuf), + BuiltInRoots, +} + +/// One fully resolved client configuration. Every field is a plain value so the pool can +/// key cached clients on it. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct HttpClientConfig { + pub verify: Verify, + pub client_certificate: Option, + pub force_ipv4: bool, + pub http2: bool, + pub user_agent: Option, + pub trust_proxy_env: bool, + pub connect_timeout: Duration, + pub request_timeout: Option, +} + +impl HttpClientConfig { + /// Port of `get_ssl_verify` + `get_ssl_configuration`: the per-call value wins, then the + /// configured (environment-overlaid) `ssl_verify`, then `SSL_CERT_FILE`, then the built-in + /// roots. Settings rustls has no equivalent for are an error instead of a silent no-op. + pub fn resolve( + settings: &HttpSettings, + per_call_ssl_verify: Option<&SslVerify>, + ) -> Result { + if let Some(level) = &settings.ssl_security_level { + return Err(Error::Unsupported { + setting: "ssl_security_level", + reason: format!("OpenSSL cipher string {level:?} has no rustls equivalent"), + }); + } + if let Some(curve) = &settings.ssl_ecdh_curve { + return Err(Error::Unsupported { + setting: "ssl_ecdh_curve", + reason: format!("key exchange group {curve:?} is fixed by the rustls provider"), + }); + } + let verify = match per_call_ssl_verify.or(settings.ssl_verify.as_ref()) { + Some(SslVerify::Disabled) => Verify::Disabled, + Some(SslVerify::CaBundle(path)) => Verify::CaBundle(path.clone()), + Some(SslVerify::Enabled) | None => settings + .ssl_cert_file + .clone() + .map_or(Verify::BuiltInRoots, Verify::CaBundle), + }; + Ok(Self { + verify, + client_certificate: settings.ssl_certificate.clone(), + force_ipv4: settings.force_ipv4, + http2: settings.http2, + user_agent: settings.user_agent.clone(), + trust_proxy_env: settings.trust_proxy_env, + connect_timeout: settings.connect_timeout, + request_timeout: settings.request_timeout, + }) + } + + /// A builder carrying every shared setting; variants add their own policy on top. + pub fn client_builder(&self) -> Result { + let base = reqwest::Client::builder().connect_timeout(self.connect_timeout); + let with_roots = match &self.verify { + Verify::Disabled => base.danger_accept_invalid_certs(true), + Verify::BuiltInRoots => base, + Verify::CaBundle(path) => { + let pem = read(path)?; + let certificates = + reqwest::Certificate::from_pem_bundle(&pem).map_err(|error| { + Error::InvalidPem { + path: path.clone(), + message: error.without_url().to_string(), + } + })?; + if certificates.is_empty() { + return Err(Error::InvalidPem { + path: path.clone(), + message: "no certificates found".into(), + }); + } + certificates.into_iter().fold( + base.tls_built_in_root_certs(false), + |builder, certificate| builder.add_root_certificate(certificate), + ) + } + }; + let with_identity = match &self.client_certificate { + None => with_roots, + Some(path) => { + let identity = reqwest::Identity::from_pem(&read(path)?).map_err(|error| { + Error::InvalidPem { + path: path.clone(), + message: error.without_url().to_string(), + } + })?; + with_roots.identity(identity) + } + }; + let with_address = if self.force_ipv4 { + with_identity.local_address(IpAddr::V4(Ipv4Addr::UNSPECIFIED)) + } else { + with_identity + }; + let with_protocol = if self.http2 { + with_address + } else { + with_address.http1_only() + }; + let with_agent = match &self.user_agent { + Some(agent) => with_protocol.user_agent(agent), + None => with_protocol, + }; + let with_proxy = if self.trust_proxy_env { + with_agent + } else { + with_agent.no_proxy() + }; + Ok(match self.request_timeout { + Some(timeout) => with_proxy.timeout(timeout), + None => with_proxy, + }) + } +} + +fn read(path: &Path) -> Result, Error> { + std::fs::read(path).map_err(|error| Error::Read { + path: path.to_path_buf(), + message: error.to_string(), + }) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn no_env(_: &str) -> Option { + None + } + + fn settings(ssl_verify: Option, ssl_cert_file: Option<&str>) -> HttpSettings { + HttpSettings { + ssl_verify, + ssl_cert_file: ssl_cert_file.map(PathBuf::from), + ..HttpSettings::default() + } + .with_environment(&no_env) + } + + #[rstest] + #[case::default(settings(None, None), None, Verify::BuiltInRoots)] + #[case::setting_disables( + settings(Some(SslVerify::Disabled), Some("/env/roots.pem")), + None, + Verify::Disabled + )] + #[case::setting_bundle( + settings(Some(SslVerify::CaBundle("/configured.pem".into())), Some("/env/roots.pem")), + None, + Verify::CaBundle("/configured.pem".into()) + )] + #[case::enabled_uses_cert_file( + settings(Some(SslVerify::Enabled), Some("/env/roots.pem")), + None, + Verify::CaBundle("/env/roots.pem".into()) + )] + #[case::unset_uses_cert_file(settings(None, Some("/env/roots.pem")), None, Verify::CaBundle("/env/roots.pem".into()))] + #[case::per_call_beats_setting( + settings(Some(SslVerify::Disabled), None), + Some(SslVerify::Enabled), + Verify::BuiltInRoots + )] + #[case::per_call_disables( + settings(Some(SslVerify::CaBundle("/configured.pem".into())), Some("/env/roots.pem")), + Some(SslVerify::Disabled), + Verify::Disabled + )] + #[case::per_call_bundle( + settings(None, Some("/env/roots.pem")), + Some(SslVerify::CaBundle("/call.pem".into())), + Verify::CaBundle("/call.pem".into()) + )] + #[case::per_call_enabled_still_honours_cert_file( + settings(Some(SslVerify::Disabled), Some("/env/roots.pem")), + Some(SslVerify::Enabled), + Verify::CaBundle("/env/roots.pem".into()) + )] + fn verify_follows_per_call_then_setting_then_cert_file( + #[case] settings: HttpSettings, + #[case] per_call: Option, + #[case] expected: Verify, + ) { + let config = HttpClientConfig::resolve(&settings, per_call.as_ref()).unwrap(); + assert_eq!(config.verify, expected); + } + + #[test] + fn ssl_verify_environment_variable_beats_the_configured_setting() { + let settings = HttpSettings { + ssl_verify: Some(SslVerify::Disabled), + ..HttpSettings::default() + } + .with_environment(&|name: &str| (name == "SSL_VERIFY").then(|| "true".to_string())); + let config = HttpClientConfig::resolve(&settings, None).unwrap(); + assert_eq!(config.verify, Verify::BuiltInRoots); + } + + #[test] + fn cipher_strings_are_rejected_rather_than_ignored() { + let settings = HttpSettings { + ssl_security_level: Some("DEFAULT@SECLEVEL=1".into()), + ..HttpSettings::default() + }; + assert!(matches!( + HttpClientConfig::resolve(&settings, None), + Err(Error::Unsupported { + setting: "ssl_security_level", + .. + }) + )); + } + + #[test] + fn ecdh_curves_are_rejected_rather_than_ignored() { + let settings = HttpSettings { + ssl_ecdh_curve: Some("X25519".into()), + ..HttpSettings::default() + }; + assert!(matches!( + HttpClientConfig::resolve(&settings, None), + Err(Error::Unsupported { + setting: "ssl_ecdh_curve", + .. + }) + )); + } + + #[test] + fn connection_settings_carry_over_unchanged() { + let settings = HttpSettings { + ssl_certificate: Some("/client.pem".into()), + force_ipv4: true, + http2: true, + user_agent: Some("litellm/1.0".into()), + trust_proxy_env: true, + connect_timeout: Duration::from_secs(7), + request_timeout: Some(Duration::from_secs(70)), + ..HttpSettings::default() + }; + let config = HttpClientConfig::resolve(&settings, None).unwrap(); + assert_eq!( + config, + HttpClientConfig { + verify: Verify::BuiltInRoots, + client_certificate: Some("/client.pem".into()), + force_ipv4: true, + http2: true, + user_agent: Some("litellm/1.0".into()), + trust_proxy_env: true, + connect_timeout: Duration::from_secs(7), + request_timeout: Some(Duration::from_secs(70)), + } + ); + } + + #[test] + fn missing_ca_bundle_is_a_read_error() { + let path = std::env::temp_dir().join("litellm-http-missing-bundle.pem"); + let config = HttpClientConfig { + verify: Verify::CaBundle(path.clone()), + ..HttpClientConfig::resolve(&HttpSettings::default(), None).unwrap() + }; + assert!(matches!( + config.client_builder(), + Err(Error::Read { path: reported, .. }) if reported == path + )); + } + + #[test] + fn non_pem_ca_bundle_is_an_invalid_pem_error() { + let path = + std::env::temp_dir().join(format!("litellm-http-not-pem-{}.pem", std::process::id())); + std::fs::write(&path, b"not a certificate").unwrap(); + let config = HttpClientConfig { + verify: Verify::CaBundle(path.clone()), + ..HttpClientConfig::resolve(&HttpSettings::default(), None).unwrap() + }; + let result = config.client_builder().map(drop); + std::fs::remove_file(&path).unwrap(); + assert!(matches!( + result, + Err(Error::InvalidPem { path: reported, .. }) if reported == path + )); + } +} diff --git a/litellm-rust/crates/http/src/lib.rs b/litellm-rust/crates/http/src/lib.rs new file mode 100644 index 00000000000..d62dd768fe1 --- /dev/null +++ b/litellm-rust/crates/http/src/lib.rs @@ -0,0 +1,11 @@ +//! Rust counterpart of `litellm/llms/custom_httpx/http_handler.py`: the plain HTTP settings +//! LiteLLM exposes, their resolution into one typed client configuration, and a pool that +//! caches `reqwest::Client`s per resolved configuration. + +mod config; +mod pool; +mod settings; + +pub use config::{Error, HttpClientConfig, Verify}; +pub use pool::{ClientVariant, HttpClientPool}; +pub use settings::{HttpSettings, SslVerify}; diff --git a/litellm-rust/crates/http/src/pool.rs b/litellm-rust/crates/http/src/pool.rs new file mode 100644 index 00000000000..065097556e1 --- /dev/null +++ b/litellm-rust/crates/http/src/pool.rs @@ -0,0 +1,172 @@ +use std::{ + collections::HashMap, + sync::{Mutex, PoisonError}, +}; + +use crate::config::{Error, HttpClientConfig}; + +/// The client shapes routes need; each is the shared base plus one policy. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum ClientVariant { + Provider, + NoRedirect, + /// Media downloads: no redirects (the fetcher validates each hop) and never a proxy. + Media, +} + +impl ClientVariant { + fn apply(self, builder: reqwest::ClientBuilder) -> reqwest::ClientBuilder { + match self { + Self::Provider => builder, + Self::NoRedirect => builder.redirect(reqwest::redirect::Policy::none()), + Self::Media => builder + .redirect(reqwest::redirect::Policy::none()) + .no_proxy(), + } + } +} + +/// Counterpart of `get_async_httpx_client`: one `reqwest::Client` per resolved configuration +/// and variant, built on first use and shared afterwards. +#[derive(Default)] +pub struct HttpClientPool { + clients: Mutex>, +} + +impl HttpClientPool { + pub fn new() -> Self { + Self::default() + } + + pub fn client( + &self, + config: &HttpClientConfig, + variant: ClientVariant, + ) -> Result { + self.client_with(config, variant, |builder| builder) + } + + /// Like [`Self::client`], with a caller hook for builder options that are not plain values + /// (a DNS resolver, for example). The hook only runs when the client is first built. + pub fn client_with( + &self, + config: &HttpClientConfig, + variant: ClientVariant, + customize: impl FnOnce(reqwest::ClientBuilder) -> reqwest::ClientBuilder, + ) -> Result { + let key = (config.clone(), variant); + let mut clients = self.clients.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(client) = clients.get(&key) { + return Ok(client.clone()); + } + let client = customize(variant.apply(config.client_builder()?)).build()?; + clients.insert(key, client.clone()); + Ok(client) + } +} + +#[cfg(test)] +mod tests { + use std::{cell::Cell, time::Duration}; + + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + use super::*; + use crate::{HttpSettings, Verify}; + + fn config(user_agent: &str) -> HttpClientConfig { + HttpClientConfig { + user_agent: Some(user_agent.into()), + ..HttpClientConfig::resolve(&HttpSettings::default(), None).unwrap() + } + } + + #[test] + fn clients_are_built_once_per_config_and_variant() { + let pool = HttpClientPool::new(); + let builds = Cell::new(0); + let build = |config: &HttpClientConfig, variant| { + pool.client_with(config, variant, |builder| { + builds.set(builds.get() + 1); + builder + }) + .unwrap() + }; + build(&config("a"), ClientVariant::Provider); + build(&config("a"), ClientVariant::Provider); + assert_eq!(builds.get(), 1); + build(&config("a"), ClientVariant::NoRedirect); + assert_eq!(builds.get(), 2); + build(&config("b"), ClientVariant::Provider); + assert_eq!(builds.get(), 3); + build(&config("b"), ClientVariant::Provider); + build(&config("a"), ClientVariant::NoRedirect); + assert_eq!(builds.get(), 3); + } + + #[test] + fn build_failures_are_not_cached() { + let pool = HttpClientPool::new(); + let missing = HttpClientConfig { + verify: Verify::CaBundle(std::env::temp_dir().join("litellm-http-absent.pem")), + ..config("a") + }; + assert!(pool.client(&missing, ClientVariant::Provider).is_err()); + assert!(pool.client(&missing, ClientVariant::Provider).is_err()); + assert!(pool.client(&config("a"), ClientVariant::Provider).is_ok()); + } + + async fn serve_once(status_line: &'static str) -> (String, tokio::task::JoinHandle) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = vec![0u8; 4096]; + let read = socket.read(&mut request).await.unwrap(); + socket + .write_all( + format!("{status_line}\r\nLocation: /elsewhere\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .as_bytes(), + ) + .await + .unwrap(); + String::from_utf8_lossy(&request[..read]).into_owned() + }); + (base, server) + } + + #[tokio::test] + async fn provider_client_sends_the_configured_user_agent_over_http1() { + let (base, server) = serve_once("HTTP/1.1 204 No Content").await; + let config = HttpClientConfig { + connect_timeout: Duration::from_secs(2), + ..config("litellm-test/9") + }; + let response = HttpClientPool::new() + .client(&config, ClientVariant::Provider) + .unwrap() + .get(&base) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 204); + assert_eq!(response.version(), reqwest::Version::HTTP_11); + let request = server.await.unwrap(); + assert!(request.contains("user-agent: litellm-test/9"), "{request}"); + } + + #[tokio::test] + async fn no_redirect_variant_returns_the_redirect_instead_of_following_it() { + let (base, server) = serve_once("HTTP/1.1 302 Found").await; + let response = HttpClientPool::new() + .client(&config("a"), ClientVariant::NoRedirect) + .unwrap() + .get(&base) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 302); + assert_eq!(response.headers()["location"], "/elsewhere"); + server.await.unwrap(); + } +} diff --git a/litellm-rust/crates/http/src/settings.rs b/litellm-rust/crates/http/src/settings.rs new file mode 100644 index 00000000000..4d936e89046 --- /dev/null +++ b/litellm-rust/crates/http/src/settings.rs @@ -0,0 +1,171 @@ +use std::{path::PathBuf, time::Duration}; + +/// `litellm.ssl_verify` / `SSL_VERIFY`: a bool or a CA bundle path. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum SslVerify { + Enabled, + Disabled, + CaBundle(PathBuf), +} + +impl SslVerify { + pub fn parse(value: &str) -> Self { + match value.trim().to_ascii_lowercase().as_str() { + "true" => Self::Enabled, + "false" => Self::Disabled, + _ => Self::CaBundle(PathBuf::from(value)), + } + } +} + +/// The plain inputs `http_handler.py` reads from `litellm.*` globals and the environment. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct HttpSettings { + pub ssl_verify: Option, + pub ssl_cert_file: Option, + pub ssl_certificate: Option, + pub ssl_security_level: Option, + pub ssl_ecdh_curve: Option, + pub force_ipv4: bool, + pub http2: bool, + pub user_agent: Option, + pub trust_proxy_env: bool, + pub connect_timeout: Duration, + pub request_timeout: Option, +} + +impl Default for HttpSettings { + fn default() -> Self { + Self { + ssl_verify: None, + ssl_cert_file: None, + ssl_certificate: None, + ssl_security_level: None, + ssl_ecdh_curve: None, + force_ipv4: false, + http2: false, + user_agent: None, + trust_proxy_env: false, + connect_timeout: Duration::from_secs(5), + request_timeout: None, + } + } +} + +impl HttpSettings { + /// Overlay the environment variables `http_handler.py` consults, with the same precedence: + /// `SSL_VERIFY`, `SSL_CERTIFICATE`, `SSL_SECURITY_LEVEL`, `SSL_ECDH_CURVE` and + /// `LITELLM_USER_AGENT` win over the configured value; `SSL_CERT_FILE` only applies when + /// verification is on without an explicit bundle; `LITELLM_HTTP2` and `AIOHTTP_TRUST_ENV` + /// can only turn their switch on. + pub fn with_environment(self, env: &(dyn Fn(&str) -> Option + Sync)) -> Self { + let enabled = + |name: &str| env(name).is_some_and(|value| value.trim().eq_ignore_ascii_case("true")); + Self { + ssl_verify: env("SSL_VERIFY") + .map(|value| SslVerify::parse(&value)) + .or(self.ssl_verify), + ssl_cert_file: env("SSL_CERT_FILE") + .map(PathBuf::from) + .or(self.ssl_cert_file), + ssl_certificate: env("SSL_CERTIFICATE") + .map(PathBuf::from) + .or(self.ssl_certificate), + ssl_security_level: env("SSL_SECURITY_LEVEL").or(self.ssl_security_level), + ssl_ecdh_curve: env("SSL_ECDH_CURVE").or(self.ssl_ecdh_curve), + http2: self.http2 || enabled("LITELLM_HTTP2"), + user_agent: env("LITELLM_USER_AGENT").or(self.user_agent), + trust_proxy_env: self.trust_proxy_env || enabled("AIOHTTP_TRUST_ENV"), + ..self + } + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn no_env(_: &str) -> Option { + None + } + + fn env_of( + values: &'static [(&'static str, &'static str)], + ) -> impl Fn(&str) -> Option + Sync { + move |name| { + values + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.to_string()) + } + } + + #[rstest] + #[case("true", SslVerify::Enabled)] + #[case(" True ", SslVerify::Enabled)] + #[case("FALSE", SslVerify::Disabled)] + #[case("/etc/ssl/bundle.pem", SslVerify::CaBundle("/etc/ssl/bundle.pem".into()))] + fn ssl_verify_parses_bools_and_treats_anything_else_as_a_bundle_path( + #[case] value: &str, + #[case] expected: SslVerify, + ) { + assert_eq!(SslVerify::parse(value), expected); + } + + #[test] + fn environment_overrides_configured_ssl_values() { + let settings = HttpSettings { + ssl_verify: Some(SslVerify::Enabled), + ssl_certificate: Some("/configured/client.pem".into()), + ssl_security_level: Some("configured".into()), + user_agent: Some("configured/1".into()), + ..HttpSettings::default() + } + .with_environment(&env_of(&[ + ("SSL_VERIFY", "false"), + ("SSL_CERT_FILE", "/env/roots.pem"), + ("SSL_CERTIFICATE", "/env/client.pem"), + ("SSL_SECURITY_LEVEL", "DEFAULT@SECLEVEL=1"), + ("SSL_ECDH_CURVE", "X25519"), + ("LITELLM_USER_AGENT", "env/2"), + ])); + assert_eq!(settings.ssl_verify, Some(SslVerify::Disabled)); + assert_eq!(settings.ssl_cert_file, Some("/env/roots.pem".into())); + assert_eq!(settings.ssl_certificate, Some("/env/client.pem".into())); + assert_eq!( + settings.ssl_security_level.as_deref(), + Some("DEFAULT@SECLEVEL=1") + ); + assert_eq!(settings.ssl_ecdh_curve.as_deref(), Some("X25519")); + assert_eq!(settings.user_agent.as_deref(), Some("env/2")); + } + + #[test] + fn missing_environment_keeps_configured_values() { + let configured = HttpSettings { + ssl_verify: Some(SslVerify::CaBundle("/configured/roots.pem".into())), + http2: true, + trust_proxy_env: true, + user_agent: Some("configured/1".into()), + ..HttpSettings::default() + }; + assert_eq!(configured.clone().with_environment(&no_env), configured); + } + + #[rstest] + #[case("true", true)] + #[case("True", true)] + #[case("false", false)] + #[case("1", false)] + fn boolean_switches_only_turn_on_for_true(#[case] value: &'static str, #[case] expected: bool) { + let env = move |name: &str| match name { + "LITELLM_HTTP2" | "AIOHTTP_TRUST_ENV" => Some(value.to_string()), + _ => None, + }; + let settings = HttpSettings::default().with_environment(&env); + assert_eq!(settings.http2, expected); + assert_eq!(settings.trust_proxy_env, expected); + } +} diff --git a/litellm-rust/crates/llms/Cargo.toml b/litellm-rust/crates/llms/Cargo.toml index 4ca6c7cb2a5..ca76aaf380e 100644 --- a/litellm-rust/crates/llms/Cargo.toml +++ b/litellm-rust/crates/llms/Cargo.toml @@ -17,6 +17,7 @@ litellm-auth-azure.workspace = true litellm-auth-gcp.workspace = true litellm-callbacks.workspace = true litellm-framing.workspace = true +litellm-http.workspace = true base64.workspace = true bytes.workspace = true data-url = "0.3.2" diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs b/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs index e6fe5d9556d..f20ec726f61 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs @@ -21,7 +21,6 @@ use crate::{ pub const OCR_RESPONSE_MAX_BYTES: usize = 64 * 1024 * 1024; pub const OCR_HTTP_TIMEOUT_SECS: u64 = 600; -pub const OCR_CONNECT_TIMEOUT_SECS: u64 = 10; pub const OCR_INLINE_MAX_BYTES: usize = 50 * 1024 * 1024; pub const OCR_DOWNLOAD_MAX_BYTES: u64 = 50 * 1024 * 1024; pub const OCR_MAX_FETCH_REDIRECTS: usize = 10; diff --git a/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs b/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs index e93ddee3c50..38ddec08da9 100644 --- a/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs +++ b/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs @@ -1,9 +1,8 @@ -use std::{sync::OnceLock, time::Duration}; - use bytes::{Bytes, BytesMut}; use futures_util::future::BoxFuture; use litellm_auth_gcp::VertexAuth; use litellm_callbacks::event::{Passthrough, WireRequest}; +use litellm_http::{ClientVariant, HttpClientConfig, HttpClientPool}; use serde::{Serialize, de::DeserializeOwned}; use serde_json::{Map, Value}; @@ -11,9 +10,8 @@ use crate::{ base_llm::ocr::{ error::Error, transformation::{ - BaseOcrConfig, DecodedOcrResponse, LiteLLMOcrResponse, OCR_CONNECT_TIMEOUT_SECS, - OcrDocument, OcrResponseContext, PreparedOcrRequest, decode_request_value, - decode_response, + BaseOcrConfig, DecodedOcrResponse, LiteLLMOcrResponse, OcrDocument, OcrResponseContext, + PreparedOcrRequest, decode_request_value, decode_response, }, }, custom_httpx::{ @@ -44,30 +42,15 @@ pub struct OcrClient { } impl OcrClient { - pub fn new(provider_http: reqwest::Client) -> Result { - let document_fetcher = MediaFetcher::new().map_err(transport::Error::from)?; + pub fn new(pool: &HttpClientPool, config: &HttpClientConfig) -> Result { Ok(Self { - provider_http, - polling_http: no_redirect_http()?, - document_fetcher, + provider_http: pool.client(config, ClientVariant::Provider)?, + polling_http: pool.client(config, ClientVariant::NoRedirect)?, + document_fetcher: MediaFetcher::new(pool, config)?, vertex_auth: VertexAuth::default(), }) } - pub fn shared() -> Result { - static CLIENT: OnceLock> = OnceLock::new(); - let client = CLIENT - .get_or_init(|| { - reqwest::Client::builder() - .connect_timeout(Duration::from_secs(OCR_CONNECT_TIMEOUT_SECS)) - .build() - .map_err(transport::Error::from) - .and_then(OcrClient::new) - }) - .clone()?; - Ok(client) - } - pub fn provider_http(&self) -> &reqwest::Client { &self.provider_http } @@ -88,21 +71,16 @@ impl OcrClient { pub fn for_test(provider_http: reqwest::Client, document_http: reqwest::Client) -> Self { Self { provider_http, - polling_http: no_redirect_http().expect("test polling client builds"), + polling_http: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("test polling client builds"), document_fetcher: MediaFetcher::for_test(document_http), vertex_auth: VertexAuth::default(), } } } -fn no_redirect_http() -> Result { - reqwest::Client::builder() - .connect_timeout(Duration::from_secs(OCR_CONNECT_TIMEOUT_SECS)) - .redirect(reqwest::redirect::Policy::none()) - .build() - .map_err(transport::Error::from) -} - /// Rust counterpart of `BaseLLMHTTPHandler.async_ocr`: prepare the provider request, /// send it, and hand the response to the config for normalization. pub async fn ocr( @@ -318,6 +296,8 @@ pub fn body_document(body: &Value) -> Result { #[cfg(test)] mod tests { + use std::time::Duration; + use super::*; #[tokio::test] diff --git a/litellm-rust/crates/llms/src/custom_httpx/media.rs b/litellm-rust/crates/llms/src/custom_httpx/media.rs index 0b7fa30e34b..aeac4894683 100644 --- a/litellm-rust/crates/llms/src/custom_httpx/media.rs +++ b/litellm-rust/crates/llms/src/custom_httpx/media.rs @@ -7,13 +7,12 @@ use std::{ time::Duration, }; +use litellm_http::{ClientVariant, HttpClientConfig, HttpClientPool}; use reqwest::{ Url, dns::{Addrs, Name, Resolve, Resolving}, }; -const MEDIA_CONNECT_TIMEOUT_SECS: u64 = 10; - #[derive(Debug, thiserror::Error)] pub enum Error { #[error("media URL rejected by network policy")] @@ -63,23 +62,30 @@ pub struct DownloadedMedia { } impl MediaFetcher { - pub fn new() -> Result { - Self::with_resolvers(Arc::new(PublicDnsResolver), Arc::new(SystemAddressResolver)) + pub fn new( + pool: &HttpClientPool, + config: &HttpClientConfig, + ) -> Result { + Self::with_resolvers( + pool, + config, + Arc::new(PublicDnsResolver), + Arc::new(SystemAddressResolver), + ) } fn with_resolvers( + pool: &HttpClientPool, + config: &HttpClientConfig, transport_resolver: Arc, address_resolver: Arc, - ) -> Result + ) -> Result where R: Resolve + 'static, { - let client = reqwest::Client::builder() - .connect_timeout(Duration::from_secs(MEDIA_CONNECT_TIMEOUT_SECS)) - .redirect(reqwest::redirect::Policy::none()) - .no_proxy() - .dns_resolver(transport_resolver) - .build()?; + let client = pool.client_with(config, ClientVariant::Media, |builder| { + builder.dns_resolver(transport_resolver) + })?; Ok(Self { client, address_resolver, @@ -281,6 +287,7 @@ impl Resolve for PublicDnsResolver { mod tests { use std::collections::HashSet; + use litellm_http::HttpSettings; use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, net::TcpListener, @@ -365,6 +372,8 @@ mod tests { blocked_hosts: HashSet<&'static str>, ) -> MediaFetcher { MediaFetcher::with_resolvers( + &HttpClientPool::new(), + &HttpClientConfig::resolve(&HttpSettings::default(), None).unwrap(), Arc::new(LoopbackDnsResolver(address)), Arc::new(TestAddressResolver { blocked_hosts }), ) @@ -542,7 +551,12 @@ mod tests { #[tokio::test] async fn rejects_url_credentials_before_network_access() { - let fetcher = MediaFetcher::new().expect("media fetcher builds"); + let fetcher = MediaFetcher::new( + &HttpClientPool::new(), + &HttpClientConfig::resolve(&litellm_http::HttpSettings::default(), None) + .expect("default settings resolve"), + ) + .expect("media fetcher builds"); let url = Url::parse("https://user:password@8.8.8.8/document").expect("credentialed URL parses"); assert!(matches!( diff --git a/litellm-rust/crates/llms/src/custom_httpx/transport.rs b/litellm-rust/crates/llms/src/custom_httpx/transport.rs index 172dd96476a..8e5e1a8832d 100644 --- a/litellm-rust/crates/llms/src/custom_httpx/transport.rs +++ b/litellm-rust/crates/llms/src/custom_httpx/transport.rs @@ -26,6 +26,12 @@ impl From for Error { } } +impl From for Error { + fn from(error: litellm_http::Error) -> Self { + Self::Connect(error.to_string()) + } +} + #[cfg(test)] mod tests { #[tokio::test] diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index e9b7f384406..762e22e433b 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -20,6 +20,7 @@ bytes.workspace = true litellm-auth.workspace = true litellm-callbacks-legacy.workspace = true litellm-core.workspace = true +litellm-http.workspace = true litellm-llms.workspace = true litellm-types.workspace = true litellm-host-python.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/http.rs b/litellm-rust/crates/python-bridge/src/http.rs new file mode 100644 index 00000000000..d2e05c3b949 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/http.rs @@ -0,0 +1,234 @@ +use std::{path::PathBuf, sync::LazyLock}; + +use litellm_http::{HttpClientConfig, HttpClientPool, HttpSettings, SslVerify}; +use pyo3::{prelude::*, types::PyDict}; + +use crate::errors::RustBridgeDeclined; + +static POOL: LazyLock = LazyLock::new(HttpClientPool::new); + +/// Keyword arguments that carry a live Python HTTP client or session. They cannot cross into +/// Rust, so a call that supplies one stays on the Python path. +const LIVE_CLIENT_ARGUMENTS: [&str; 3] = ["client", "shared_session", "aclient_session"]; + +pub(crate) fn pool() -> &'static HttpClientPool { + &POOL +} + +/// The client configuration for one call: the process settings from the `litellm` module and +/// the environment, narrowed by the call's own `ssl_verify`. +pub(crate) fn call_config( + py: Python<'_>, + kwargs: &Bound<'_, PyDict>, +) -> PyResult { + decline_live_clients(kwargs)?; + let settings = settings(py.import("litellm")?.as_any())? + .with_environment(&|name| std::env::var(name).ok()); + let per_call = kwargs + .get_item("ssl_verify")? + .map(|value| ssl_verify(&value, "ssl_verify")) + .transpose()? + .flatten(); + HttpClientConfig::resolve(&settings, per_call.as_ref()) + .map_err(|error| RustBridgeDeclined::new_err(error.to_string())) +} + +pub(crate) fn decline_live_clients(kwargs: &Bound<'_, PyDict>) -> PyResult<()> { + for name in LIVE_CLIENT_ARGUMENTS { + if kwargs.get_item(name)?.is_some_and(|value| !value.is_none()) { + return Err(RustBridgeDeclined::new_err(format!( + "{name} is a live Python HTTP client and cannot be used by the Rust route" + ))); + } + } + Ok(()) +} + +/// Read the `litellm.*` globals `http_handler.py` consults. `globals` is the `litellm` module in +/// production and any attribute holder in tests. +pub(crate) fn settings(globals: &Bound<'_, PyAny>) -> PyResult { + Ok(HttpSettings { + ssl_verify: ssl_verify(&globals.getattr("ssl_verify")?, "litellm.ssl_verify")?, + ssl_certificate: optional_path(globals, "ssl_certificate")?, + ssl_security_level: globals.getattr("ssl_security_level")?.extract()?, + ssl_ecdh_curve: globals.getattr("ssl_ecdh_curve")?.extract()?, + force_ipv4: globals.getattr("force_ipv4")?.extract()?, + http2: globals.getattr("http2")?.extract()?, + trust_proxy_env: globals.getattr("aiohttp_trust_env")?.extract()?, + ..HttpSettings::default() + }) +} + +fn optional_path(globals: &Bound<'_, PyAny>, name: &str) -> PyResult> { + Ok(globals + .getattr(name)? + .extract::>()? + .map(PathBuf::from)) +} + +fn ssl_verify(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + if value.is_none() { + return Ok(None); + } + if let Ok(enabled) = value.extract::() { + return Ok(Some(if enabled { + SslVerify::Enabled + } else { + SslVerify::Disabled + })); + } + if let Ok(path) = value.extract::() { + return Ok(Some(SslVerify::CaBundle(PathBuf::from(path)))); + } + Err(RustBridgeDeclined::new_err(format!( + "{name} is a live Python object and cannot be used by the Rust route" + ))) +} + +#[cfg(test)] +mod tests { + use litellm_http::Verify; + use rstest::rstest; + + use super::*; + + fn eval<'py>(py: Python<'py>, source: &std::ffi::CStr) -> Bound<'py, PyDict> { + let locals = PyDict::new(py); + py.run(source, Some(&locals), Some(&locals)).unwrap(); + locals + } + + fn globals<'py>(py: Python<'py>, overrides: &str) -> Bound<'py, PyAny> { + let source = format!( + " +import types +globals = types.SimpleNamespace( + ssl_verify=True, + ssl_certificate=None, + ssl_security_level=None, + ssl_ecdh_curve=None, + force_ipv4=False, + http2=False, + aiohttp_trust_env=False, +) +{overrides} +" + ); + let source = std::ffi::CString::new(source).unwrap(); + eval(py, &source).get_item("globals").unwrap().unwrap() + } + + #[test] + fn default_globals_produce_default_settings_with_verification_on() { + Python::initialize(); + Python::attach(|py| { + let settings = settings(&globals(py, "")).unwrap(); + assert_eq!( + settings, + HttpSettings { + ssl_verify: Some(SslVerify::Enabled), + ..HttpSettings::default() + } + ); + }); + } + + #[test] + fn globals_flow_into_settings() { + Python::initialize(); + Python::attach(|py| { + let settings = settings(&globals( + py, + " +globals.ssl_verify = '/etc/ssl/corp.pem' +globals.ssl_certificate = '/etc/ssl/client.pem' +globals.ssl_security_level = '2' +globals.ssl_ecdh_curve = 'X25519' +globals.force_ipv4 = True +globals.http2 = True +globals.aiohttp_trust_env = True +", + )) + .unwrap(); + assert_eq!( + settings, + HttpSettings { + ssl_verify: Some(SslVerify::CaBundle("/etc/ssl/corp.pem".into())), + ssl_certificate: Some("/etc/ssl/client.pem".into()), + ssl_security_level: Some("2".into()), + ssl_ecdh_curve: Some("X25519".into()), + force_ipv4: true, + http2: true, + trust_proxy_env: true, + ..HttpSettings::default() + } + ); + }); + } + + #[test] + fn disabled_verification_global_resolves_to_disabled() { + Python::initialize(); + Python::attach(|py| { + let settings = settings(&globals(py, "globals.ssl_verify = False")).unwrap(); + assert_eq!(settings.ssl_verify, Some(SslVerify::Disabled)); + let config = HttpClientConfig::resolve(&settings, None).unwrap(); + assert_eq!(config.verify, Verify::Disabled); + }); + } + + #[test] + fn ssl_context_global_declines_instead_of_being_dropped() { + Python::initialize(); + Python::attach(|py| { + let error = settings(&globals(py, "globals.ssl_verify = object()")).unwrap_err(); + assert!(error.is_instance_of::(py)); + assert!(error.value(py).to_string().contains("litellm.ssl_verify")); + }); + } + + #[rstest] + #[case::client("client")] + #[case::shared_session("shared_session")] + #[case::aclient_session("aclient_session")] + fn live_python_clients_decline_before_dispatch(#[case] name: &str) { + Python::initialize(); + Python::attach(|py| { + let kwargs = PyDict::new(py); + kwargs + .set_item(name, py.eval(c"object()", None, None).unwrap()) + .unwrap(); + let error = decline_live_clients(&kwargs).unwrap_err(); + assert!(error.is_instance_of::(py)); + assert!(error.value(py).to_string().contains(name)); + }); + } + + #[test] + fn none_valued_client_arguments_are_not_live_clients() { + Python::initialize(); + Python::attach(|py| { + let kwargs = PyDict::new(py); + for name in LIVE_CLIENT_ARGUMENTS { + kwargs.set_item(name, py.None()).unwrap(); + } + decline_live_clients(&kwargs).unwrap(); + }); + } + + #[rstest] + #[case::disabled(c"False", Some(SslVerify::Disabled))] + #[case::enabled(c"True", Some(SslVerify::Enabled))] + #[case::bundle(c"'/tmp/ca.pem'", Some(SslVerify::CaBundle("/tmp/ca.pem".into())))] + #[case::unset(c"None", None)] + fn per_call_ssl_verify_values_project( + #[case] source: &std::ffi::CStr, + #[case] expected: Option, + ) { + Python::initialize(); + Python::attach(|py| { + let value = py.eval(source, None, None).unwrap(); + assert_eq!(ssl_verify(&value, "ssl_verify").unwrap(), expected); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index ca699e7c483..11cb0a7f655 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -1,6 +1,7 @@ mod credentials; mod diagnostics; mod errors; +mod http; mod marshal; mod routes; mod token_counter; diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs index b5bb941708d..f252e1dc47b 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs @@ -12,6 +12,8 @@ use pyo3::{ types::{PyDict, PyTuple}, }; +use crate::{errors::RustBridgeDeclined, http}; + const SURFACE: LegacySurface = LegacySurface { call_type: "ocr", input_description: "OCR document processing", @@ -29,7 +31,9 @@ fn run_ocr( kwargs: Bound<'_, PyDict>, asynchronous: bool, ) -> PyResult> { - let client = OcrClient::shared().map_err(errors::to_pyerr)?; + let config = http::call_config(py, &kwargs)?; + let client = OcrClient::new(http::pool(), &config) + .map_err(|error| RustBridgeDeclined::new_err(error.to_string()))?; run_legacy_call( py, if asynchronous { ASYNC_SURFACE } else { SURFACE }, From 5a474fd7996e96f90e18c539108381d811cd5e63 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Fri, 18 Sep 2026 23:31:27 +0000 Subject: [PATCH 02/15] refactor(rust): inject VertexAuth into OcrClient so the bridge keeps one token cache Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/Cargo.lock | 2 ++ litellm-rust/crates/core/Cargo.toml | 1 + litellm-rust/crates/core/src/ocr/client.rs | 4 +++- litellm-rust/crates/core/tests/ocr.rs | 3 +++ .../crates/llms/src/custom_httpx/llm_http_handler.rs | 8 ++++++-- litellm-rust/crates/python-bridge/Cargo.toml | 1 + litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs | 7 ++++++- 7 files changed, 22 insertions(+), 4 deletions(-) diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index ffbdc80efd3..ed0fee3411f 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -2056,6 +2056,7 @@ dependencies = [ "futures-util", "litellm-auth", "litellm-auth-aws", + "litellm-auth-gcp", "litellm-callbacks", "litellm-core-utils", "litellm-http", @@ -2175,6 +2176,7 @@ dependencies = [ "criterion", "futures-util", "litellm-auth", + "litellm-auth-gcp", "litellm-callbacks-legacy", "litellm-core", "litellm-host-python", diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index 047188c74f9..ce8463affc1 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -15,6 +15,7 @@ futures-util.workspace = true base64.workspace = true litellm-auth.workspace = true litellm-auth-aws.workspace = true +litellm-auth-gcp.workspace = true litellm-http.workspace = true litellm-llms.workspace = true moka.workspace = true diff --git a/litellm-rust/crates/core/src/ocr/client.rs b/litellm-rust/crates/core/src/ocr/client.rs index 21c81505cff..a380037487e 100644 --- a/litellm-rust/crates/core/src/ocr/client.rs +++ b/litellm-rust/crates/core/src/ocr/client.rs @@ -1,3 +1,4 @@ +use litellm_auth_gcp::VertexAuth; use litellm_http::{HttpClientConfig, HttpClientPool}; use litellm_llms::{ base_llm::ocr::{error::Error, transformation::LiteLLMOcrResponse}, @@ -19,7 +20,8 @@ pub async fn perform( pub async fn ocr( pool: &HttpClientPool, config: &HttpClientConfig, + vertex_auth: VertexAuth, request: LiteLLMOcrRequest, ) -> Result { - perform(&OcrClient::new(pool, config)?, request).await + perform(&OcrClient::new(pool, config, vertex_auth)?, request).await } diff --git a/litellm-rust/crates/core/tests/ocr.rs b/litellm-rust/crates/core/tests/ocr.rs index d59c6179aa5..4ce6d4ee8f6 100644 --- a/litellm-rust/crates/core/tests/ocr.rs +++ b/litellm-rust/crates/core/tests/ocr.rs @@ -1,5 +1,6 @@ use std::sync::{Arc, Mutex}; +use litellm_auth_gcp::VertexAuth; use litellm_callbacks::{ event::{CallEvent, WireRequest}, host::{Host, HostOp, HostResult}, @@ -182,6 +183,7 @@ async fn facade_uses_the_injected_http_pool_configuration() { crate::ocr::client::ocr( &HttpClientPool::new(), &config, + VertexAuth::default(), wire_request("mistral/model", &base, json!({})), ) .await @@ -200,6 +202,7 @@ async fn unbuildable_http_configuration_fails_before_dispatch() { let error = crate::ocr::client::ocr( &HttpClientPool::new(), &config, + VertexAuth::default(), wire_request("mistral/model", &base, json!({})), ) .await diff --git a/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs b/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs index 38ddec08da9..9826d73a061 100644 --- a/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs +++ b/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs @@ -42,12 +42,16 @@ pub struct OcrClient { } impl OcrClient { - pub fn new(pool: &HttpClientPool, config: &HttpClientConfig) -> Result { + pub fn new( + pool: &HttpClientPool, + config: &HttpClientConfig, + vertex_auth: VertexAuth, + ) -> Result { Ok(Self { provider_http: pool.client(config, ClientVariant::Provider)?, polling_http: pool.client(config, ClientVariant::NoRedirect)?, document_fetcher: MediaFetcher::new(pool, config)?, - vertex_auth: VertexAuth::default(), + vertex_auth, }) } diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 762e22e433b..c66701548d1 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -20,6 +20,7 @@ bytes.workspace = true litellm-auth.workspace = true litellm-callbacks-legacy.workspace = true litellm-core.workspace = true +litellm-auth-gcp.workspace = true litellm-http.workspace = true litellm-llms.workspace = true litellm-types.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs index f252e1dc47b..be188434275 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs @@ -3,7 +3,10 @@ mod errors; mod host; mod project; +use std::sync::LazyLock; + use host::OcrRouteHost; +use litellm_auth_gcp::VertexAuth; use litellm_callbacks_legacy::{LegacySurface, PublicCall, run_legacy_call}; use litellm_core::ocr::route::ocr_machine; use litellm_llms::custom_httpx::llm_http_handler::OcrClient; @@ -24,6 +27,8 @@ const ASYNC_SURFACE: LegacySurface = LegacySurface { ..SURFACE }; +static VERTEX_AUTH: LazyLock = LazyLock::new(VertexAuth::default); + fn run_ocr( py: Python<'_>, request: Bound<'_, PyAny>, @@ -32,7 +37,7 @@ fn run_ocr( asynchronous: bool, ) -> PyResult> { let config = http::call_config(py, &kwargs)?; - let client = OcrClient::new(http::pool(), &config) + let client = OcrClient::new(http::pool(), &config, VERTEX_AUTH.clone()) .map_err(|error| RustBridgeDeclined::new_err(error.to_string()))?; run_legacy_call( py, From b2d6cd1fcfde4bffb48473ff14b84fa221733864 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Fri, 18 Sep 2026 17:23:26 -0700 Subject: [PATCH 03/15] refactor(rust): read litellm HTTP globals through one Python shim and tighten the http pool Drop the core ocr() facade so VertexAuth and the http pool stay out of litellm-core's public API, move the http Error enum to error.rs, and inject the media DNS resolver into HttpClientPool instead of a per-call builder hook the cache key ignored. The bridge now reads litellm.* HTTP settings only through litellm/rust_bridge/settings.py, pinned by python_settings.json, while env overrides stay in Rust. This adds the Python default User-Agent, parses string ssl_verify globals like get_ssl_verify, drops per-call ssl_verify that Python OCR never honored, and removes the unused request_timeout. Co-Authored-By: Claude Opus 5 --- litellm-rust/crates/core/Cargo.toml | 4 +- litellm-rust/crates/core/src/ocr/client.rs | 11 - litellm-rust/crates/core/tests/ocr.rs | 41 +-- litellm-rust/crates/http/src/config.rs | 93 ++----- litellm-rust/crates/http/src/error.rs | 22 ++ litellm-rust/crates/http/src/lib.rs | 4 +- litellm-rust/crates/http/src/pool.rs | 253 +++++++++++------- litellm-rust/crates/http/src/settings.rs | 2 - .../crates/llms/src/custom_httpx/media.rs | 33 +-- .../crates/python-bridge/python_settings.json | 12 + litellm-rust/crates/python-bridge/src/http.rs | 175 ++++++------ litellm-rust/crates/python-bridge/src/lib.rs | 1 + .../python-bridge/src/python_settings.rs | 48 ++++ litellm/llms/custom_httpx/http_handler.py | 6 +- litellm/rust_bridge/settings.py | 37 +++ .../test_litellm/rust_bridge/test_settings.py | 50 ++++ 16 files changed, 467 insertions(+), 325 deletions(-) create mode 100644 litellm-rust/crates/http/src/error.rs create mode 100644 litellm-rust/crates/python-bridge/python_settings.json create mode 100644 litellm-rust/crates/python-bridge/src/python_settings.rs create mode 100644 litellm/rust_bridge/settings.py create mode 100644 tests/test_litellm/rust_bridge/test_settings.py diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index 1a769ec9708..ab04fb8d4ae 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -15,8 +15,6 @@ futures-util.workspace = true base64.workspace = true litellm-auth.workspace = true litellm-auth-aws.workspace = true -litellm-auth-gcp.workspace = true -litellm-http.workspace = true litellm-llms.workspace = true moka.workspace = true mime_guess = "2.0.5" @@ -37,6 +35,8 @@ url.workspace = true veil.workspace = true [dev-dependencies] +litellm-auth-gcp.workspace = true +litellm-http.workspace = true litellm-llms = { workspace = true, features = ["test-support"] } rstest.workspace = true rstest_reuse.workspace = true diff --git a/litellm-rust/crates/core/src/ocr/client.rs b/litellm-rust/crates/core/src/ocr/client.rs index f0b24623a88..c7b4751bd9e 100644 --- a/litellm-rust/crates/core/src/ocr/client.rs +++ b/litellm-rust/crates/core/src/ocr/client.rs @@ -1,5 +1,3 @@ -use litellm_auth_gcp::VertexAuth; -use litellm_http::{HttpClientConfig, HttpClientPool}; use litellm_llms::{ base_llm::ocr::{error::Error, transformation::LiteLLMOcrResponse}, custom_httpx::llm_http_handler::OcrClient, @@ -16,12 +14,3 @@ pub async fn perform( ) -> Result { litellm_host::run::run(ocr_machine(client.clone()), &LocalOcrHost::new(request)).await } - -pub async fn ocr( - pool: &HttpClientPool, - config: &HttpClientConfig, - vertex_auth: VertexAuth, - request: LiteLLMOcrRequest, -) -> Result { - perform(&OcrClient::new(pool, config, vertex_auth)?, request).await -} diff --git a/litellm-rust/crates/core/tests/ocr.rs b/litellm-rust/crates/core/tests/ocr.rs index 2e414c58541..2ae162d964f 100644 --- a/litellm-rust/crates/core/tests/ocr.rs +++ b/litellm-rust/crates/core/tests/ocr.rs @@ -6,13 +6,13 @@ use litellm_host::{ host::{Host, HostOp, HostResult}, machine::{HostFailure, Machine, MachineStep}, }; -use litellm_http::{HttpClientConfig, HttpClientPool, HttpSettings, Verify}; +use litellm_http::{HttpClientConfig, HttpClientPool, HttpSettings}; use litellm_llms::{ base_llm::ocr::{ error::Error as OcrError, transformation::{LiteLLMOcrResponse, OCR_RESPONSE_MAX_BYTES, OcrTransportConfig}, }, - custom_httpx::llm_http_handler::OcrClient, + custom_httpx::{llm_http_handler::OcrClient, media::PublicDnsResolver}, }; use rstest::rstest; use serde_json::{Value, json}; @@ -173,48 +173,25 @@ async fn facade_retains_native_response_when_requested() { } #[tokio::test] -async fn facade_uses_the_injected_http_pool_configuration() { +async fn ocr_client_uses_the_injected_http_pool_configuration() { let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; let settings = HttpSettings { user_agent: Some("host-owned/1".into()), ..HttpSettings::default() }; - let config = HttpClientConfig::resolve(&settings, None).unwrap(); - crate::ocr::client::ocr( - &HttpClientPool::new(), - &config, + let client = OcrClient::new( + &HttpClientPool::new(Arc::new(PublicDnsResolver)), + &HttpClientConfig::resolve(&settings).unwrap(), VertexAuth::default(), - wire_request("mistral/model", &base, json!({})), ) - .await .unwrap(); + crate::ocr::client::perform(&client, wire_request("mistral/model", &base, json!({}))) + .await + .unwrap(); server.await.unwrap(); assert!(seen.lock().unwrap()[0].contains("user-agent: host-owned/1")); } -#[tokio::test] -async fn unbuildable_http_configuration_fails_before_dispatch() { - let (base, _seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; - let config = HttpClientConfig { - verify: Verify::CaBundle(std::env::temp_dir().join("litellm-ocr-missing-bundle.pem")), - ..HttpClientConfig::resolve(&HttpSettings::default(), None).unwrap() - }; - let error = crate::ocr::client::ocr( - &HttpClientPool::new(), - &config, - VertexAuth::default(), - wire_request("mistral/model", &base, json!({})), - ) - .await - .unwrap_err(); - server.abort(); - assert!(matches!( - error, - OcrError::Transport(litellm_llms::custom_httpx::transport::Error::Connect(_)) - )); - assert!(error.to_string().contains("litellm-ocr-missing-bundle.pem")); -} - fn event_name(event: &CallEvent) -> &'static str { match event { CallEvent::Started { .. } => "started", diff --git a/litellm-rust/crates/http/src/config.rs b/litellm-rust/crates/http/src/config.rs index 86f1a9b43b6..f27092c1fb5 100644 --- a/litellm-rust/crates/http/src/config.rs +++ b/litellm-rust/crates/http/src/config.rs @@ -4,28 +4,10 @@ use std::{ time::Duration, }; -use crate::settings::{HttpSettings, SslVerify}; - -#[derive(Clone, Debug, thiserror::Error, PartialEq, Eq)] -pub enum Error { - #[error("{setting} cannot be expressed with rustls: {reason}")] - Unsupported { - setting: &'static str, - reason: String, - }, - #[error("could not read {}: {message}", path.display())] - Read { path: PathBuf, message: String }, - #[error("{} is not a PEM file: {message}", path.display())] - InvalidPem { path: PathBuf, message: String }, - #[error("could not build the HTTP client: {0}")] - Client(String), -} - -impl From for Error { - fn from(error: reqwest::Error) -> Self { - Self::Client(error.without_url().to_string()) - } -} +use crate::{ + error::Error, + settings::{HttpSettings, SslVerify}, +}; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub enum Verify { @@ -45,17 +27,13 @@ pub struct HttpClientConfig { pub user_agent: Option, pub trust_proxy_env: bool, pub connect_timeout: Duration, - pub request_timeout: Option, } impl HttpClientConfig { - /// Port of `get_ssl_verify` + `get_ssl_configuration`: the per-call value wins, then the - /// configured (environment-overlaid) `ssl_verify`, then `SSL_CERT_FILE`, then the built-in - /// roots. Settings rustls has no equivalent for are an error instead of a silent no-op. - pub fn resolve( - settings: &HttpSettings, - per_call_ssl_verify: Option<&SslVerify>, - ) -> Result { + /// Port of `get_ssl_verify` + `get_ssl_configuration`: the configured (environment-overlaid) + /// `ssl_verify`, then `SSL_CERT_FILE`, then the built-in roots. Settings rustls has no + /// equivalent for are an error instead of a silent no-op. + pub fn resolve(settings: &HttpSettings) -> Result { if let Some(level) = &settings.ssl_security_level { return Err(Error::Unsupported { setting: "ssl_security_level", @@ -68,7 +46,7 @@ impl HttpClientConfig { reason: format!("key exchange group {curve:?} is fixed by the rustls provider"), }); } - let verify = match per_call_ssl_verify.or(settings.ssl_verify.as_ref()) { + let verify = match &settings.ssl_verify { Some(SslVerify::Disabled) => Verify::Disabled, Some(SslVerify::CaBundle(path)) => Verify::CaBundle(path.clone()), Some(SslVerify::Enabled) | None => settings @@ -84,7 +62,6 @@ impl HttpClientConfig { user_agent: settings.user_agent.clone(), trust_proxy_env: settings.trust_proxy_env, connect_timeout: settings.connect_timeout, - request_timeout: settings.request_timeout, }) } @@ -141,14 +118,10 @@ impl HttpClientConfig { Some(agent) => with_protocol.user_agent(agent), None => with_protocol, }; - let with_proxy = if self.trust_proxy_env { + Ok(if self.trust_proxy_env { with_agent } else { with_agent.no_proxy() - }; - Ok(match self.request_timeout { - Some(timeout) => with_proxy.timeout(timeout), - None => with_proxy, }) } } @@ -180,49 +153,25 @@ mod tests { } #[rstest] - #[case::default(settings(None, None), None, Verify::BuiltInRoots)] + #[case::default(settings(None, None), Verify::BuiltInRoots)] #[case::setting_disables( settings(Some(SslVerify::Disabled), Some("/env/roots.pem")), - None, Verify::Disabled )] #[case::setting_bundle( settings(Some(SslVerify::CaBundle("/configured.pem".into())), Some("/env/roots.pem")), - None, Verify::CaBundle("/configured.pem".into()) )] #[case::enabled_uses_cert_file( settings(Some(SslVerify::Enabled), Some("/env/roots.pem")), - None, Verify::CaBundle("/env/roots.pem".into()) )] - #[case::unset_uses_cert_file(settings(None, Some("/env/roots.pem")), None, Verify::CaBundle("/env/roots.pem".into()))] - #[case::per_call_beats_setting( - settings(Some(SslVerify::Disabled), None), - Some(SslVerify::Enabled), - Verify::BuiltInRoots - )] - #[case::per_call_disables( - settings(Some(SslVerify::CaBundle("/configured.pem".into())), Some("/env/roots.pem")), - Some(SslVerify::Disabled), - Verify::Disabled - )] - #[case::per_call_bundle( - settings(None, Some("/env/roots.pem")), - Some(SslVerify::CaBundle("/call.pem".into())), - Verify::CaBundle("/call.pem".into()) - )] - #[case::per_call_enabled_still_honours_cert_file( - settings(Some(SslVerify::Disabled), Some("/env/roots.pem")), - Some(SslVerify::Enabled), - Verify::CaBundle("/env/roots.pem".into()) - )] - fn verify_follows_per_call_then_setting_then_cert_file( + #[case::unset_uses_cert_file(settings(None, Some("/env/roots.pem")), Verify::CaBundle("/env/roots.pem".into()))] + fn verify_follows_setting_then_cert_file( #[case] settings: HttpSettings, - #[case] per_call: Option, #[case] expected: Verify, ) { - let config = HttpClientConfig::resolve(&settings, per_call.as_ref()).unwrap(); + let config = HttpClientConfig::resolve(&settings).unwrap(); assert_eq!(config.verify, expected); } @@ -233,7 +182,7 @@ mod tests { ..HttpSettings::default() } .with_environment(&|name: &str| (name == "SSL_VERIFY").then(|| "true".to_string())); - let config = HttpClientConfig::resolve(&settings, None).unwrap(); + let config = HttpClientConfig::resolve(&settings).unwrap(); assert_eq!(config.verify, Verify::BuiltInRoots); } @@ -244,7 +193,7 @@ mod tests { ..HttpSettings::default() }; assert!(matches!( - HttpClientConfig::resolve(&settings, None), + HttpClientConfig::resolve(&settings), Err(Error::Unsupported { setting: "ssl_security_level", .. @@ -259,7 +208,7 @@ mod tests { ..HttpSettings::default() }; assert!(matches!( - HttpClientConfig::resolve(&settings, None), + HttpClientConfig::resolve(&settings), Err(Error::Unsupported { setting: "ssl_ecdh_curve", .. @@ -276,10 +225,9 @@ mod tests { user_agent: Some("litellm/1.0".into()), trust_proxy_env: true, connect_timeout: Duration::from_secs(7), - request_timeout: Some(Duration::from_secs(70)), ..HttpSettings::default() }; - let config = HttpClientConfig::resolve(&settings, None).unwrap(); + let config = HttpClientConfig::resolve(&settings).unwrap(); assert_eq!( config, HttpClientConfig { @@ -290,7 +238,6 @@ mod tests { user_agent: Some("litellm/1.0".into()), trust_proxy_env: true, connect_timeout: Duration::from_secs(7), - request_timeout: Some(Duration::from_secs(70)), } ); } @@ -300,7 +247,7 @@ mod tests { let path = std::env::temp_dir().join("litellm-http-missing-bundle.pem"); let config = HttpClientConfig { verify: Verify::CaBundle(path.clone()), - ..HttpClientConfig::resolve(&HttpSettings::default(), None).unwrap() + ..HttpClientConfig::resolve(&HttpSettings::default()).unwrap() }; assert!(matches!( config.client_builder(), @@ -315,7 +262,7 @@ mod tests { std::fs::write(&path, b"not a certificate").unwrap(); let config = HttpClientConfig { verify: Verify::CaBundle(path.clone()), - ..HttpClientConfig::resolve(&HttpSettings::default(), None).unwrap() + ..HttpClientConfig::resolve(&HttpSettings::default()).unwrap() }; let result = config.client_builder().map(drop); std::fs::remove_file(&path).unwrap(); diff --git a/litellm-rust/crates/http/src/error.rs b/litellm-rust/crates/http/src/error.rs new file mode 100644 index 00000000000..27899f06cf1 --- /dev/null +++ b/litellm-rust/crates/http/src/error.rs @@ -0,0 +1,22 @@ +use std::path::PathBuf; + +#[derive(Clone, Debug, thiserror::Error, PartialEq, Eq)] +pub enum Error { + #[error("{setting} cannot be expressed with rustls: {reason}")] + Unsupported { + setting: &'static str, + reason: String, + }, + #[error("could not read {}: {message}", path.display())] + Read { path: PathBuf, message: String }, + #[error("{} is not a PEM file: {message}", path.display())] + InvalidPem { path: PathBuf, message: String }, + #[error("could not build the HTTP client: {0}")] + Client(String), +} + +impl From for Error { + fn from(error: reqwest::Error) -> Self { + Self::Client(error.without_url().to_string()) + } +} diff --git a/litellm-rust/crates/http/src/lib.rs b/litellm-rust/crates/http/src/lib.rs index d62dd768fe1..9c88e3101a7 100644 --- a/litellm-rust/crates/http/src/lib.rs +++ b/litellm-rust/crates/http/src/lib.rs @@ -3,9 +3,11 @@ //! caches `reqwest::Client`s per resolved configuration. mod config; +mod error; mod pool; mod settings; -pub use config::{Error, HttpClientConfig, Verify}; +pub use config::{HttpClientConfig, Verify}; +pub use error::Error; pub use pool::{ClientVariant, HttpClientPool}; pub use settings::{HttpSettings, SslVerify}; diff --git a/litellm-rust/crates/http/src/pool.rs b/litellm-rust/crates/http/src/pool.rs index 065097556e1..03c01e968ac 100644 --- a/litellm-rust/crates/http/src/pool.rs +++ b/litellm-rust/crates/http/src/pool.rs @@ -1,112 +1,174 @@ use std::{ collections::HashMap, - sync::{Mutex, PoisonError}, + sync::{Arc, Mutex, PoisonError}, }; -use crate::config::{Error, HttpClientConfig}; +use reqwest::dns::Resolve; + +use crate::{config::HttpClientConfig, error::Error}; /// The client shapes routes need; each is the shared base plus one policy. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum ClientVariant { Provider, NoRedirect, - /// Media downloads: no redirects (the fetcher validates each hop) and never a proxy. + /// Media downloads: no redirects (the fetcher validates each hop), never a proxy, and the + /// pool's media resolver. Media, } -impl ClientVariant { - fn apply(self, builder: reqwest::ClientBuilder) -> reqwest::ClientBuilder { - match self { - Self::Provider => builder, - Self::NoRedirect => builder.redirect(reqwest::redirect::Policy::none()), - Self::Media => builder - .redirect(reqwest::redirect::Policy::none()) - .no_proxy(), - } - } -} - /// Counterpart of `get_async_httpx_client`: one `reqwest::Client` per resolved configuration /// and variant, built on first use and shared afterwards. -#[derive(Default)] pub struct HttpClientPool { + media_resolver: Arc, clients: Mutex>, } impl HttpClientPool { - pub fn new() -> Self { - Self::default() + pub fn new(media_resolver: Arc) -> Self { + Self { + media_resolver, + clients: Mutex::default(), + } } pub fn client( &self, config: &HttpClientConfig, variant: ClientVariant, - ) -> Result { - self.client_with(config, variant, |builder| builder) - } - - /// Like [`Self::client`], with a caller hook for builder options that are not plain values - /// (a DNS resolver, for example). The hook only runs when the client is first built. - pub fn client_with( - &self, - config: &HttpClientConfig, - variant: ClientVariant, - customize: impl FnOnce(reqwest::ClientBuilder) -> reqwest::ClientBuilder, ) -> Result { let key = (config.clone(), variant); let mut clients = self.clients.lock().unwrap_or_else(PoisonError::into_inner); if let Some(client) = clients.get(&key) { return Ok(client.clone()); } - let client = customize(variant.apply(config.client_builder()?)).build()?; + let client = self.apply(variant, config.client_builder()?).build()?; clients.insert(key, client.clone()); Ok(client) } + + fn apply( + &self, + variant: ClientVariant, + builder: reqwest::ClientBuilder, + ) -> reqwest::ClientBuilder { + match variant { + ClientVariant::Provider => builder, + ClientVariant::NoRedirect => builder.redirect(reqwest::redirect::Policy::none()), + ClientVariant::Media => builder + .redirect(reqwest::redirect::Policy::none()) + .no_proxy() + .dns_resolver2(Arc::clone(&self.media_resolver)), + } + } } #[cfg(test)] mod tests { - use std::{cell::Cell, time::Duration}; + use std::{ + net::SocketAddr, + sync::atomic::{AtomicUsize, Ordering}, + time::Duration, + }; - use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use reqwest::dns::{Addrs, Name, Resolving}; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + }; use super::*; use crate::{HttpSettings, Verify}; - fn config(user_agent: &str) -> HttpClientConfig { - HttpClientConfig { - user_agent: Some(user_agent.into()), - ..HttpClientConfig::resolve(&HttpSettings::default(), None).unwrap() + struct FixedResolver(SocketAddr); + + impl Resolve for FixedResolver { + fn resolve(&self, _: Name) -> Resolving { + let addrs: Addrs = Box::new(std::iter::once(self.0)); + Box::pin(std::future::ready(Ok(addrs))) } } - #[test] - fn clients_are_built_once_per_config_and_variant() { - let pool = HttpClientPool::new(); - let builds = Cell::new(0); - let build = |config: &HttpClientConfig, variant| { - pool.client_with(config, variant, |builder| { - builds.set(builds.get() + 1); - builder - }) + fn pool() -> HttpClientPool { + HttpClientPool::new(Arc::new(FixedResolver(([192, 0, 2, 1], 80).into()))) + } + + fn config(user_agent: &str) -> HttpClientConfig { + HttpClientConfig { + user_agent: Some(user_agent.into()), + ..HttpClientConfig::resolve(&HttpSettings::default()).unwrap() + } + } + + /// Answers every request on every connection with `status_line` and counts connections, + /// so a reused client shows up as a reused keep-alive connection. + async fn serve( + status_line: &'static str, + ) -> (SocketAddr, Arc, Arc>>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let connections = Arc::new(AtomicUsize::new(0)); + let requests = Arc::new(Mutex::new(Vec::new())); + let (accepted, seen) = (Arc::clone(&connections), Arc::clone(&requests)); + tokio::spawn(async move { + loop { + let (mut socket, _) = listener.accept().await.unwrap(); + accepted.fetch_add(1, Ordering::SeqCst); + let seen = Arc::clone(&seen); + tokio::spawn(async move { + let mut buffer = vec![0u8; 4096]; + while let Ok(read) = socket.read(&mut buffer).await { + if read == 0 { + return; + } + seen.lock() + .unwrap() + .push(String::from_utf8_lossy(&buffer[..read]).into_owned()); + let response = format!( + "{status_line}\r\nLocation: /elsewhere\r\nContent-Length: 0\r\n\r\n" + ); + if socket.write_all(response.as_bytes()).await.is_err() { + return; + } + } + }); + } + }); + (address, connections, requests) + } + + async fn get( + pool: &HttpClientPool, + config: &HttpClientConfig, + variant: ClientVariant, + url: &str, + ) -> reqwest::Response { + pool.client(config, variant) .unwrap() - }; - build(&config("a"), ClientVariant::Provider); - build(&config("a"), ClientVariant::Provider); - assert_eq!(builds.get(), 1); - build(&config("a"), ClientVariant::NoRedirect); - assert_eq!(builds.get(), 2); - build(&config("b"), ClientVariant::Provider); - assert_eq!(builds.get(), 3); - build(&config("b"), ClientVariant::Provider); - build(&config("a"), ClientVariant::NoRedirect); - assert_eq!(builds.get(), 3); + .get(url) + .timeout(Duration::from_secs(5)) + .send() + .await + .unwrap() + } + + #[tokio::test] + async fn clients_are_shared_per_config_and_variant() { + let (address, connections, _) = serve("HTTP/1.1 204 No Content").await; + let url = format!("http://{address}"); + let pool = pool(); + get(&pool, &config("a"), ClientVariant::Provider, &url).await; + get(&pool, &config("a"), ClientVariant::Provider, &url).await; + assert_eq!(connections.load(Ordering::SeqCst), 1); + get(&pool, &config("a"), ClientVariant::NoRedirect, &url).await; + assert_eq!(connections.load(Ordering::SeqCst), 2); + get(&pool, &config("b"), ClientVariant::Provider, &url).await; + assert_eq!(connections.load(Ordering::SeqCst), 3); } #[test] fn build_failures_are_not_cached() { - let pool = HttpClientPool::new(); + let pool = pool(); let missing = HttpClientConfig { verify: Verify::CaBundle(std::env::temp_dir().join("litellm-http-absent.pem")), ..config("a") @@ -116,57 +178,52 @@ mod tests { assert!(pool.client(&config("a"), ClientVariant::Provider).is_ok()); } - async fn serve_once(status_line: &'static str) -> (String, tokio::task::JoinHandle) { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let base = format!("http://{}", listener.local_addr().unwrap()); - let server = tokio::spawn(async move { - let (mut socket, _) = listener.accept().await.unwrap(); - let mut request = vec![0u8; 4096]; - let read = socket.read(&mut request).await.unwrap(); - socket - .write_all( - format!("{status_line}\r\nLocation: /elsewhere\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") - .as_bytes(), - ) - .await - .unwrap(); - String::from_utf8_lossy(&request[..read]).into_owned() - }); - (base, server) - } - #[tokio::test] async fn provider_client_sends_the_configured_user_agent_over_http1() { - let (base, server) = serve_once("HTTP/1.1 204 No Content").await; - let config = HttpClientConfig { - connect_timeout: Duration::from_secs(2), - ..config("litellm-test/9") - }; - let response = HttpClientPool::new() - .client(&config, ClientVariant::Provider) - .unwrap() - .get(&base) - .send() - .await - .unwrap(); + let (address, _, requests) = serve("HTTP/1.1 204 No Content").await; + let response = get( + &pool(), + &config("litellm-test/9"), + ClientVariant::Provider, + &format!("http://{address}"), + ) + .await; assert_eq!(response.status(), 204); assert_eq!(response.version(), reqwest::Version::HTTP_11); - let request = server.await.unwrap(); + let request = requests.lock().unwrap()[0].clone(); assert!(request.contains("user-agent: litellm-test/9"), "{request}"); } #[tokio::test] async fn no_redirect_variant_returns_the_redirect_instead_of_following_it() { - let (base, server) = serve_once("HTTP/1.1 302 Found").await; - let response = HttpClientPool::new() - .client(&config("a"), ClientVariant::NoRedirect) - .unwrap() - .get(&base) - .send() - .await - .unwrap(); + let (address, _, _) = serve("HTTP/1.1 302 Found").await; + let response = get( + &pool(), + &config("a"), + ClientVariant::NoRedirect, + &format!("http://{address}"), + ) + .await; assert_eq!(response.status(), 302); assert_eq!(response.headers()["location"], "/elsewhere"); - server.await.unwrap(); + } + + #[tokio::test] + async fn media_variant_resolves_through_the_injected_resolver() { + let (address, _, requests) = serve("HTTP/1.1 204 No Content").await; + let pool = HttpClientPool::new(Arc::new(FixedResolver(address))); + let url = format!("http://media.invalid:{}/doc", address.port()); + let response = get(&pool, &config("a"), ClientVariant::Media, &url).await; + assert_eq!(response.status(), 204); + assert!(requests.lock().unwrap()[0].contains("host: media.invalid")); + assert!( + pool.client(&config("a"), ClientVariant::Provider) + .unwrap() + .get(&url) + .timeout(Duration::from_secs(5)) + .send() + .await + .is_err() + ); } } diff --git a/litellm-rust/crates/http/src/settings.rs b/litellm-rust/crates/http/src/settings.rs index 4d936e89046..45aab0d6fa5 100644 --- a/litellm-rust/crates/http/src/settings.rs +++ b/litellm-rust/crates/http/src/settings.rs @@ -31,7 +31,6 @@ pub struct HttpSettings { pub user_agent: Option, pub trust_proxy_env: bool, pub connect_timeout: Duration, - pub request_timeout: Option, } impl Default for HttpSettings { @@ -47,7 +46,6 @@ impl Default for HttpSettings { user_agent: None, trust_proxy_env: false, connect_timeout: Duration::from_secs(5), - request_timeout: None, } } } diff --git a/litellm-rust/crates/llms/src/custom_httpx/media.rs b/litellm-rust/crates/llms/src/custom_httpx/media.rs index aeac4894683..a1c4fe68734 100644 --- a/litellm-rust/crates/llms/src/custom_httpx/media.rs +++ b/litellm-rust/crates/llms/src/custom_httpx/media.rs @@ -66,26 +66,15 @@ impl MediaFetcher { pool: &HttpClientPool, config: &HttpClientConfig, ) -> Result { - Self::with_resolvers( - pool, - config, - Arc::new(PublicDnsResolver), - Arc::new(SystemAddressResolver), - ) + Self::with_address_resolver(pool, config, Arc::new(SystemAddressResolver)) } - fn with_resolvers( + fn with_address_resolver( pool: &HttpClientPool, config: &HttpClientConfig, - transport_resolver: Arc, address_resolver: Arc, - ) -> Result - where - R: Resolve + 'static, - { - let client = pool.client_with(config, ClientVariant::Media, |builder| { - builder.dns_resolver(transport_resolver) - })?; + ) -> Result { + let client = pool.client(config, ClientVariant::Media)?; Ok(Self { client, address_resolver, @@ -242,7 +231,7 @@ fn is_blocked_ip(ip: IpAddr) -> bool { } #[derive(Default)] -struct PublicDnsResolver; +pub struct PublicDnsResolver; struct SystemAddressResolver; @@ -371,10 +360,9 @@ mod tests { address: SocketAddr, blocked_hosts: HashSet<&'static str>, ) -> MediaFetcher { - MediaFetcher::with_resolvers( - &HttpClientPool::new(), - &HttpClientConfig::resolve(&HttpSettings::default(), None).unwrap(), - Arc::new(LoopbackDnsResolver(address)), + MediaFetcher::with_address_resolver( + &HttpClientPool::new(Arc::new(LoopbackDnsResolver(address))), + &HttpClientConfig::resolve(&HttpSettings::default()).unwrap(), Arc::new(TestAddressResolver { blocked_hosts }), ) .expect("test fetcher builds") @@ -552,9 +540,8 @@ mod tests { #[tokio::test] async fn rejects_url_credentials_before_network_access() { let fetcher = MediaFetcher::new( - &HttpClientPool::new(), - &HttpClientConfig::resolve(&litellm_http::HttpSettings::default(), None) - .expect("default settings resolve"), + &HttpClientPool::new(Arc::new(PublicDnsResolver)), + &HttpClientConfig::resolve(&HttpSettings::default()).expect("default settings resolve"), ) .expect("media fetcher builds"); let url = diff --git a/litellm-rust/crates/python-bridge/python_settings.json b/litellm-rust/crates/python-bridge/python_settings.json new file mode 100644 index 00000000000..a6cd959c6de --- /dev/null +++ b/litellm-rust/crates/python-bridge/python_settings.json @@ -0,0 +1,12 @@ +{ + "http_settings": [ + "ssl_verify", + "ssl_certificate", + "ssl_security_level", + "ssl_ecdh_curve", + "force_ipv4", + "http2", + "aiohttp_trust_env", + "user_agent" + ] +} diff --git a/litellm-rust/crates/python-bridge/src/http.rs b/litellm-rust/crates/python-bridge/src/http.rs index d2e05c3b949..de6fb5bb96d 100644 --- a/litellm-rust/crates/python-bridge/src/http.rs +++ b/litellm-rust/crates/python-bridge/src/http.rs @@ -1,11 +1,16 @@ -use std::{path::PathBuf, sync::LazyLock}; +use std::{ + path::PathBuf, + sync::{Arc, LazyLock}, +}; use litellm_http::{HttpClientConfig, HttpClientPool, HttpSettings, SslVerify}; +use litellm_llms::custom_httpx::media::PublicDnsResolver; use pyo3::{prelude::*, types::PyDict}; -use crate::errors::RustBridgeDeclined; +use crate::{errors::RustBridgeDeclined, python_settings::PythonSettings}; -static POOL: LazyLock = LazyLock::new(HttpClientPool::new); +static POOL: LazyLock = + LazyLock::new(|| HttpClientPool::new(Arc::new(PublicDnsResolver))); /// Keyword arguments that carry a live Python HTTP client or session. They cannot cross into /// Rust, so a call that supplies one stays on the Python path. @@ -15,21 +20,16 @@ pub(crate) fn pool() -> &'static HttpClientPool { &POOL } -/// The client configuration for one call: the process settings from the `litellm` module and -/// the environment, narrowed by the call's own `ssl_verify`. +/// The client configuration for one call: the `litellm.*` HTTP settings with the environment +/// overlaid, the same way `http_handler.py` combines them. pub(crate) fn call_config( py: Python<'_>, kwargs: &Bound<'_, PyDict>, ) -> PyResult { decline_live_clients(kwargs)?; - let settings = settings(py.import("litellm")?.as_any())? + let settings = settings(&PythonSettings::Http.read(py)?)? .with_environment(&|name| std::env::var(name).ok()); - let per_call = kwargs - .get_item("ssl_verify")? - .map(|value| ssl_verify(&value, "ssl_verify")) - .transpose()? - .flatten(); - HttpClientConfig::resolve(&settings, per_call.as_ref()) + HttpClientConfig::resolve(&settings) .map_err(|error| RustBridgeDeclined::new_err(error.to_string())) } @@ -44,45 +44,47 @@ pub(crate) fn decline_live_clients(kwargs: &Bound<'_, PyDict>) -> PyResult<()> { Ok(()) } -/// Read the `litellm.*` globals `http_handler.py` consults. `globals` is the `litellm` module in -/// production and any attribute holder in tests. -pub(crate) fn settings(globals: &Bound<'_, PyAny>) -> PyResult { +#[derive(FromPyObject)] +struct PythonHttpSettings<'py> { + ssl_verify: Bound<'py, PyAny>, + ssl_certificate: Option, + ssl_security_level: Option, + ssl_ecdh_curve: Option, + force_ipv4: bool, + http2: bool, + aiohttp_trust_env: bool, + user_agent: String, +} + +fn settings(value: &Bound<'_, PyAny>) -> PyResult { + let python: PythonHttpSettings = value.extract()?; Ok(HttpSettings { - ssl_verify: ssl_verify(&globals.getattr("ssl_verify")?, "litellm.ssl_verify")?, - ssl_certificate: optional_path(globals, "ssl_certificate")?, - ssl_security_level: globals.getattr("ssl_security_level")?.extract()?, - ssl_ecdh_curve: globals.getattr("ssl_ecdh_curve")?.extract()?, - force_ipv4: globals.getattr("force_ipv4")?.extract()?, - http2: globals.getattr("http2")?.extract()?, - trust_proxy_env: globals.getattr("aiohttp_trust_env")?.extract()?, + ssl_verify: Some(ssl_verify(&python.ssl_verify)?), + ssl_certificate: python.ssl_certificate.map(PathBuf::from), + ssl_security_level: python.ssl_security_level, + ssl_ecdh_curve: python.ssl_ecdh_curve, + force_ipv4: python.force_ipv4, + http2: python.http2, + user_agent: Some(python.user_agent), + trust_proxy_env: python.aiohttp_trust_env, ..HttpSettings::default() }) } -fn optional_path(globals: &Bound<'_, PyAny>, name: &str) -> PyResult> { - Ok(globals - .getattr(name)? - .extract::>()? - .map(PathBuf::from)) -} - -fn ssl_verify(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { - if value.is_none() { - return Ok(None); - } +fn ssl_verify(value: &Bound<'_, PyAny>) -> PyResult { if let Ok(enabled) = value.extract::() { - return Ok(Some(if enabled { + return Ok(if enabled { SslVerify::Enabled } else { SslVerify::Disabled - })); + }); } if let Ok(path) = value.extract::() { - return Ok(Some(SslVerify::CaBundle(PathBuf::from(path)))); + return Ok(SslVerify::parse(&path)); } - Err(RustBridgeDeclined::new_err(format!( - "{name} is a live Python object and cannot be used by the Rust route" - ))) + Err(RustBridgeDeclined::new_err( + "litellm.ssl_verify is a live Python object and cannot be used by the Rust route", + )) } #[cfg(test)] @@ -91,18 +93,16 @@ mod tests { use rstest::rstest; use super::*; + use crate::python_settings::CONTRACT; - fn eval<'py>(py: Python<'py>, source: &std::ffi::CStr) -> Bound<'py, PyDict> { - let locals = PyDict::new(py); - py.run(source, Some(&locals), Some(&locals)).unwrap(); - locals - } - - fn globals<'py>(py: Python<'py>, overrides: &str) -> Bound<'py, PyAny> { + /// A stand-in for `http_settings()` carrying exactly the fields the contract declares, so a + /// field Rust reads but Python does not return fails here. + fn python_settings<'py>(py: Python<'py>, overrides: &str) -> Bound<'py, PyAny> { let source = format!( " +import json import types -globals = types.SimpleNamespace( +defaults = dict( ssl_verify=True, ssl_certificate=None, ssl_security_level=None, @@ -110,23 +110,29 @@ globals = types.SimpleNamespace( force_ipv4=False, http2=False, aiohttp_trust_env=False, + user_agent='litellm/test', ) -{overrides} +defaults.update(dict({overrides})) +settings = types.SimpleNamespace(**{{name: defaults[name] for name in json.loads(contract)['http_settings']}}) " ); + let locals = PyDict::new(py); + locals.set_item("contract", CONTRACT).unwrap(); let source = std::ffi::CString::new(source).unwrap(); - eval(py, &source).get_item("globals").unwrap().unwrap() + py.run(&source, Some(&locals), Some(&locals)).unwrap(); + locals.get_item("settings").unwrap().unwrap() } #[test] - fn default_globals_produce_default_settings_with_verification_on() { + fn default_python_settings_produce_default_settings_with_verification_on() { Python::initialize(); Python::attach(|py| { - let settings = settings(&globals(py, "")).unwrap(); + let settings = settings(&python_settings(py, "")).unwrap(); assert_eq!( settings, HttpSettings { ssl_verify: Some(SslVerify::Enabled), + user_agent: Some("litellm/test".into()), ..HttpSettings::default() } ); @@ -134,19 +140,20 @@ globals = types.SimpleNamespace( } #[test] - fn globals_flow_into_settings() { + fn python_settings_flow_into_settings() { Python::initialize(); Python::attach(|py| { - let settings = settings(&globals( + let settings = settings(&python_settings( py, " -globals.ssl_verify = '/etc/ssl/corp.pem' -globals.ssl_certificate = '/etc/ssl/client.pem' -globals.ssl_security_level = '2' -globals.ssl_ecdh_curve = 'X25519' -globals.force_ipv4 = True -globals.http2 = True -globals.aiohttp_trust_env = True +ssl_verify='/etc/ssl/corp.pem', +ssl_certificate='/etc/ssl/client.pem', +ssl_security_level='2', +ssl_ecdh_curve='X25519', +force_ipv4=True, +http2=True, +aiohttp_trust_env=True, +user_agent='litellm/9.9.9', ", )) .unwrap(); @@ -159,6 +166,7 @@ globals.aiohttp_trust_env = True ssl_ecdh_curve: Some("X25519".into()), force_ipv4: true, http2: true, + user_agent: Some("litellm/9.9.9".into()), trust_proxy_env: true, ..HttpSettings::default() } @@ -167,13 +175,32 @@ globals.aiohttp_trust_env = True } #[test] - fn disabled_verification_global_resolves_to_disabled() { + fn user_agent_environment_variable_beats_the_python_default() { Python::initialize(); Python::attach(|py| { - let settings = settings(&globals(py, "globals.ssl_verify = False")).unwrap(); - assert_eq!(settings.ssl_verify, Some(SslVerify::Disabled)); - let config = HttpClientConfig::resolve(&settings, None).unwrap(); - assert_eq!(config.verify, Verify::Disabled); + let settings = settings(&python_settings(py, "")) + .unwrap() + .with_environment(&|name| { + (name == "LITELLM_USER_AGENT").then(|| "operator/1".to_string()) + }); + assert_eq!(settings.user_agent.as_deref(), Some("operator/1")); + }); + } + + #[rstest] + #[case::disabled("ssl_verify=False", Verify::Disabled)] + #[case::disabled_string("ssl_verify='False'", Verify::Disabled)] + #[case::enabled_string("ssl_verify='true'", Verify::BuiltInRoots)] + #[case::bundle("ssl_verify='/tmp/ca.pem'", Verify::CaBundle("/tmp/ca.pem".into()))] + fn ssl_verify_global_resolves_like_get_ssl_verify( + #[case] overrides: &str, + #[case] expected: Verify, + ) { + Python::initialize(); + Python::attach(|py| { + let settings = settings(&python_settings(py, overrides)).unwrap(); + let config = HttpClientConfig::resolve(&settings).unwrap(); + assert_eq!(config.verify, expected); }); } @@ -181,7 +208,7 @@ globals.aiohttp_trust_env = True fn ssl_context_global_declines_instead_of_being_dropped() { Python::initialize(); Python::attach(|py| { - let error = settings(&globals(py, "globals.ssl_verify = object()")).unwrap_err(); + let error = settings(&python_settings(py, "ssl_verify=object()")).unwrap_err(); assert!(error.is_instance_of::(py)); assert!(error.value(py).to_string().contains("litellm.ssl_verify")); }); @@ -215,20 +242,4 @@ globals.aiohttp_trust_env = True decline_live_clients(&kwargs).unwrap(); }); } - - #[rstest] - #[case::disabled(c"False", Some(SslVerify::Disabled))] - #[case::enabled(c"True", Some(SslVerify::Enabled))] - #[case::bundle(c"'/tmp/ca.pem'", Some(SslVerify::CaBundle("/tmp/ca.pem".into())))] - #[case::unset(c"None", None)] - fn per_call_ssl_verify_values_project( - #[case] source: &std::ffi::CStr, - #[case] expected: Option, - ) { - Python::initialize(); - Python::attach(|py| { - let value = py.eval(source, None, None).unwrap(); - assert_eq!(ssl_verify(&value, "ssl_verify").unwrap(), expected); - }); - } } diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 11cb0a7f655..7eba0d201be 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -3,6 +3,7 @@ mod diagnostics; mod errors; mod http; mod marshal; +mod python_settings; mod routes; mod token_counter; diff --git a/litellm-rust/crates/python-bridge/src/python_settings.rs b/litellm-rust/crates/python-bridge/src/python_settings.rs new file mode 100644 index 00000000000..0c6554f0970 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/python_settings.rs @@ -0,0 +1,48 @@ +use pyo3::prelude::*; + +const MODULE: &str = "litellm.rust_bridge.settings"; + +/// Every group of `litellm.*` module globals the native routes read. Environment overrides are +/// applied on the Rust side, so each function returns only what the Python process configured. +/// A group is deleted once Rust owns loading that configuration, so this enum only shrinks. +/// +/// `litellm/rust_bridge/settings.py` is the only Python module behind it, and +/// `python_settings.json` pins the fields each function returns on both sides. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum PythonSettings { + Http, +} + +impl PythonSettings { + #[cfg(test)] + pub(crate) const ALL: [Self; 1] = [Self::Http]; + + pub(crate) fn name(self) -> &'static str { + match self { + Self::Http => "http_settings", + } + } + + pub(crate) fn read(self, py: Python<'_>) -> PyResult> { + py.import(MODULE)?.getattr(self.name())?.call0() + } +} + +#[cfg(test)] +pub(crate) const CONTRACT: &str = include_str!("../python_settings.json"); + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + + use super::{CONTRACT, PythonSettings}; + + #[test] + fn every_settings_group_is_in_the_python_contract() { + let contract: serde_json::Map = + serde_json::from_str(CONTRACT).unwrap(); + let declared: BTreeSet<&str> = contract.keys().map(String::as_str).collect(); + let read: BTreeSet<&str> = PythonSettings::ALL.map(PythonSettings::name).into(); + assert_eq!(read, declared); + } +} diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 05dff0cb9d8..6b90394043f 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -150,7 +150,11 @@ def get_default_headers() -> dict: if user_agent is not None: return {"User-Agent": user_agent} - return {"User-Agent": f"litellm/{version}"} + return {"User-Agent": default_user_agent()} + + +def default_user_agent() -> str: + return f"litellm/{version}" # Initialize headers (User-Agent) diff --git a/litellm/rust_bridge/settings.py b/litellm/rust_bridge/settings.py new file mode 100644 index 00000000000..a8229b12d13 --- /dev/null +++ b/litellm/rust_bridge/settings.py @@ -0,0 +1,37 @@ +"""The `litellm.*` module globals the native routes read. + +Environment variables that override these are applied in Rust, so nothing here reads `os.environ`. +`litellm-rust/crates/python-bridge/python_settings.json` pins the fields each function returns. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class HttpSettings: + ssl_verify: bool | str + ssl_certificate: str | None + ssl_security_level: str | None + ssl_ecdh_curve: str | None + force_ipv4: bool + http2: bool + aiohttp_trust_env: bool + user_agent: str + + +def http_settings() -> HttpSettings: + import litellm + from litellm.llms.custom_httpx.http_handler import default_user_agent + + return HttpSettings( + ssl_verify=litellm.ssl_verify, + ssl_certificate=litellm.ssl_certificate, + ssl_security_level=litellm.ssl_security_level, + ssl_ecdh_curve=litellm.ssl_ecdh_curve, + force_ipv4=litellm.force_ipv4, + http2=litellm.http2, + aiohttp_trust_env=litellm.aiohttp_trust_env, + user_agent=default_user_agent(), + ) diff --git a/tests/test_litellm/rust_bridge/test_settings.py b/tests/test_litellm/rust_bridge/test_settings.py new file mode 100644 index 00000000000..618fa400136 --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_settings.py @@ -0,0 +1,50 @@ +import dataclasses +from pathlib import Path +from typing import Final + +import pytest +from pydantic import TypeAdapter + +import litellm +from litellm.llms.custom_httpx.http_handler import default_user_agent +from litellm.rust_bridge import settings + +CONTRACT_PATH: Final = Path(__file__).parents[3] / "litellm-rust/crates/python-bridge/python_settings.json" + + +def test_the_rust_contract_matches_the_returned_fields() -> None: + contract: Final = TypeAdapter(dict[str, list[str]]).validate_json(CONTRACT_PATH.read_text()) + + assert contract == {"http_settings": [field.name for field in dataclasses.fields(settings.http_settings())]} + + +def test_http_settings_reads_the_litellm_globals(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "ssl_verify", "/etc/ssl/corp.pem") + monkeypatch.setattr(litellm, "ssl_certificate", "/etc/ssl/client.pem") + monkeypatch.setattr(litellm, "ssl_security_level", "DEFAULT@SECLEVEL=1") + monkeypatch.setattr(litellm, "ssl_ecdh_curve", "X25519") + monkeypatch.setattr(litellm, "force_ipv4", True) + monkeypatch.setattr(litellm, "http2", True) + monkeypatch.setattr(litellm, "aiohttp_trust_env", True) + + assert settings.http_settings() == settings.HttpSettings( + ssl_verify="/etc/ssl/corp.pem", + ssl_certificate="/etc/ssl/client.pem", + ssl_security_level="DEFAULT@SECLEVEL=1", + ssl_ecdh_curve="X25519", + force_ipv4=True, + http2=True, + aiohttp_trust_env=True, + user_agent=settings.http_settings().user_agent, + ) + + +def test_http_settings_ignores_environment_overrides(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_USER_AGENT", "operator/1") + monkeypatch.setenv("SSL_VERIFY", "false") + monkeypatch.setattr(litellm, "ssl_verify", True) + + result: Final = settings.http_settings() + + assert result.user_agent == default_user_agent() + assert result.ssl_verify is True From 542ad7dbacb4448878da75432fd837cec4885b56 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Sat, 19 Sep 2026 00:27:54 +0000 Subject: [PATCH 04/15] fix(ocr): forward the supplied client on the Python path and build pooled clients outside the lock Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/crates/http/src/pool.rs | 12 +++++++----- litellm/ocr/main.py | 8 ++++++++ tests/test_litellm/ocr/test_main.py | 29 ++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/litellm-rust/crates/http/src/pool.rs b/litellm-rust/crates/http/src/pool.rs index 03c01e968ac..613ce9c2831 100644 --- a/litellm-rust/crates/http/src/pool.rs +++ b/litellm-rust/crates/http/src/pool.rs @@ -1,6 +1,6 @@ use std::{ collections::HashMap, - sync::{Arc, Mutex, PoisonError}, + sync::{Arc, Mutex, MutexGuard, PoisonError}, }; use reqwest::dns::Resolve; @@ -38,13 +38,15 @@ impl HttpClientPool { variant: ClientVariant, ) -> Result { let key = (config.clone(), variant); - let mut clients = self.clients.lock().unwrap_or_else(PoisonError::into_inner); - if let Some(client) = clients.get(&key) { + if let Some(client) = self.lock().get(&key) { return Ok(client.clone()); } let client = self.apply(variant, config.client_builder()?).build()?; - clients.insert(key, client.clone()); - Ok(client) + Ok(self.lock().entry(key).or_insert(client).clone()) + } + + fn lock(&self) -> MutexGuard<'_, HashMap<(HttpClientConfig, ClientVariant), reqwest::Client>> { + self.clients.lock().unwrap_or_else(PoisonError::into_inner) } fn apply( diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 06830ed4b53..851d9162964 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -25,6 +25,7 @@ from litellm.llms.base_llm.ocr.transformation import ( OCRResponse, parse_ocr_request_format, ) +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import CustomPricingLiteLLMParams @@ -52,6 +53,11 @@ class _PreparedOCRRequest: litellm_logging_obj: LiteLLMLoggingObj +def _supplied_client(kwargs: Mapping[str, object]) -> HTTPHandler | AsyncHTTPHandler | None: + candidate: Final = kwargs.get("client") + return candidate if isinstance(candidate, (HTTPHandler, AsyncHTTPHandler)) else None + + def _prepare_ocr_request( model: str, document: Mapping[str, object], @@ -238,6 +244,7 @@ async def aocr( api_key=prepared.api_key, api_base=prepared.api_base, custom_llm_provider=prepared.custom_llm_provider, + client=_supplied_client(kwargs), aocr=True, headers=prepared.extra_headers, provider_config=prepared.provider_config, @@ -404,6 +411,7 @@ def ocr( api_key=prepared.api_key, api_base=prepared.api_base, custom_llm_provider=prepared.custom_llm_provider, + client=_supplied_client(kwargs), aocr=_is_async, headers=prepared.extra_headers, provider_config=prepared.provider_config, diff --git a/tests/test_litellm/ocr/test_main.py b/tests/test_litellm/ocr/test_main.py index 5531a2639c0..32e5637ee09 100644 --- a/tests/test_litellm/ocr/test_main.py +++ b/tests/test_litellm/ocr/test_main.py @@ -113,6 +113,35 @@ async def test_python_request_response_and_callbacks( assert logger.log_pre_api_call.call_count == 1 +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +async def test_python_uses_the_supplied_client(provider: Mock, asynchronous: bool) -> None: + supplied: Final = Mock(return_value=provider.return_value) + transport: Final = httpx.MockTransport(supplied) + arguments: Final = { + "model": "mistral/mistral-ocr-latest", + "document": dict(PRICING_DOCUMENT), + "api_key": "test-key", + "api_base": "https://ocr.test/v1", + } + + async def call() -> OCRResponse: + if not asynchronous: + with httpx.Client(transport=transport) as sync_client: + return litellm.ocr(**arguments, client=HTTPHandler(client=sync_client)) + async with httpx.AsyncClient(transport=transport) as async_client: + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + handler.client = async_client + return await litellm.aocr(**arguments, client=handler) + + response: Final = await call() + assert response.pages[0].markdown == "parsed document" + assert supplied.call_count == 1 + assert str(supplied.call_args.args[0].url) == "https://ocr.test/v1/ocr" + assert provider.call_count == 0 + + @pytest.mark.asyncio @pytest.mark.parametrize("asynchronous", [False, True]) async def test_python_provider_errors_keep_public_exception(provider: Mock, asynchronous: bool) -> None: From 988676a0b81370c645f20a12547164b6eba40585 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Sat, 19 Sep 2026 00:30:43 +0000 Subject: [PATCH 05/15] test(rust): parse the settings contract through Python so the bridge keeps to the interop boundary Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../python-bridge/src/python_settings.rs | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/litellm-rust/crates/python-bridge/src/python_settings.rs b/litellm-rust/crates/python-bridge/src/python_settings.rs index 0c6554f0970..c5f9f309615 100644 --- a/litellm-rust/crates/python-bridge/src/python_settings.rs +++ b/litellm-rust/crates/python-bridge/src/python_settings.rs @@ -33,16 +33,32 @@ pub(crate) const CONTRACT: &str = include_str!("../python_settings.json"); #[cfg(test)] mod tests { - use std::collections::BTreeSet; + use std::{collections::BTreeSet, ffi::CString}; + + use pyo3::{prelude::*, types::PyDict}; use super::{CONTRACT, PythonSettings}; #[test] fn every_settings_group_is_in_the_python_contract() { - let contract: serde_json::Map = - serde_json::from_str(CONTRACT).unwrap(); - let declared: BTreeSet<&str> = contract.keys().map(String::as_str).collect(); - let read: BTreeSet<&str> = PythonSettings::ALL.map(PythonSettings::name).into(); - assert_eq!(read, declared); + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + locals.set_item("contract", CONTRACT).unwrap(); + let source = CString::new("import json\nkeys = list(json.loads(contract))").unwrap(); + py.run(&source, Some(&locals), Some(&locals)).unwrap(); + let declared: BTreeSet = locals + .get_item("keys") + .unwrap() + .unwrap() + .extract::>() + .unwrap() + .into_iter() + .collect(); + let read: BTreeSet = PythonSettings::ALL + .map(|group| group.name().to_owned()) + .into(); + assert_eq!(read, declared); + }); } } From a3aceec2f865b30c800b8ea9587582e13d6398ef Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Fri, 18 Sep 2026 17:37:08 -0700 Subject: [PATCH 06/15] fix(rust): match Python proxy, ssl_verify and client expiry behavior in the http pool Honor environment proxies whenever Python would use httpx (sync calls, HTTP/2, aiohttp disabled), apply the per-call ssl_verify argument, ignore empty or missing SSL env values the way http_handler.py does, expire pooled clients after an hour so rotated certificates reload, keep the client certificate off media downloads, and decline instead of raising when a litellm global has an unexpected type --- litellm-rust/crates/http/src/config.rs | 21 ++-- litellm-rust/crates/http/src/lib.rs | 4 - litellm-rust/crates/http/src/pool.rs | 80 +++++++++--- litellm-rust/crates/http/src/settings.rs | 82 ++++++++++-- .../llms/src/custom_httpx/llm_http_handler.rs | 2 +- .../crates/llms/src/custom_httpx/transport.rs | 6 - .../crates/python-bridge/python_settings.json | 1 + litellm-rust/crates/python-bridge/src/http.rs | 118 +++++++++++++++--- .../python-bridge/src/python_settings.rs | 6 - .../python-bridge/src/routes/ocr/mod.rs | 2 +- litellm/rust_bridge/settings.py | 8 +- .../test_litellm/rust_bridge/test_settings.py | 4 +- 12 files changed, 262 insertions(+), 72 deletions(-) diff --git a/litellm-rust/crates/http/src/config.rs b/litellm-rust/crates/http/src/config.rs index f27092c1fb5..7772e6cce5b 100644 --- a/litellm-rust/crates/http/src/config.rs +++ b/litellm-rust/crates/http/src/config.rs @@ -16,8 +16,6 @@ pub enum Verify { BuiltInRoots, } -/// One fully resolved client configuration. Every field is a plain value so the pool can -/// key cached clients on it. #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct HttpClientConfig { pub verify: Verify, @@ -30,9 +28,6 @@ pub struct HttpClientConfig { } impl HttpClientConfig { - /// Port of `get_ssl_verify` + `get_ssl_configuration`: the configured (environment-overlaid) - /// `ssl_verify`, then `SSL_CERT_FILE`, then the built-in roots. Settings rustls has no - /// equivalent for are an error instead of a silent no-op. pub fn resolve(settings: &HttpSettings) -> Result { if let Some(level) = &settings.ssl_security_level { return Err(Error::Unsupported { @@ -60,12 +55,11 @@ impl HttpClientConfig { force_ipv4: settings.force_ipv4, http2: settings.http2, user_agent: settings.user_agent.clone(), - trust_proxy_env: settings.trust_proxy_env, + trust_proxy_env: settings.trust_proxy_env || settings.http2 || settings.httpx_transport, connect_timeout: settings.connect_timeout, }) } - /// A builder carrying every shared setting; variants add their own policy on top. pub fn client_builder(&self) -> Result { let base = reqwest::Client::builder().connect_timeout(self.connect_timeout); let with_roots = match &self.verify { @@ -242,6 +236,19 @@ mod tests { ); } + #[rstest] + #[case::aiohttp_default(HttpSettings::default(), false)] + #[case::aiohttp_trust_env(HttpSettings { trust_proxy_env: true, ..HttpSettings::default() }, true)] + #[case::http2_uses_httpx(HttpSettings { http2: true, ..HttpSettings::default() }, true)] + #[case::aiohttp_disabled(HttpSettings { httpx_transport: true, ..HttpSettings::default() }, true)] + fn environment_proxies_apply_whenever_python_would_use_httpx( + #[case] settings: HttpSettings, + #[case] expected: bool, + ) { + let config = HttpClientConfig::resolve(&settings).unwrap(); + assert_eq!(config.trust_proxy_env, expected); + } + #[test] fn missing_ca_bundle_is_a_read_error() { let path = std::env::temp_dir().join("litellm-http-missing-bundle.pem"); diff --git a/litellm-rust/crates/http/src/lib.rs b/litellm-rust/crates/http/src/lib.rs index 9c88e3101a7..c02a82539ff 100644 --- a/litellm-rust/crates/http/src/lib.rs +++ b/litellm-rust/crates/http/src/lib.rs @@ -1,7 +1,3 @@ -//! Rust counterpart of `litellm/llms/custom_httpx/http_handler.py`: the plain HTTP settings -//! LiteLLM exposes, their resolution into one typed client configuration, and a pool that -//! caches `reqwest::Client`s per resolved configuration. - mod config; mod error; mod pool; diff --git a/litellm-rust/crates/http/src/pool.rs b/litellm-rust/crates/http/src/pool.rs index 613ce9c2831..0d9b1abf504 100644 --- a/litellm-rust/crates/http/src/pool.rs +++ b/litellm-rust/crates/http/src/pool.rs @@ -1,33 +1,44 @@ use std::{ collections::HashMap, sync::{Arc, Mutex, MutexGuard, PoisonError}, + time::{Duration, Instant}, }; use reqwest::dns::Resolve; use crate::{config::HttpClientConfig, error::Error}; -/// The client shapes routes need; each is the shared base plus one policy. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum ClientVariant { Provider, NoRedirect, - /// Media downloads: no redirects (the fetcher validates each hop), never a proxy, and the - /// pool's media resolver. Media, } -/// Counterpart of `get_async_httpx_client`: one `reqwest::Client` per resolved configuration -/// and variant, built on first use and shared afterwards. +const CLIENT_TTL: Duration = Duration::from_secs(3600); + +struct PooledClient { + client: reqwest::Client, + built_at: Instant, +} + +type Clients = HashMap<(HttpClientConfig, ClientVariant), PooledClient>; + pub struct HttpClientPool { media_resolver: Arc, - clients: Mutex>, + ttl: Duration, + clients: Mutex, } impl HttpClientPool { pub fn new(media_resolver: Arc) -> Self { + Self::with_ttl(media_resolver, CLIENT_TTL) + } + + pub fn with_ttl(media_resolver: Arc, ttl: Duration) -> Self { Self { media_resolver, + ttl, clients: Mutex::default(), } } @@ -37,15 +48,31 @@ impl HttpClientPool { config: &HttpClientConfig, variant: ClientVariant, ) -> Result { - let key = (config.clone(), variant); - if let Some(client) = self.lock().get(&key) { - return Ok(client.clone()); + let effective = match variant { + ClientVariant::Media => HttpClientConfig { + client_certificate: None, + ..config.clone() + }, + ClientVariant::Provider | ClientVariant::NoRedirect => config.clone(), + }; + let key = (effective, variant); + if let Some(pooled) = self.lock().get(&key) + && pooled.built_at.elapsed() < self.ttl + { + return Ok(pooled.client.clone()); } - let client = self.apply(variant, config.client_builder()?).build()?; - Ok(self.lock().entry(key).or_insert(client).clone()) + let client = self.apply(variant, key.0.client_builder()?).build()?; + self.lock().insert( + key, + PooledClient { + client: client.clone(), + built_at: Instant::now(), + }, + ); + Ok(client) } - fn lock(&self) -> MutexGuard<'_, HashMap<(HttpClientConfig, ClientVariant), reqwest::Client>> { + fn lock(&self) -> MutexGuard<'_, Clients> { self.clients.lock().unwrap_or_else(PoisonError::into_inner) } @@ -102,8 +129,6 @@ mod tests { } } - /// Answers every request on every connection with `status_line` and counts connections, - /// so a reused client shows up as a reused keep-alive connection. async fn serve( status_line: &'static str, ) -> (SocketAddr, Arc, Arc>>) { @@ -168,6 +193,33 @@ mod tests { assert_eq!(connections.load(Ordering::SeqCst), 3); } + #[tokio::test] + async fn expired_clients_are_rebuilt() { + let (address, connections, _) = serve("HTTP/1.1 204 No Content").await; + let url = format!("http://{address}"); + let pool = HttpClientPool::with_ttl( + Arc::new(FixedResolver(([192, 0, 2, 1], 80).into())), + Duration::ZERO, + ); + get(&pool, &config("a"), ClientVariant::Provider, &url).await; + get(&pool, &config("a"), ClientVariant::Provider, &url).await; + assert_eq!(connections.load(Ordering::SeqCst), 2); + } + + #[test] + fn media_variant_never_loads_the_client_certificate() { + let pool = pool(); + let with_identity = HttpClientConfig { + client_certificate: Some(std::env::temp_dir().join("litellm-http-absent-client.pem")), + ..config("a") + }; + assert!( + pool.client(&with_identity, ClientVariant::Provider) + .is_err() + ); + assert!(pool.client(&with_identity, ClientVariant::Media).is_ok()); + } + #[test] fn build_failures_are_not_cached() { let pool = pool(); diff --git a/litellm-rust/crates/http/src/settings.rs b/litellm-rust/crates/http/src/settings.rs index 45aab0d6fa5..55ac471bba9 100644 --- a/litellm-rust/crates/http/src/settings.rs +++ b/litellm-rust/crates/http/src/settings.rs @@ -1,6 +1,8 @@ -use std::{path::PathBuf, time::Duration}; +use std::{ + path::{Path, PathBuf}, + time::Duration, +}; -/// `litellm.ssl_verify` / `SSL_VERIFY`: a bool or a CA bundle path. #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub enum SslVerify { Enabled, @@ -18,7 +20,6 @@ impl SslVerify { } } -/// The plain inputs `http_handler.py` reads from `litellm.*` globals and the environment. #[derive(Clone, Debug, PartialEq, Eq)] pub struct HttpSettings { pub ssl_verify: Option, @@ -28,6 +29,7 @@ pub struct HttpSettings { pub ssl_ecdh_curve: Option, pub force_ipv4: bool, pub http2: bool, + pub httpx_transport: bool, pub user_agent: Option, pub trust_proxy_env: bool, pub connect_timeout: Duration, @@ -43,6 +45,7 @@ impl Default for HttpSettings { ssl_ecdh_curve: None, force_ipv4: false, http2: false, + httpx_transport: false, user_agent: None, trust_proxy_env: false, connect_timeout: Duration::from_secs(5), @@ -51,11 +54,6 @@ impl Default for HttpSettings { } impl HttpSettings { - /// Overlay the environment variables `http_handler.py` consults, with the same precedence: - /// `SSL_VERIFY`, `SSL_CERTIFICATE`, `SSL_SECURITY_LEVEL`, `SSL_ECDH_CURVE` and - /// `LITELLM_USER_AGENT` win over the configured value; `SSL_CERT_FILE` only applies when - /// verification is on without an explicit bundle; `LITELLM_HTTP2` and `AIOHTTP_TRUST_ENV` - /// can only turn their switch on. pub fn with_environment(self, env: &(dyn Fn(&str) -> Option + Sync)) -> Self { let enabled = |name: &str| env(name).is_some_and(|value| value.trim().eq_ignore_ascii_case("true")); @@ -68,15 +66,32 @@ impl HttpSettings { .or(self.ssl_cert_file), ssl_certificate: env("SSL_CERTIFICATE") .map(PathBuf::from) - .or(self.ssl_certificate), - ssl_security_level: env("SSL_SECURITY_LEVEL").or(self.ssl_security_level), - ssl_ecdh_curve: env("SSL_ECDH_CURVE").or(self.ssl_ecdh_curve), + .or(self.ssl_certificate) + .filter(|path| !path.as_os_str().is_empty()), + ssl_security_level: env("SSL_SECURITY_LEVEL") + .or(self.ssl_security_level) + .filter(|level| !level.is_empty()), + ssl_ecdh_curve: env("SSL_ECDH_CURVE") + .or(self.ssl_ecdh_curve) + .filter(|curve| !curve.is_empty()), http2: self.http2 || enabled("LITELLM_HTTP2"), + httpx_transport: self.httpx_transport || enabled("DISABLE_AIOHTTP_TRANSPORT"), user_agent: env("LITELLM_USER_AGENT").or(self.user_agent), trust_proxy_env: self.trust_proxy_env || enabled("AIOHTTP_TRUST_ENV"), ..self } } + + pub fn without_missing_files(self, exists: &dyn Fn(&Path) -> bool) -> Self { + Self { + ssl_verify: match self.ssl_verify { + Some(SslVerify::CaBundle(path)) if !exists(&path) => Some(SslVerify::Enabled), + other => other, + }, + ssl_cert_file: self.ssl_cert_file.filter(|path| exists(path)), + ..self + } + } } #[cfg(test)] @@ -152,6 +167,46 @@ mod tests { assert_eq!(configured.clone().with_environment(&no_env), configured); } + #[test] + fn empty_environment_values_clear_the_setting_like_python_truthiness() { + let settings = HttpSettings { + ssl_certificate: Some("/configured/client.pem".into()), + ssl_security_level: Some("configured".into()), + ssl_ecdh_curve: Some("X25519".into()), + ..HttpSettings::default() + } + .with_environment(&env_of(&[ + ("SSL_CERTIFICATE", ""), + ("SSL_SECURITY_LEVEL", ""), + ("SSL_ECDH_CURVE", ""), + ])); + assert_eq!(settings.ssl_certificate, None); + assert_eq!(settings.ssl_security_level, None); + assert_eq!(settings.ssl_ecdh_curve, None); + } + + #[test] + fn missing_files_fall_back_to_default_verification() { + let settings = HttpSettings { + ssl_verify: Some(SslVerify::CaBundle("/absent/roots.pem".into())), + ssl_cert_file: Some("/absent/env.pem".into()), + ..HttpSettings::default() + } + .without_missing_files(&|_| false); + assert_eq!(settings.ssl_verify, Some(SslVerify::Enabled)); + assert_eq!(settings.ssl_cert_file, None); + } + + #[test] + fn existing_files_are_kept() { + let settings = HttpSettings { + ssl_verify: Some(SslVerify::CaBundle("/present/roots.pem".into())), + ssl_cert_file: Some("/present/env.pem".into()), + ..HttpSettings::default() + }; + assert_eq!(settings.clone().without_missing_files(&|_| true), settings); + } + #[rstest] #[case("true", true)] #[case("True", true)] @@ -159,11 +214,14 @@ mod tests { #[case("1", false)] fn boolean_switches_only_turn_on_for_true(#[case] value: &'static str, #[case] expected: bool) { let env = move |name: &str| match name { - "LITELLM_HTTP2" | "AIOHTTP_TRUST_ENV" => Some(value.to_string()), + "LITELLM_HTTP2" | "AIOHTTP_TRUST_ENV" | "DISABLE_AIOHTTP_TRANSPORT" => { + Some(value.to_string()) + } _ => None, }; let settings = HttpSettings::default().with_environment(&env); assert_eq!(settings.http2, expected); + assert_eq!(settings.httpx_transport, expected); assert_eq!(settings.trust_proxy_env, expected); } } diff --git a/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs b/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs index 425b71efc78..876fa0aae87 100644 --- a/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs +++ b/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs @@ -42,7 +42,7 @@ impl OcrClient { pool: &HttpClientPool, config: &HttpClientConfig, vertex_auth: VertexAuth, - ) -> Result { + ) -> Result { Ok(Self { provider_http: pool.client(config, ClientVariant::Provider)?, polling_http: pool.client(config, ClientVariant::NoRedirect)?, diff --git a/litellm-rust/crates/llms/src/custom_httpx/transport.rs b/litellm-rust/crates/llms/src/custom_httpx/transport.rs index 8e5e1a8832d..172dd96476a 100644 --- a/litellm-rust/crates/llms/src/custom_httpx/transport.rs +++ b/litellm-rust/crates/llms/src/custom_httpx/transport.rs @@ -26,12 +26,6 @@ impl From for Error { } } -impl From for Error { - fn from(error: litellm_http::Error) -> Self { - Self::Connect(error.to_string()) - } -} - #[cfg(test)] mod tests { #[tokio::test] diff --git a/litellm-rust/crates/python-bridge/python_settings.json b/litellm-rust/crates/python-bridge/python_settings.json index a6cd959c6de..64dd01a0a84 100644 --- a/litellm-rust/crates/python-bridge/python_settings.json +++ b/litellm-rust/crates/python-bridge/python_settings.json @@ -7,6 +7,7 @@ "force_ipv4", "http2", "aiohttp_trust_env", + "disable_aiohttp_transport", "user_agent" ] } diff --git a/litellm-rust/crates/python-bridge/src/http.rs b/litellm-rust/crates/python-bridge/src/http.rs index de6fb5bb96d..cf7ca05515e 100644 --- a/litellm-rust/crates/python-bridge/src/http.rs +++ b/litellm-rust/crates/python-bridge/src/http.rs @@ -1,5 +1,5 @@ use std::{ - path::PathBuf, + path::{Path, PathBuf}, sync::{Arc, LazyLock}, }; @@ -12,27 +12,46 @@ use crate::{errors::RustBridgeDeclined, python_settings::PythonSettings}; static POOL: LazyLock = LazyLock::new(|| HttpClientPool::new(Arc::new(PublicDnsResolver))); -/// Keyword arguments that carry a live Python HTTP client or session. They cannot cross into -/// Rust, so a call that supplies one stays on the Python path. const LIVE_CLIENT_ARGUMENTS: [&str; 3] = ["client", "shared_session", "aclient_session"]; pub(crate) fn pool() -> &'static HttpClientPool { &POOL } -/// The client configuration for one call: the `litellm.*` HTTP settings with the environment -/// overlaid, the same way `http_handler.py` combines them. pub(crate) fn call_config( py: Python<'_>, kwargs: &Bound<'_, PyDict>, + asynchronous: bool, ) -> PyResult { decline_live_clients(kwargs)?; - let settings = settings(&PythonSettings::Http.read(py)?)? + let configured = settings(&PythonSettings::Http.read(py)?)? .with_environment(&|name| std::env::var(name).ok()); + let settings = for_call(configured, call_ssl_verify(kwargs)?, asynchronous) + .without_missing_files(&|path: &Path| path.exists()); HttpClientConfig::resolve(&settings) .map_err(|error| RustBridgeDeclined::new_err(error.to_string())) } +fn call_ssl_verify(kwargs: &Bound<'_, PyDict>) -> PyResult> { + kwargs + .get_item("ssl_verify")? + .filter(|value| !value.is_none()) + .map(|value| ssl_verify(&value, "the ssl_verify argument")) + .transpose() +} + +fn for_call( + configured: HttpSettings, + call_ssl_verify: Option, + asynchronous: bool, +) -> HttpSettings { + HttpSettings { + ssl_verify: call_ssl_verify.or(configured.ssl_verify), + httpx_transport: configured.httpx_transport || !asynchronous, + ..configured + } +} + pub(crate) fn decline_live_clients(kwargs: &Bound<'_, PyDict>) -> PyResult<()> { for name in LIVE_CLIENT_ARGUMENTS { if kwargs.get_item(name)?.is_some_and(|value| !value.is_none()) { @@ -53,25 +72,31 @@ struct PythonHttpSettings<'py> { force_ipv4: bool, http2: bool, aiohttp_trust_env: bool, + disable_aiohttp_transport: bool, user_agent: String, } fn settings(value: &Bound<'_, PyAny>) -> PyResult { - let python: PythonHttpSettings = value.extract()?; + let python: PythonHttpSettings = value.extract().map_err(|error: PyErr| { + RustBridgeDeclined::new_err(format!( + "litellm HTTP settings cannot be used by the Rust route: {error}" + )) + })?; Ok(HttpSettings { - ssl_verify: Some(ssl_verify(&python.ssl_verify)?), + ssl_verify: Some(ssl_verify(&python.ssl_verify, "litellm.ssl_verify")?), ssl_certificate: python.ssl_certificate.map(PathBuf::from), ssl_security_level: python.ssl_security_level, ssl_ecdh_curve: python.ssl_ecdh_curve, force_ipv4: python.force_ipv4, http2: python.http2, + httpx_transport: python.disable_aiohttp_transport, user_agent: Some(python.user_agent), trust_proxy_env: python.aiohttp_trust_env, ..HttpSettings::default() }) } -fn ssl_verify(value: &Bound<'_, PyAny>) -> PyResult { +fn ssl_verify(value: &Bound<'_, PyAny>, source: &str) -> PyResult { if let Ok(enabled) = value.extract::() { return Ok(if enabled { SslVerify::Enabled @@ -82,9 +107,9 @@ fn ssl_verify(value: &Bound<'_, PyAny>) -> PyResult { if let Ok(path) = value.extract::() { return Ok(SslVerify::parse(&path)); } - Err(RustBridgeDeclined::new_err( - "litellm.ssl_verify is a live Python object and cannot be used by the Rust route", - )) + Err(RustBridgeDeclined::new_err(format!( + "{source} is a live Python object and cannot be used by the Rust route" + ))) } #[cfg(test)] @@ -95,8 +120,6 @@ mod tests { use super::*; use crate::python_settings::CONTRACT; - /// A stand-in for `http_settings()` carrying exactly the fields the contract declares, so a - /// field Rust reads but Python does not return fails here. fn python_settings<'py>(py: Python<'py>, overrides: &str) -> Bound<'py, PyAny> { let source = format!( " @@ -110,6 +133,7 @@ defaults = dict( force_ipv4=False, http2=False, aiohttp_trust_env=False, + disable_aiohttp_transport=False, user_agent='litellm/test', ) defaults.update(dict({overrides})) @@ -153,6 +177,7 @@ ssl_ecdh_curve='X25519', force_ipv4=True, http2=True, aiohttp_trust_env=True, +disable_aiohttp_transport=True, user_agent='litellm/9.9.9', ", )) @@ -166,6 +191,7 @@ user_agent='litellm/9.9.9', ssl_ecdh_curve: Some("X25519".into()), force_ipv4: true, http2: true, + httpx_transport: true, user_agent: Some("litellm/9.9.9".into()), trust_proxy_env: true, ..HttpSettings::default() @@ -214,6 +240,70 @@ user_agent='litellm/9.9.9', }); } + #[test] + fn mistyped_python_settings_decline_instead_of_raising() { + Python::initialize(); + Python::attach(|py| { + let error = settings(&python_settings(py, "force_ipv4='yes'")).unwrap_err(); + assert!(error.is_instance_of::(py)); + }); + } + + #[test] + fn call_ssl_verify_beats_the_configured_and_environment_value() { + Python::initialize(); + Python::attach(|py| { + let kwargs = PyDict::new(py); + kwargs.set_item("ssl_verify", false).unwrap(); + let configured = HttpSettings { + ssl_verify: Some(SslVerify::Enabled), + ..HttpSettings::default() + }; + let settings = for_call(configured, call_ssl_verify(&kwargs).unwrap(), true); + assert_eq!(settings.ssl_verify, Some(SslVerify::Disabled)); + }); + } + + #[test] + fn absent_call_ssl_verify_keeps_the_configured_value() { + Python::initialize(); + Python::attach(|py| { + let kwargs = PyDict::new(py); + kwargs.set_item("ssl_verify", py.None()).unwrap(); + let configured = HttpSettings { + ssl_verify: Some(SslVerify::Disabled), + ..HttpSettings::default() + }; + let settings = for_call(configured.clone(), call_ssl_verify(&kwargs).unwrap(), true); + assert_eq!(settings, configured); + }); + } + + #[test] + fn live_ssl_context_argument_declines() { + Python::initialize(); + Python::attach(|py| { + let kwargs = PyDict::new(py); + kwargs + .set_item("ssl_verify", py.eval(c"object()", None, None).unwrap()) + .unwrap(); + let error = call_ssl_verify(&kwargs).unwrap_err(); + assert!(error.is_instance_of::(py)); + }); + } + + #[rstest] + #[case::asynchronous(true, false)] + #[case::synchronous(false, true)] + fn synchronous_calls_honor_environment_proxies_like_httpx( + #[case] asynchronous: bool, + #[case] expected: bool, + ) { + let settings = for_call(HttpSettings::default(), None, asynchronous); + let config = HttpClientConfig::resolve(&settings).unwrap(); + assert_eq!(config.trust_proxy_env, expected); + } + #[rstest] #[case::client("client")] #[case::shared_session("shared_session")] diff --git a/litellm-rust/crates/python-bridge/src/python_settings.rs b/litellm-rust/crates/python-bridge/src/python_settings.rs index c5f9f309615..dcb46e7d2b5 100644 --- a/litellm-rust/crates/python-bridge/src/python_settings.rs +++ b/litellm-rust/crates/python-bridge/src/python_settings.rs @@ -2,12 +2,6 @@ use pyo3::prelude::*; const MODULE: &str = "litellm.rust_bridge.settings"; -/// Every group of `litellm.*` module globals the native routes read. Environment overrides are -/// applied on the Rust side, so each function returns only what the Python process configured. -/// A group is deleted once Rust owns loading that configuration, so this enum only shrinks. -/// -/// `litellm/rust_bridge/settings.py` is the only Python module behind it, and -/// `python_settings.json` pins the fields each function returns on both sides. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum PythonSettings { Http, diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs index d9aeeb234f7..174d0ff18c8 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs @@ -37,7 +37,7 @@ fn run_ocr( kwargs: Bound<'_, PyDict>, asynchronous: bool, ) -> PyResult> { - let config = http::call_config(py, &kwargs)?; + let config = http::call_config(py, &kwargs, asynchronous)?; let client = OcrClient::new(http::pool(), &config, VERTEX_AUTH.clone()) .map_err(|error| RustBridgeDeclined::new_err(error.to_string()))?; run_legacy_call( diff --git a/litellm/rust_bridge/settings.py b/litellm/rust_bridge/settings.py index a8229b12d13..ad478fb28b5 100644 --- a/litellm/rust_bridge/settings.py +++ b/litellm/rust_bridge/settings.py @@ -1,9 +1,3 @@ -"""The `litellm.*` module globals the native routes read. - -Environment variables that override these are applied in Rust, so nothing here reads `os.environ`. -`litellm-rust/crates/python-bridge/python_settings.json` pins the fields each function returns. -""" - from __future__ import annotations from dataclasses import dataclass @@ -18,6 +12,7 @@ class HttpSettings: force_ipv4: bool http2: bool aiohttp_trust_env: bool + disable_aiohttp_transport: bool user_agent: str @@ -33,5 +28,6 @@ def http_settings() -> HttpSettings: force_ipv4=litellm.force_ipv4, http2=litellm.http2, aiohttp_trust_env=litellm.aiohttp_trust_env, + disable_aiohttp_transport=litellm.disable_aiohttp_transport, user_agent=default_user_agent(), ) diff --git a/tests/test_litellm/rust_bridge/test_settings.py b/tests/test_litellm/rust_bridge/test_settings.py index 618fa400136..dce2324de08 100644 --- a/tests/test_litellm/rust_bridge/test_settings.py +++ b/tests/test_litellm/rust_bridge/test_settings.py @@ -26,6 +26,7 @@ def test_http_settings_reads_the_litellm_globals(monkeypatch: pytest.MonkeyPatch monkeypatch.setattr(litellm, "force_ipv4", True) monkeypatch.setattr(litellm, "http2", True) monkeypatch.setattr(litellm, "aiohttp_trust_env", True) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) assert settings.http_settings() == settings.HttpSettings( ssl_verify="/etc/ssl/corp.pem", @@ -35,7 +36,8 @@ def test_http_settings_reads_the_litellm_globals(monkeypatch: pytest.MonkeyPatch force_ipv4=True, http2=True, aiohttp_trust_env=True, - user_agent=settings.http_settings().user_agent, + disable_aiohttp_transport=True, + user_agent=default_user_agent(), ) From 8d2476465fb4b6a84110ec5e9a264602c0e6e6d1 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Fri, 18 Sep 2026 17:58:07 -0700 Subject: [PATCH 07/15] fix(rust): honor environment proxies by default and name the cause in transport errors Python's aiohttp transport reads HTTP(S)_PROXY on every request unless disable_aiohttp_trust_env is set, so the Rust clients now do the same instead of requiring aiohttp_trust_env. Transport error messages include reqwest's source chain, so a rejected certificate or refused connection is no longer reported as just 'error sending request' --- litellm-rust/crates/http/src/config.rs | 25 +++++++++--- litellm-rust/crates/http/src/settings.rs | 11 ++++-- .../crates/llms/src/custom_httpx/transport.rs | 38 ++++++++++++++++++- .../crates/python-bridge/python_settings.json | 1 + litellm-rust/crates/python-bridge/src/http.rs | 13 ++++++- litellm/rust_bridge/settings.py | 2 + .../test_litellm/rust_bridge/test_settings.py | 2 + 7 files changed, 79 insertions(+), 13 deletions(-) diff --git a/litellm-rust/crates/http/src/config.rs b/litellm-rust/crates/http/src/config.rs index 7772e6cce5b..24a52315f7c 100644 --- a/litellm-rust/crates/http/src/config.rs +++ b/litellm-rust/crates/http/src/config.rs @@ -55,7 +55,10 @@ impl HttpClientConfig { force_ipv4: settings.force_ipv4, http2: settings.http2, user_agent: settings.user_agent.clone(), - trust_proxy_env: settings.trust_proxy_env || settings.http2 || settings.httpx_transport, + trust_proxy_env: !settings.ignore_proxy_env + || settings.trust_proxy_env + || settings.http2 + || settings.httpx_transport, connect_timeout: settings.connect_timeout, }) } @@ -237,11 +240,21 @@ mod tests { } #[rstest] - #[case::aiohttp_default(HttpSettings::default(), false)] - #[case::aiohttp_trust_env(HttpSettings { trust_proxy_env: true, ..HttpSettings::default() }, true)] - #[case::http2_uses_httpx(HttpSettings { http2: true, ..HttpSettings::default() }, true)] - #[case::aiohttp_disabled(HttpSettings { httpx_transport: true, ..HttpSettings::default() }, true)] - fn environment_proxies_apply_whenever_python_would_use_httpx( + #[case::aiohttp_default(HttpSettings::default(), true)] + #[case::aiohttp_opted_out(HttpSettings { ignore_proxy_env: true, ..HttpSettings::default() }, false)] + #[case::session_trust_env_beats_opt_out( + HttpSettings { ignore_proxy_env: true, trust_proxy_env: true, ..HttpSettings::default() }, + true + )] + #[case::http2_uses_httpx( + HttpSettings { ignore_proxy_env: true, http2: true, ..HttpSettings::default() }, + true + )] + #[case::aiohttp_disabled( + HttpSettings { ignore_proxy_env: true, httpx_transport: true, ..HttpSettings::default() }, + true + )] + fn environment_proxies_apply_unless_the_aiohttp_transport_opts_out( #[case] settings: HttpSettings, #[case] expected: bool, ) { diff --git a/litellm-rust/crates/http/src/settings.rs b/litellm-rust/crates/http/src/settings.rs index 55ac471bba9..c572c56ef3a 100644 --- a/litellm-rust/crates/http/src/settings.rs +++ b/litellm-rust/crates/http/src/settings.rs @@ -32,6 +32,7 @@ pub struct HttpSettings { pub httpx_transport: bool, pub user_agent: Option, pub trust_proxy_env: bool, + pub ignore_proxy_env: bool, pub connect_timeout: Duration, } @@ -48,6 +49,7 @@ impl Default for HttpSettings { httpx_transport: false, user_agent: None, trust_proxy_env: false, + ignore_proxy_env: false, connect_timeout: Duration::from_secs(5), } } @@ -78,6 +80,7 @@ impl HttpSettings { httpx_transport: self.httpx_transport || enabled("DISABLE_AIOHTTP_TRANSPORT"), user_agent: env("LITELLM_USER_AGENT").or(self.user_agent), trust_proxy_env: self.trust_proxy_env || enabled("AIOHTTP_TRUST_ENV"), + ignore_proxy_env: self.ignore_proxy_env || enabled("DISABLE_AIOHTTP_TRUST_ENV"), ..self } } @@ -214,14 +217,16 @@ mod tests { #[case("1", false)] fn boolean_switches_only_turn_on_for_true(#[case] value: &'static str, #[case] expected: bool) { let env = move |name: &str| match name { - "LITELLM_HTTP2" | "AIOHTTP_TRUST_ENV" | "DISABLE_AIOHTTP_TRANSPORT" => { - Some(value.to_string()) - } + "LITELLM_HTTP2" + | "AIOHTTP_TRUST_ENV" + | "DISABLE_AIOHTTP_TRANSPORT" + | "DISABLE_AIOHTTP_TRUST_ENV" => Some(value.to_string()), _ => None, }; let settings = HttpSettings::default().with_environment(&env); assert_eq!(settings.http2, expected); assert_eq!(settings.httpx_transport, expected); assert_eq!(settings.trust_proxy_env, expected); + assert_eq!(settings.ignore_proxy_env, expected); } } diff --git a/litellm-rust/crates/llms/src/custom_httpx/transport.rs b/litellm-rust/crates/llms/src/custom_httpx/transport.rs index 172dd96476a..c42cdf410f6 100644 --- a/litellm-rust/crates/llms/src/custom_httpx/transport.rs +++ b/litellm-rust/crates/llms/src/custom_httpx/transport.rs @@ -11,7 +11,7 @@ pub enum Error { impl Error { pub fn from_reqwest_before_dispatch(error: reqwest::Error) -> Self { let before_dispatch = !error.is_timeout() && (error.is_connect() || error.is_builder()); - let message = error.without_url().to_string(); + let message = describe(error); if before_dispatch { Self::Connect(message) } else { @@ -22,10 +22,18 @@ impl Error { impl From for Error { fn from(error: reqwest::Error) -> Self { - Self::Network(error.without_url().to_string()) + Self::Network(describe(error)) } } +fn describe(error: reqwest::Error) -> String { + let error = error.without_url(); + std::iter::successors(std::error::Error::source(&error), |cause| cause.source()) + .fold(error.to_string(), |message, cause| { + format!("{message}: {cause}") + }) +} + #[cfg(test)] mod tests { #[tokio::test] @@ -47,6 +55,32 @@ mod tests { assert!(!error.to_string().contains("private")); } + fn root_cause(error: &dyn std::error::Error) -> Option { + match error.source() { + Some(cause) => root_cause(cause).or_else(|| Some(cause.to_string())), + None => None, + } + } + + #[tokio::test] + async fn network_error_message_names_the_underlying_cause() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind"); + let address = listener.local_addr().expect("address"); + drop(listener); + let error = reqwest::Client::builder() + .no_proxy() + .build() + .expect("client") + .get(format!("http://{address}/private?api_key=secret")) + .send() + .await + .expect_err("nothing listens on the port"); + let root_cause = root_cause(&error).expect("reqwest reports a cause"); + let message = crate::custom_httpx::transport::Error::from(error).to_string(); + assert!(message.contains(&root_cause), "{message}"); + assert!(!message.contains("secret")); + } + #[tokio::test] async fn request_timeout_is_not_safe_to_retry_as_a_connect_failure() { use std::time::Duration; diff --git a/litellm-rust/crates/python-bridge/python_settings.json b/litellm-rust/crates/python-bridge/python_settings.json index 64dd01a0a84..40e36a900d3 100644 --- a/litellm-rust/crates/python-bridge/python_settings.json +++ b/litellm-rust/crates/python-bridge/python_settings.json @@ -7,6 +7,7 @@ "force_ipv4", "http2", "aiohttp_trust_env", + "disable_aiohttp_trust_env", "disable_aiohttp_transport", "user_agent" ] diff --git a/litellm-rust/crates/python-bridge/src/http.rs b/litellm-rust/crates/python-bridge/src/http.rs index cf7ca05515e..2ab6517b61d 100644 --- a/litellm-rust/crates/python-bridge/src/http.rs +++ b/litellm-rust/crates/python-bridge/src/http.rs @@ -72,6 +72,7 @@ struct PythonHttpSettings<'py> { force_ipv4: bool, http2: bool, aiohttp_trust_env: bool, + disable_aiohttp_trust_env: bool, disable_aiohttp_transport: bool, user_agent: String, } @@ -92,6 +93,7 @@ fn settings(value: &Bound<'_, PyAny>) -> PyResult { httpx_transport: python.disable_aiohttp_transport, user_agent: Some(python.user_agent), trust_proxy_env: python.aiohttp_trust_env, + ignore_proxy_env: python.disable_aiohttp_trust_env, ..HttpSettings::default() }) } @@ -133,6 +135,7 @@ defaults = dict( force_ipv4=False, http2=False, aiohttp_trust_env=False, + disable_aiohttp_trust_env=False, disable_aiohttp_transport=False, user_agent='litellm/test', ) @@ -177,6 +180,7 @@ ssl_ecdh_curve='X25519', force_ipv4=True, http2=True, aiohttp_trust_env=True, +disable_aiohttp_trust_env=True, disable_aiohttp_transport=True, user_agent='litellm/9.9.9', ", @@ -194,6 +198,7 @@ user_agent='litellm/9.9.9', httpx_transport: true, user_agent: Some("litellm/9.9.9".into()), trust_proxy_env: true, + ignore_proxy_env: true, ..HttpSettings::default() } ); @@ -295,11 +300,15 @@ user_agent='litellm/9.9.9', #[rstest] #[case::asynchronous(true, false)] #[case::synchronous(false, true)] - fn synchronous_calls_honor_environment_proxies_like_httpx( + fn synchronous_calls_honor_environment_proxies_even_when_aiohttp_opts_out( #[case] asynchronous: bool, #[case] expected: bool, ) { - let settings = for_call(HttpSettings::default(), None, asynchronous); + let opted_out = HttpSettings { + ignore_proxy_env: true, + ..HttpSettings::default() + }; + let settings = for_call(opted_out, None, asynchronous); let config = HttpClientConfig::resolve(&settings).unwrap(); assert_eq!(config.trust_proxy_env, expected); } diff --git a/litellm/rust_bridge/settings.py b/litellm/rust_bridge/settings.py index ad478fb28b5..491312c97b6 100644 --- a/litellm/rust_bridge/settings.py +++ b/litellm/rust_bridge/settings.py @@ -12,6 +12,7 @@ class HttpSettings: force_ipv4: bool http2: bool aiohttp_trust_env: bool + disable_aiohttp_trust_env: bool disable_aiohttp_transport: bool user_agent: str @@ -28,6 +29,7 @@ def http_settings() -> HttpSettings: force_ipv4=litellm.force_ipv4, http2=litellm.http2, aiohttp_trust_env=litellm.aiohttp_trust_env, + disable_aiohttp_trust_env=litellm.disable_aiohttp_trust_env, disable_aiohttp_transport=litellm.disable_aiohttp_transport, user_agent=default_user_agent(), ) diff --git a/tests/test_litellm/rust_bridge/test_settings.py b/tests/test_litellm/rust_bridge/test_settings.py index dce2324de08..f4f9cbc8eec 100644 --- a/tests/test_litellm/rust_bridge/test_settings.py +++ b/tests/test_litellm/rust_bridge/test_settings.py @@ -26,6 +26,7 @@ def test_http_settings_reads_the_litellm_globals(monkeypatch: pytest.MonkeyPatch monkeypatch.setattr(litellm, "force_ipv4", True) monkeypatch.setattr(litellm, "http2", True) monkeypatch.setattr(litellm, "aiohttp_trust_env", True) + monkeypatch.setattr(litellm, "disable_aiohttp_trust_env", True) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) assert settings.http_settings() == settings.HttpSettings( @@ -36,6 +37,7 @@ def test_http_settings_reads_the_litellm_globals(monkeypatch: pytest.MonkeyPatch force_ipv4=True, http2=True, aiohttp_trust_env=True, + disable_aiohttp_trust_env=True, disable_aiohttp_transport=True, user_agent=default_user_agent(), ) From 157fa589478f37c0e5fbcf4c92b933bfaddfa4b8 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Fri, 18 Sep 2026 18:09:14 -0700 Subject: [PATCH 08/15] fix(rust): leave calls with a custom URL policy on the Python route litellm.user_url_validation and litellm.user_url_allowed_hosts are only implemented by the Python document fetcher, so an allowlisted internal document was rejected by the Rust route's network policy. The bridge now declines when either is changed from its default --- .../crates/python-bridge/python_settings.json | 4 ++ litellm-rust/crates/python-bridge/src/http.rs | 49 +++++++++++++++++++ .../python-bridge/src/python_settings.rs | 4 +- litellm/rust_bridge/settings.py | 16 ++++++ .../test_litellm/rust_bridge/test_settings.py | 15 +++++- 5 files changed, 86 insertions(+), 2 deletions(-) diff --git a/litellm-rust/crates/python-bridge/python_settings.json b/litellm-rust/crates/python-bridge/python_settings.json index 40e36a900d3..a6f5ee9c6f4 100644 --- a/litellm-rust/crates/python-bridge/python_settings.json +++ b/litellm-rust/crates/python-bridge/python_settings.json @@ -10,5 +10,9 @@ "disable_aiohttp_trust_env", "disable_aiohttp_transport", "user_agent" + ], + "url_policy": [ + "user_url_validation", + "user_url_allowed_hosts" ] } diff --git a/litellm-rust/crates/python-bridge/src/http.rs b/litellm-rust/crates/python-bridge/src/http.rs index 2ab6517b61d..77542855fec 100644 --- a/litellm-rust/crates/python-bridge/src/http.rs +++ b/litellm-rust/crates/python-bridge/src/http.rs @@ -24,6 +24,7 @@ pub(crate) fn call_config( asynchronous: bool, ) -> PyResult { decline_live_clients(kwargs)?; + decline_custom_url_policy(&PythonSettings::UrlPolicy.read(py)?)?; let configured = settings(&PythonSettings::Http.read(py)?)? .with_environment(&|name| std::env::var(name).ok()); let settings = for_call(configured, call_ssl_verify(kwargs)?, asynchronous) @@ -63,6 +64,23 @@ pub(crate) fn decline_live_clients(kwargs: &Bound<'_, PyDict>) -> PyResult<()> { Ok(()) } +#[derive(FromPyObject)] +struct PythonUrlPolicy { + user_url_validation: bool, + user_url_allowed_hosts: Vec, +} + +fn decline_custom_url_policy(value: &Bound<'_, PyAny>) -> PyResult<()> { + match value.extract::() { + Ok(policy) if policy.user_url_validation && policy.user_url_allowed_hosts.is_empty() => { + Ok(()) + } + Ok(_) | Err(_) => Err(RustBridgeDeclined::new_err( + "litellm.user_url_validation / user_url_allowed_hosts are applied by the Python route", + )), + } +} + #[derive(FromPyObject)] struct PythonHttpSettings<'py> { ssl_verify: Bound<'py, PyAny>, @@ -245,6 +263,37 @@ user_agent='litellm/9.9.9', }); } + fn url_policy<'py>(py: Python<'py>, fields: &str) -> Bound<'py, PyAny> { + let source = std::ffi::CString::new(format!( + "import types\npolicy = types.SimpleNamespace({fields})" + )) + .unwrap(); + let locals = PyDict::new(py); + py.run(&source, Some(&locals), Some(&locals)).unwrap(); + locals.get_item("policy").unwrap().unwrap() + } + + #[test] + fn default_url_policy_stays_on_the_rust_route() { + Python::initialize(); + Python::attach(|py| { + let policy = url_policy(py, "user_url_validation=True, user_url_allowed_hosts=[]"); + decline_custom_url_policy(&policy).unwrap(); + }); + } + + #[rstest] + #[case::validation_off("user_url_validation=False, user_url_allowed_hosts=[]")] + #[case::allowlist("user_url_validation=True, user_url_allowed_hosts=['docs.internal']")] + #[case::mistyped("user_url_validation=True, user_url_allowed_hosts=None")] + fn custom_url_policy_declines_so_python_applies_it(#[case] fields: &str) { + Python::initialize(); + Python::attach(|py| { + let error = decline_custom_url_policy(&url_policy(py, fields)).unwrap_err(); + assert!(error.is_instance_of::(py)); + }); + } + #[test] fn mistyped_python_settings_decline_instead_of_raising() { Python::initialize(); diff --git a/litellm-rust/crates/python-bridge/src/python_settings.rs b/litellm-rust/crates/python-bridge/src/python_settings.rs index dcb46e7d2b5..b7855566850 100644 --- a/litellm-rust/crates/python-bridge/src/python_settings.rs +++ b/litellm-rust/crates/python-bridge/src/python_settings.rs @@ -5,15 +5,17 @@ const MODULE: &str = "litellm.rust_bridge.settings"; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum PythonSettings { Http, + UrlPolicy, } impl PythonSettings { #[cfg(test)] - pub(crate) const ALL: [Self; 1] = [Self::Http]; + pub(crate) const ALL: [Self; 2] = [Self::Http, Self::UrlPolicy]; pub(crate) fn name(self) -> &'static str { match self { Self::Http => "http_settings", + Self::UrlPolicy => "url_policy", } } diff --git a/litellm/rust_bridge/settings.py b/litellm/rust_bridge/settings.py index 491312c97b6..bccfd01ec73 100644 --- a/litellm/rust_bridge/settings.py +++ b/litellm/rust_bridge/settings.py @@ -1,5 +1,6 @@ from __future__ import annotations +from collections.abc import Sequence from dataclasses import dataclass @@ -17,6 +18,21 @@ class HttpSettings: user_agent: str +@dataclass(frozen=True, slots=True) +class UrlPolicy: + user_url_validation: bool + user_url_allowed_hosts: Sequence[str] + + +def url_policy() -> UrlPolicy: + import litellm + + return UrlPolicy( + user_url_validation=litellm.user_url_validation, + user_url_allowed_hosts=litellm.user_url_allowed_hosts, + ) + + def http_settings() -> HttpSettings: import litellm from litellm.llms.custom_httpx.http_handler import default_user_agent diff --git a/tests/test_litellm/rust_bridge/test_settings.py b/tests/test_litellm/rust_bridge/test_settings.py index f4f9cbc8eec..7e7b1c6743b 100644 --- a/tests/test_litellm/rust_bridge/test_settings.py +++ b/tests/test_litellm/rust_bridge/test_settings.py @@ -15,7 +15,20 @@ CONTRACT_PATH: Final = Path(__file__).parents[3] / "litellm-rust/crates/python-b def test_the_rust_contract_matches_the_returned_fields() -> None: contract: Final = TypeAdapter(dict[str, list[str]]).validate_json(CONTRACT_PATH.read_text()) - assert contract == {"http_settings": [field.name for field in dataclasses.fields(settings.http_settings())]} + assert contract == { + "http_settings": [field.name for field in dataclasses.fields(settings.http_settings())], + "url_policy": [field.name for field in dataclasses.fields(settings.url_policy())], + } + + +def test_url_policy_reads_the_litellm_globals(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "user_url_validation", False) + monkeypatch.setattr(litellm, "user_url_allowed_hosts", ["docs.internal:8443"]) + + assert settings.url_policy() == settings.UrlPolicy( + user_url_validation=False, + user_url_allowed_hosts=["docs.internal:8443"], + ) def test_http_settings_reads_the_litellm_globals(monkeypatch: pytest.MonkeyPatch) -> None: From c635c35b3d2968dbf76ebec98eee833e2b5c8a0f Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Fri, 18 Sep 2026 18:47:36 -0700 Subject: [PATCH 09/15] fix(rust): keep native OCR on the proxy by declining only a supplied client The proxy attaches its shared aiohttp session to every request as shared_session, so declining on it sent every proxy OCR call to Python, which never uses that session for OCR. aclient_session is a litellm global and never a call argument, so that check could not match. The proxy-shaped lifecycle test now asserts the call was served by Rust --- litellm-rust/crates/python-bridge/src/http.rs | 46 +++++++++---------- tests/test_litellm_rust/ocr/test_lifecycle.py | 1 + 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/litellm-rust/crates/python-bridge/src/http.rs b/litellm-rust/crates/python-bridge/src/http.rs index 77542855fec..c5952c53132 100644 --- a/litellm-rust/crates/python-bridge/src/http.rs +++ b/litellm-rust/crates/python-bridge/src/http.rs @@ -12,8 +12,6 @@ use crate::{errors::RustBridgeDeclined, python_settings::PythonSettings}; static POOL: LazyLock = LazyLock::new(|| HttpClientPool::new(Arc::new(PublicDnsResolver))); -const LIVE_CLIENT_ARGUMENTS: [&str; 3] = ["client", "shared_session", "aclient_session"]; - pub(crate) fn pool() -> &'static HttpClientPool { &POOL } @@ -23,7 +21,7 @@ pub(crate) fn call_config( kwargs: &Bound<'_, PyDict>, asynchronous: bool, ) -> PyResult { - decline_live_clients(kwargs)?; + decline_live_client(kwargs)?; decline_custom_url_policy(&PythonSettings::UrlPolicy.read(py)?)?; let configured = settings(&PythonSettings::Http.read(py)?)? .with_environment(&|name| std::env::var(name).ok()); @@ -53,13 +51,14 @@ fn for_call( } } -pub(crate) fn decline_live_clients(kwargs: &Bound<'_, PyDict>) -> PyResult<()> { - for name in LIVE_CLIENT_ARGUMENTS { - if kwargs.get_item(name)?.is_some_and(|value| !value.is_none()) { - return Err(RustBridgeDeclined::new_err(format!( - "{name} is a live Python HTTP client and cannot be used by the Rust route" - ))); - } +fn decline_live_client(kwargs: &Bound<'_, PyDict>) -> PyResult<()> { + if kwargs + .get_item("client")? + .is_some_and(|value| !value.is_none()) + { + return Err(RustBridgeDeclined::new_err( + "client is a live Python HTTP client and cannot be used by the Rust route", + )); } Ok(()) } @@ -362,32 +361,29 @@ user_agent='litellm/9.9.9', assert_eq!(config.trust_proxy_env, expected); } - #[rstest] - #[case::client("client")] - #[case::shared_session("shared_session")] - #[case::aclient_session("aclient_session")] - fn live_python_clients_decline_before_dispatch(#[case] name: &str) { + #[test] + fn live_python_client_declines_before_dispatch() { Python::initialize(); Python::attach(|py| { let kwargs = PyDict::new(py); kwargs - .set_item(name, py.eval(c"object()", None, None).unwrap()) + .set_item("client", py.eval(c"object()", None, None).unwrap()) .unwrap(); - let error = decline_live_clients(&kwargs).unwrap_err(); + let error = decline_live_client(&kwargs).unwrap_err(); assert!(error.is_instance_of::(py)); - assert!(error.value(py).to_string().contains(name)); }); } - #[test] - fn none_valued_client_arguments_are_not_live_clients() { + #[rstest] + #[case::absent_client("{}")] + #[case::none_client("{'client': None}")] + #[case::proxy_shared_session("{'shared_session': object()}")] + fn calls_without_a_python_client_stay_on_the_rust_route(#[case] kwargs: &str) { Python::initialize(); Python::attach(|py| { - let kwargs = PyDict::new(py); - for name in LIVE_CLIENT_ARGUMENTS { - kwargs.set_item(name, py.None()).unwrap(); - } - decline_live_clients(&kwargs).unwrap(); + let source = std::ffi::CString::new(kwargs).unwrap(); + let kwargs = py.eval(&source, None, None).unwrap(); + decline_live_client(kwargs.cast::().unwrap()).unwrap(); }); } } diff --git a/tests/test_litellm_rust/ocr/test_lifecycle.py b/tests/test_litellm_rust/ocr/test_lifecycle.py index 5fca927bea3..264a666c685 100644 --- a/tests/test_litellm_rust/ocr/test_lifecycle.py +++ b/tests/test_litellm_rust/ocr/test_lifecycle.py @@ -39,6 +39,7 @@ async def test_proxy_metadata_remains_python_owned(ocr_server: RecordingServer) ) events: Final = await recorder.wait_for_async("async_log_success_event") assert response.pages[0].markdown == "native OCR response" + assert response._hidden_params["additional_headers"]["x-litellm-rust"] == "true" assert events[0].kwargs["litellm_params"]["metadata"]["user_api_key_auth"].user_id == "ocr-user" assert "metadata" not in ocr_server.requests[0].body From 0119f5001581eea9e202ddc6e8b6543da9424422 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Fri, 18 Sep 2026 18:47:36 -0700 Subject: [PATCH 10/15] fix(rust): restore the 10s connect timeout and share media clients across proxy settings Python OCR passes the call timeout per request, so its connect timeout is the call timeout and never the 5s handler default. 10s is what every Rust route uses on main. The media client never uses a proxy, so trust_proxy_env no longer splits its pool key --- litellm-rust/crates/http/src/pool.rs | 17 ++++++++++++++++- litellm-rust/crates/http/src/settings.rs | 2 +- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/litellm-rust/crates/http/src/pool.rs b/litellm-rust/crates/http/src/pool.rs index 0d9b1abf504..b51e6711419 100644 --- a/litellm-rust/crates/http/src/pool.rs +++ b/litellm-rust/crates/http/src/pool.rs @@ -51,6 +51,7 @@ impl HttpClientPool { let effective = match variant { ClientVariant::Media => HttpClientConfig { client_certificate: None, + trust_proxy_env: false, ..config.clone() }, ClientVariant::Provider | ClientVariant::NoRedirect => config.clone(), @@ -86,7 +87,6 @@ impl HttpClientPool { ClientVariant::NoRedirect => builder.redirect(reqwest::redirect::Policy::none()), ClientVariant::Media => builder .redirect(reqwest::redirect::Policy::none()) - .no_proxy() .dns_resolver2(Arc::clone(&self.media_resolver)), } } @@ -206,6 +206,21 @@ mod tests { assert_eq!(connections.load(Ordering::SeqCst), 2); } + #[tokio::test] + async fn media_clients_are_shared_across_proxy_settings_they_never_use() { + let (address, connections, _) = serve("HTTP/1.1 204 No Content").await; + let pool = HttpClientPool::new(Arc::new(FixedResolver(address))); + let url = format!("http://media.invalid:{}/doc", address.port()); + for trust_proxy_env in [true, false] { + let config = HttpClientConfig { + trust_proxy_env, + ..config("a") + }; + get(&pool, &config, ClientVariant::Media, &url).await; + } + assert_eq!(connections.load(Ordering::SeqCst), 1); + } + #[test] fn media_variant_never_loads_the_client_certificate() { let pool = pool(); diff --git a/litellm-rust/crates/http/src/settings.rs b/litellm-rust/crates/http/src/settings.rs index c572c56ef3a..8aaf7f21f2c 100644 --- a/litellm-rust/crates/http/src/settings.rs +++ b/litellm-rust/crates/http/src/settings.rs @@ -50,7 +50,7 @@ impl Default for HttpSettings { user_agent: None, trust_proxy_env: false, ignore_proxy_env: false, - connect_timeout: Duration::from_secs(5), + connect_timeout: Duration::from_secs(10), } } } From fb41bc3ed658b9023937c5f477c6311256c2dd68 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Fri, 18 Sep 2026 19:39:07 -0700 Subject: [PATCH 11/15] revert(ocr): stop forwarding client= on the Python path Python becomes a thin SDK interface over Rust, so a live Python HTTP client has no effect on either route. This puts the Python OCR path back to what main does --- litellm/ocr/main.py | 8 -------- tests/test_litellm/ocr/test_main.py | 29 ----------------------------- 2 files changed, 37 deletions(-) diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 851d9162964..06830ed4b53 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -25,7 +25,6 @@ from litellm.llms.base_llm.ocr.transformation import ( OCRResponse, parse_ocr_request_format, ) -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import CustomPricingLiteLLMParams @@ -53,11 +52,6 @@ class _PreparedOCRRequest: litellm_logging_obj: LiteLLMLoggingObj -def _supplied_client(kwargs: Mapping[str, object]) -> HTTPHandler | AsyncHTTPHandler | None: - candidate: Final = kwargs.get("client") - return candidate if isinstance(candidate, (HTTPHandler, AsyncHTTPHandler)) else None - - def _prepare_ocr_request( model: str, document: Mapping[str, object], @@ -244,7 +238,6 @@ async def aocr( api_key=prepared.api_key, api_base=prepared.api_base, custom_llm_provider=prepared.custom_llm_provider, - client=_supplied_client(kwargs), aocr=True, headers=prepared.extra_headers, provider_config=prepared.provider_config, @@ -411,7 +404,6 @@ def ocr( api_key=prepared.api_key, api_base=prepared.api_base, custom_llm_provider=prepared.custom_llm_provider, - client=_supplied_client(kwargs), aocr=_is_async, headers=prepared.extra_headers, provider_config=prepared.provider_config, diff --git a/tests/test_litellm/ocr/test_main.py b/tests/test_litellm/ocr/test_main.py index 32e5637ee09..5531a2639c0 100644 --- a/tests/test_litellm/ocr/test_main.py +++ b/tests/test_litellm/ocr/test_main.py @@ -113,35 +113,6 @@ async def test_python_request_response_and_callbacks( assert logger.log_pre_api_call.call_count == 1 -@pytest.mark.asyncio -@pytest.mark.parametrize("asynchronous", [False, True]) -async def test_python_uses_the_supplied_client(provider: Mock, asynchronous: bool) -> None: - supplied: Final = Mock(return_value=provider.return_value) - transport: Final = httpx.MockTransport(supplied) - arguments: Final = { - "model": "mistral/mistral-ocr-latest", - "document": dict(PRICING_DOCUMENT), - "api_key": "test-key", - "api_base": "https://ocr.test/v1", - } - - async def call() -> OCRResponse: - if not asynchronous: - with httpx.Client(transport=transport) as sync_client: - return litellm.ocr(**arguments, client=HTTPHandler(client=sync_client)) - async with httpx.AsyncClient(transport=transport) as async_client: - handler: Final = AsyncHTTPHandler() - await handler.client.aclose() - handler.client = async_client - return await litellm.aocr(**arguments, client=handler) - - response: Final = await call() - assert response.pages[0].markdown == "parsed document" - assert supplied.call_count == 1 - assert str(supplied.call_args.args[0].url) == "https://ocr.test/v1/ocr" - assert provider.call_count == 0 - - @pytest.mark.asyncio @pytest.mark.parametrize("asynchronous", [False, True]) async def test_python_provider_errors_keep_public_exception(provider: Mock, asynchronous: bool) -> None: From 51010ea486666b736c9d289e9df6b17eff5b7d7d Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Fri, 18 Sep 2026 19:39:07 -0700 Subject: [PATCH 12/15] feat(rust): serve every gateway HTTP setting natively instead of declining to Python litellm-http now builds the rustls config itself, so one route-neutral place covers roots, the client certificate, ALPN, ssl_ecdh_curve and ssl_security_level. A curve picks the single key exchange group. A cipher string restricts the TLS 1.2 suites it names, and entries rustls cannot express, such as @SECLEVEL=1, are logged once and skipped. user_url_validation and user_url_allowed_hosts are applied by the media fetcher. Document downloads honor the environment proxy whenever provider calls do, keeping the per-hop address check, and stay on the pinned resolver when no proxy applies. AIOHTTP_SO_KEEPALIVE, AIOHTTP_TCP_KEEPIDLE, AIOHTTP_TCP_KEEPINTVL, AIOHTTP_TCP_KEEPCNT and AIOHTTP_KEEPALIVE_TIMEOUT map onto the client. A client= argument and a live SSLContext are ignored --- litellm-rust/Cargo.lock | 4 + litellm-rust/Cargo.toml | 3 + litellm-rust/crates/core/tests/ocr.rs | 8 +- litellm-rust/crates/http/Cargo.toml | 4 + litellm-rust/crates/http/src/config.rs | 235 +++++----- litellm-rust/crates/http/src/error.rs | 5 - litellm-rust/crates/http/src/lib.rs | 8 +- litellm-rust/crates/http/src/pool.rs | 29 +- litellm-rust/crates/http/src/proxy.rs | 15 + litellm-rust/crates/http/src/settings.rs | 52 +++ litellm-rust/crates/http/src/tls.rs | 402 ++++++++++++++++++ .../llms/src/custom_httpx/llm_http_handler.rs | 5 +- .../crates/llms/src/custom_httpx/media.rs | 207 ++++++++- litellm-rust/crates/python-bridge/src/http.rs | 174 +++----- .../python-bridge/src/python_settings.rs | 5 + .../python-bridge/src/routes/ocr/mod.rs | 9 +- litellm/rust_bridge/settings.py | 6 + .../test_litellm/rust_bridge/test_settings.py | 8 + 18 files changed, 944 insertions(+), 235 deletions(-) create mode 100644 litellm-rust/crates/http/src/proxy.rs create mode 100644 litellm-rust/crates/http/src/tls.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 7f2d2e6b28b..d4b32659ba1 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -2135,10 +2135,14 @@ dependencies = [ name = "litellm-http" version = "0.1.0" dependencies = [ + "http 1.4.2", + "hyper-util", "reqwest 0.12.28", "rstest", + "rustls 0.23.42", "thiserror 2.0.19", "tokio", + "webpki-roots", ] [[package]] diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 02f4cc6b3ab..8634dce92d0 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -27,6 +27,8 @@ litellm-token-counter = { path = "crates/token-counter" } litellm-host-python = { path = "crates/host-python" } bytes = "1" +http = "1" +hyper-util = { version = "0.1.20", default-features = false, features = ["client-proxy"] } proptest = "1.7.0" pyo3 = "0.29.2" pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] } @@ -50,6 +52,7 @@ base64 = "0.22" moka = { version = "0.12.16", features = ["future"] } strum = { version = "0.28.0", features = ["derive"] } url = "2.5.8" +webpki-roots = "1" time = { version = "0.3.53", features = ["parsing"] } criterion = "0.8.2" fancy-regex = "0.19.2" diff --git a/litellm-rust/crates/core/tests/ocr.rs b/litellm-rust/crates/core/tests/ocr.rs index 2ae162d964f..a6b26bd8a27 100644 --- a/litellm-rust/crates/core/tests/ocr.rs +++ b/litellm-rust/crates/core/tests/ocr.rs @@ -12,7 +12,10 @@ use litellm_llms::{ error::Error as OcrError, transformation::{LiteLLMOcrResponse, OCR_RESPONSE_MAX_BYTES, OcrTransportConfig}, }, - custom_httpx::{llm_http_handler::OcrClient, media::PublicDnsResolver}, + custom_httpx::{ + llm_http_handler::OcrClient, + media::{PublicDnsResolver, UrlPolicy}, + }, }; use rstest::rstest; use serde_json::{Value, json}; @@ -181,7 +184,8 @@ async fn ocr_client_uses_the_injected_http_pool_configuration() { }; let client = OcrClient::new( &HttpClientPool::new(Arc::new(PublicDnsResolver)), - &HttpClientConfig::resolve(&settings).unwrap(), + &HttpClientConfig::resolve(&settings).config, + UrlPolicy::default(), VertexAuth::default(), ) .unwrap(); diff --git a/litellm-rust/crates/http/Cargo.toml b/litellm-rust/crates/http/Cargo.toml index 48ea4e66cef..0ac09a9d155 100644 --- a/litellm-rust/crates/http/Cargo.toml +++ b/litellm-rust/crates/http/Cargo.toml @@ -6,8 +6,12 @@ license.workspace = true repository.workspace = true [dependencies] +http.workspace = true +hyper-util.workspace = true reqwest.workspace = true +rustls.workspace = true thiserror.workspace = true +webpki-roots.workspace = true [dev-dependencies] rstest.workspace = true diff --git a/litellm-rust/crates/http/src/config.rs b/litellm-rust/crates/http/src/config.rs index 24a52315f7c..ebe557788ca 100644 --- a/litellm-rust/crates/http/src/config.rs +++ b/litellm-rust/crates/http/src/config.rs @@ -1,12 +1,13 @@ use std::{ net::{IpAddr, Ipv4Addr}, - path::{Path, PathBuf}, + path::PathBuf, time::Duration, }; use crate::{ error::Error, - settings::{HttpSettings, SslVerify}, + settings::{HttpSettings, SslVerify, TcpKeepalive}, + tls::{self, KeyExchangeGroup, Tls12CipherSuite, Unsupported}, }; #[derive(Clone, Debug, PartialEq, Eq, Hash)] @@ -20,27 +21,38 @@ pub enum Verify { pub struct HttpClientConfig { pub verify: Verify, pub client_certificate: Option, + pub key_exchange_group: Option, + pub tls12_cipher_suites: Option>, pub force_ipv4: bool, pub http2: bool, pub user_agent: Option, pub trust_proxy_env: bool, pub connect_timeout: Duration, + pub tcp_keepalive: Option, + pub pool_idle_timeout: Duration, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Resolution { + pub config: HttpClientConfig, + pub unsupported: Vec, } impl HttpClientConfig { - pub fn resolve(settings: &HttpSettings) -> Result { - if let Some(level) = &settings.ssl_security_level { - return Err(Error::Unsupported { - setting: "ssl_security_level", - reason: format!("OpenSSL cipher string {level:?} has no rustls equivalent"), - }); - } - if let Some(curve) = &settings.ssl_ecdh_curve { - return Err(Error::Unsupported { - setting: "ssl_ecdh_curve", - reason: format!("key exchange group {curve:?} is fixed by the rustls provider"), - }); - } + pub fn resolve(settings: &HttpSettings) -> Resolution { + let (key_exchange_group, unsupported_curve) = match settings + .ssl_ecdh_curve + .as_deref() + .map(KeyExchangeGroup::from_openssl_name) + { + None => (None, None), + Some(Ok(group)) => (Some(group), None), + Some(Err(unsupported)) => (None, Some(unsupported)), + }; + let ciphers = settings + .ssl_security_level + .as_deref() + .map(tls::parse_cipher_string); let verify = match &settings.ssl_verify { Some(SslVerify::Disabled) => Verify::Disabled, Some(SslVerify::CaBundle(path)) => Verify::CaBundle(path.clone()), @@ -49,62 +61,50 @@ impl HttpClientConfig { .clone() .map_or(Verify::BuiltInRoots, Verify::CaBundle), }; - Ok(Self { - verify, - client_certificate: settings.ssl_certificate.clone(), - force_ipv4: settings.force_ipv4, - http2: settings.http2, - user_agent: settings.user_agent.clone(), - trust_proxy_env: !settings.ignore_proxy_env - || settings.trust_proxy_env - || settings.http2 - || settings.httpx_transport, - connect_timeout: settings.connect_timeout, - }) + let (tls12_cipher_suites, unsupported_ciphers) = ciphers + .map_or((None, Vec::new()), |ciphers| { + (ciphers.tls12_cipher_suites, ciphers.unsupported) + }); + Resolution { + config: Self { + verify, + client_certificate: settings.ssl_certificate.clone(), + key_exchange_group, + tls12_cipher_suites, + force_ipv4: settings.force_ipv4, + http2: settings.http2, + user_agent: settings.user_agent.clone(), + trust_proxy_env: !settings.ignore_proxy_env + || settings.trust_proxy_env + || settings.http2 + || settings.httpx_transport, + connect_timeout: settings.connect_timeout, + tcp_keepalive: settings.tcp_keepalive, + pool_idle_timeout: settings.pool_idle_timeout, + }, + unsupported: unsupported_curve + .into_iter() + .chain(unsupported_ciphers) + .collect(), + } } pub fn client_builder(&self) -> Result { - let base = reqwest::Client::builder().connect_timeout(self.connect_timeout); - let with_roots = match &self.verify { - Verify::Disabled => base.danger_accept_invalid_certs(true), - Verify::BuiltInRoots => base, - Verify::CaBundle(path) => { - let pem = read(path)?; - let certificates = - reqwest::Certificate::from_pem_bundle(&pem).map_err(|error| { - Error::InvalidPem { - path: path.clone(), - message: error.without_url().to_string(), - } - })?; - if certificates.is_empty() { - return Err(Error::InvalidPem { - path: path.clone(), - message: "no certificates found".into(), - }); - } - certificates.into_iter().fold( - base.tls_built_in_root_certs(false), - |builder, certificate| builder.add_root_certificate(certificate), - ) - } - }; - let with_identity = match &self.client_certificate { - None => with_roots, - Some(path) => { - let identity = reqwest::Identity::from_pem(&read(path)?).map_err(|error| { - Error::InvalidPem { - path: path.clone(), - message: error.without_url().to_string(), - } - })?; - with_roots.identity(identity) - } + let base = reqwest::Client::builder() + .use_preconfigured_tls(tls::client_config(self)?) + .connect_timeout(self.connect_timeout) + .pool_idle_timeout(self.pool_idle_timeout); + let with_keepalive = match self.tcp_keepalive { + None => base, + Some(keepalive) => base + .tcp_keepalive(keepalive.idle) + .tcp_keepalive_interval(keepalive.interval) + .tcp_keepalive_retries(keepalive.retries), }; let with_address = if self.force_ipv4 { - with_identity.local_address(IpAddr::V4(Ipv4Addr::UNSPECIFIED)) + with_keepalive.local_address(IpAddr::V4(Ipv4Addr::UNSPECIFIED)) } else { - with_identity + with_keepalive }; let with_protocol = if self.http2 { with_address @@ -123,13 +123,6 @@ impl HttpClientConfig { } } -fn read(path: &Path) -> Result, Error> { - std::fs::read(path).map_err(|error| Error::Read { - path: path.to_path_buf(), - message: error.to_string(), - }) -} - #[cfg(test)] mod tests { use rstest::rstest; @@ -168,7 +161,7 @@ mod tests { #[case] settings: HttpSettings, #[case] expected: Verify, ) { - let config = HttpClientConfig::resolve(&settings).unwrap(); + let config = HttpClientConfig::resolve(&settings).config; assert_eq!(config.verify, expected); } @@ -179,42 +172,88 @@ mod tests { ..HttpSettings::default() } .with_environment(&|name: &str| (name == "SSL_VERIFY").then(|| "true".to_string())); - let config = HttpClientConfig::resolve(&settings).unwrap(); + let config = HttpClientConfig::resolve(&settings).config; assert_eq!(config.verify, Verify::BuiltInRoots); } + #[rstest] + #[case::x25519("X25519", Some(KeyExchangeGroup::X25519))] + #[case::openssl_p256("prime256v1", Some(KeyExchangeGroup::Secp256r1))] + #[case::p384("secp384r1", Some(KeyExchangeGroup::Secp384r1))] + fn ecdh_curve_selects_the_single_key_exchange_group( + #[case] curve: &str, + #[case] expected: Option, + ) { + let settings = HttpSettings { + ssl_ecdh_curve: Some(curve.into()), + ..HttpSettings::default() + }; + let resolution = HttpClientConfig::resolve(&settings); + assert_eq!(resolution.config.key_exchange_group, expected); + assert_eq!(resolution.unsupported, []); + } + #[test] - fn cipher_strings_are_rejected_rather_than_ignored() { + fn unsupported_ecdh_curve_keeps_the_defaults_and_is_reported() { + let settings = HttpSettings { + ssl_ecdh_curve: Some("secp521r1".into()), + ..HttpSettings::default() + }; + let resolution = HttpClientConfig::resolve(&settings); + assert_eq!(resolution.config.key_exchange_group, None); + assert_eq!( + resolution.unsupported, + [Unsupported::EcdhCurve("secp521r1".into())] + ); + } + + #[test] + fn legacy_security_level_keeps_every_suite_and_is_reported_unsupported() { let settings = HttpSettings { ssl_security_level: Some("DEFAULT@SECLEVEL=1".into()), ..HttpSettings::default() }; - assert!(matches!( - HttpClientConfig::resolve(&settings), - Err(Error::Unsupported { - setting: "ssl_security_level", - .. - }) - )); + let resolution = HttpClientConfig::resolve(&settings); + assert_eq!(resolution.config.tls12_cipher_suites, None); + assert_eq!( + resolution.unsupported, + [Unsupported::SecurityLevel("@SECLEVEL=1".into())] + ); } #[test] - fn ecdh_curves_are_rejected_rather_than_ignored() { + fn named_suites_restrict_tls12_and_unsupported_entries_are_reported() { let settings = HttpSettings { - ssl_ecdh_curve: Some("X25519".into()), + ssl_security_level: Some( + "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:!aNULL:AES256-SHA@SECLEVEL=2" + .into(), + ), ..HttpSettings::default() }; - assert!(matches!( - HttpClientConfig::resolve(&settings), - Err(Error::Unsupported { - setting: "ssl_ecdh_curve", - .. - }) - )); + let resolution = HttpClientConfig::resolve(&settings); + assert_eq!( + resolution.config.tls12_cipher_suites, + Some(vec![ + Tls12CipherSuite::EcdheEcdsaAes128Gcm, + Tls12CipherSuite::EcdheRsaAes256Gcm + ]) + ); + assert_eq!( + resolution.unsupported, + [ + Unsupported::CipherToken("!aNULL".into()), + Unsupported::CipherToken("AES256-SHA".into()) + ] + ); } #[test] fn connection_settings_carry_over_unchanged() { + let keepalive = TcpKeepalive { + idle: Duration::from_secs(60), + interval: Duration::from_secs(30), + retries: 5, + }; let settings = HttpSettings { ssl_certificate: Some("/client.pem".into()), force_ipv4: true, @@ -222,19 +261,25 @@ mod tests { user_agent: Some("litellm/1.0".into()), trust_proxy_env: true, connect_timeout: Duration::from_secs(7), + tcp_keepalive: Some(keepalive), + pool_idle_timeout: Duration::from_secs(45), ..HttpSettings::default() }; - let config = HttpClientConfig::resolve(&settings).unwrap(); + let config = HttpClientConfig::resolve(&settings).config; assert_eq!( config, HttpClientConfig { verify: Verify::BuiltInRoots, client_certificate: Some("/client.pem".into()), + key_exchange_group: None, + tls12_cipher_suites: None, force_ipv4: true, http2: true, user_agent: Some("litellm/1.0".into()), trust_proxy_env: true, connect_timeout: Duration::from_secs(7), + tcp_keepalive: Some(keepalive), + pool_idle_timeout: Duration::from_secs(45), } ); } @@ -258,7 +303,7 @@ mod tests { #[case] settings: HttpSettings, #[case] expected: bool, ) { - let config = HttpClientConfig::resolve(&settings).unwrap(); + let config = HttpClientConfig::resolve(&settings).config; assert_eq!(config.trust_proxy_env, expected); } @@ -267,7 +312,7 @@ mod tests { let path = std::env::temp_dir().join("litellm-http-missing-bundle.pem"); let config = HttpClientConfig { verify: Verify::CaBundle(path.clone()), - ..HttpClientConfig::resolve(&HttpSettings::default()).unwrap() + ..HttpClientConfig::resolve(&HttpSettings::default()).config }; assert!(matches!( config.client_builder(), @@ -282,7 +327,7 @@ mod tests { std::fs::write(&path, b"not a certificate").unwrap(); let config = HttpClientConfig { verify: Verify::CaBundle(path.clone()), - ..HttpClientConfig::resolve(&HttpSettings::default()).unwrap() + ..HttpClientConfig::resolve(&HttpSettings::default()).config }; let result = config.client_builder().map(drop); std::fs::remove_file(&path).unwrap(); diff --git a/litellm-rust/crates/http/src/error.rs b/litellm-rust/crates/http/src/error.rs index 27899f06cf1..697d0cf59c8 100644 --- a/litellm-rust/crates/http/src/error.rs +++ b/litellm-rust/crates/http/src/error.rs @@ -2,11 +2,6 @@ use std::path::PathBuf; #[derive(Clone, Debug, thiserror::Error, PartialEq, Eq)] pub enum Error { - #[error("{setting} cannot be expressed with rustls: {reason}")] - Unsupported { - setting: &'static str, - reason: String, - }, #[error("could not read {}: {message}", path.display())] Read { path: PathBuf, message: String }, #[error("{} is not a PEM file: {message}", path.display())] diff --git a/litellm-rust/crates/http/src/lib.rs b/litellm-rust/crates/http/src/lib.rs index c02a82539ff..45f370d3a9c 100644 --- a/litellm-rust/crates/http/src/lib.rs +++ b/litellm-rust/crates/http/src/lib.rs @@ -1,9 +1,13 @@ mod config; mod error; mod pool; +mod proxy; mod settings; +mod tls; -pub use config::{HttpClientConfig, Verify}; +pub use config::{HttpClientConfig, Resolution, Verify}; pub use error::Error; pub use pool::{ClientVariant, HttpClientPool}; -pub use settings::{HttpSettings, SslVerify}; +pub use proxy::EnvironmentProxies; +pub use settings::{HttpSettings, SslVerify, TcpKeepalive}; +pub use tls::{KeyExchangeGroup, Tls12CipherSuite, Unsupported, client_config}; diff --git a/litellm-rust/crates/http/src/pool.rs b/litellm-rust/crates/http/src/pool.rs index b51e6711419..e6e0de9bc5f 100644 --- a/litellm-rust/crates/http/src/pool.rs +++ b/litellm-rust/crates/http/src/pool.rs @@ -13,6 +13,7 @@ pub enum ClientVariant { Provider, NoRedirect, Media, + UnpinnedMedia, } const CLIENT_TTL: Duration = Duration::from_secs(3600); @@ -54,6 +55,10 @@ impl HttpClientPool { trust_proxy_env: false, ..config.clone() }, + ClientVariant::UnpinnedMedia => HttpClientConfig { + client_certificate: None, + ..config.clone() + }, ClientVariant::Provider | ClientVariant::NoRedirect => config.clone(), }; let key = (effective, variant); @@ -84,7 +89,9 @@ impl HttpClientPool { ) -> reqwest::ClientBuilder { match variant { ClientVariant::Provider => builder, - ClientVariant::NoRedirect => builder.redirect(reqwest::redirect::Policy::none()), + ClientVariant::NoRedirect | ClientVariant::UnpinnedMedia => { + builder.redirect(reqwest::redirect::Policy::none()) + } ClientVariant::Media => builder .redirect(reqwest::redirect::Policy::none()) .dns_resolver2(Arc::clone(&self.media_resolver)), @@ -125,7 +132,7 @@ mod tests { fn config(user_agent: &str) -> HttpClientConfig { HttpClientConfig { user_agent: Some(user_agent.into()), - ..HttpClientConfig::resolve(&HttpSettings::default()).unwrap() + ..HttpClientConfig::resolve(&HttpSettings::default()).config } } @@ -233,6 +240,10 @@ mod tests { .is_err() ); assert!(pool.client(&with_identity, ClientVariant::Media).is_ok()); + assert!( + pool.client(&with_identity, ClientVariant::UnpinnedMedia) + .is_ok() + ); } #[test] @@ -277,6 +288,20 @@ mod tests { assert_eq!(response.headers()["location"], "/elsewhere"); } + #[tokio::test] + async fn unpinned_media_variant_uses_the_system_resolver_and_returns_redirects() { + let (address, _, _) = serve("HTTP/1.1 302 Found").await; + let pool = HttpClientPool::new(Arc::new(FixedResolver(([192, 0, 2, 1], 80).into()))); + let response = get( + &pool, + &config("a"), + ClientVariant::UnpinnedMedia, + &format!("http://localhost:{}/doc", address.port()), + ) + .await; + assert_eq!(response.status(), 302); + } + #[tokio::test] async fn media_variant_resolves_through_the_injected_resolver() { let (address, _, requests) = serve("HTTP/1.1 204 No Content").await; diff --git a/litellm-rust/crates/http/src/proxy.rs b/litellm-rust/crates/http/src/proxy.rs new file mode 100644 index 00000000000..4dc4bf778b8 --- /dev/null +++ b/litellm-rust/crates/http/src/proxy.rs @@ -0,0 +1,15 @@ +use hyper_util::client::proxy::matcher::Matcher; + +pub struct EnvironmentProxies(Matcher); + +impl EnvironmentProxies { + pub fn from_environment() -> Self { + Self(Matcher::from_system()) + } + + pub fn apply_to(&self, url: &reqwest::Url) -> bool { + url.as_str() + .parse::() + .is_ok_and(|uri| self.0.intercept(&uri).is_some()) + } +} diff --git a/litellm-rust/crates/http/src/settings.rs b/litellm-rust/crates/http/src/settings.rs index 8aaf7f21f2c..be2f4f42fb4 100644 --- a/litellm-rust/crates/http/src/settings.rs +++ b/litellm-rust/crates/http/src/settings.rs @@ -20,6 +20,13 @@ impl SslVerify { } } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct TcpKeepalive { + pub idle: Duration, + pub interval: Duration, + pub retries: u32, +} + #[derive(Clone, Debug, PartialEq, Eq)] pub struct HttpSettings { pub ssl_verify: Option, @@ -34,6 +41,8 @@ pub struct HttpSettings { pub trust_proxy_env: bool, pub ignore_proxy_env: bool, pub connect_timeout: Duration, + pub tcp_keepalive: Option, + pub pool_idle_timeout: Duration, } impl Default for HttpSettings { @@ -51,6 +60,8 @@ impl Default for HttpSettings { trust_proxy_env: false, ignore_proxy_env: false, connect_timeout: Duration::from_secs(10), + tcp_keepalive: None, + pool_idle_timeout: Duration::from_secs(120), } } } @@ -59,6 +70,10 @@ impl HttpSettings { pub fn with_environment(self, env: &(dyn Fn(&str) -> Option + Sync)) -> Self { let enabled = |name: &str| env(name).is_some_and(|value| value.trim().eq_ignore_ascii_case("true")); + let number = |name: &str| env(name).and_then(|value| value.trim().parse::().ok()); + let seconds = |name: &str, default: u32| { + Duration::from_secs(u64::from(number(name).unwrap_or(default))) + }; Self { ssl_verify: env("SSL_VERIFY") .map(|value| SslVerify::parse(&value)) @@ -81,6 +96,17 @@ impl HttpSettings { user_agent: env("LITELLM_USER_AGENT").or(self.user_agent), trust_proxy_env: self.trust_proxy_env || enabled("AIOHTTP_TRUST_ENV"), ignore_proxy_env: self.ignore_proxy_env || enabled("DISABLE_AIOHTTP_TRUST_ENV"), + tcp_keepalive: enabled("AIOHTTP_SO_KEEPALIVE") + .then(|| TcpKeepalive { + idle: seconds("AIOHTTP_TCP_KEEPIDLE", 60), + interval: seconds("AIOHTTP_TCP_KEEPINTVL", 30), + retries: number("AIOHTTP_TCP_KEEPCNT").unwrap_or(5), + }) + .or(self.tcp_keepalive), + pool_idle_timeout: number("AIOHTTP_KEEPALIVE_TIMEOUT") + .map_or(self.pool_idle_timeout, |timeout| { + Duration::from_secs(u64::from(timeout)) + }), ..self } } @@ -188,6 +214,32 @@ mod tests { assert_eq!(settings.ssl_ecdh_curve, None); } + #[test] + fn socket_keepalive_follows_the_aiohttp_variables_with_python_defaults() { + let tuned = HttpSettings::default().with_environment(&env_of(&[ + ("AIOHTTP_SO_KEEPALIVE", "True"), + ("AIOHTTP_TCP_KEEPIDLE", "45"), + ("AIOHTTP_KEEPALIVE_TIMEOUT", "30"), + ])); + assert_eq!( + tuned.tcp_keepalive, + Some(TcpKeepalive { + idle: Duration::from_secs(45), + interval: Duration::from_secs(30), + retries: 5, + }) + ); + assert_eq!(tuned.pool_idle_timeout, Duration::from_secs(30)); + } + + #[test] + fn socket_keepalive_stays_off_unless_enabled() { + let settings = + HttpSettings::default().with_environment(&env_of(&[("AIOHTTP_TCP_KEEPIDLE", "45")])); + assert_eq!(settings.tcp_keepalive, None); + assert_eq!(settings.pool_idle_timeout, Duration::from_secs(120)); + } + #[test] fn missing_files_fall_back_to_default_verification() { let settings = HttpSettings { diff --git a/litellm-rust/crates/http/src/tls.rs b/litellm-rust/crates/http/src/tls.rs new file mode 100644 index 00000000000..605200c0be6 --- /dev/null +++ b/litellm-rust/crates/http/src/tls.rs @@ -0,0 +1,402 @@ +use std::{fmt, path::Path, sync::Arc}; + +use rustls::{ + CipherSuite, ClientConfig, DigitallySignedStruct, RootCertStore, SignatureScheme, + client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}, + crypto::{CryptoProvider, SupportedKxGroup, ring}, + pki_types::{CertificateDer, PrivateKeyDer, ServerName, UnixTime, pem::PemObject}, +}; + +use crate::{ + config::{HttpClientConfig, Verify}, + error::Error, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum KeyExchangeGroup { + X25519, + Secp256r1, + Secp384r1, +} + +impl KeyExchangeGroup { + pub(crate) fn from_openssl_name(name: &str) -> Result { + match name.trim().to_ascii_lowercase().as_str() { + "x25519" => Ok(Self::X25519), + "prime256v1" | "secp256r1" | "p-256" => Ok(Self::Secp256r1), + "secp384r1" | "p-384" => Ok(Self::Secp384r1), + _ => Err(Unsupported::EcdhCurve(name.to_owned())), + } + } + + fn supported(self) -> &'static dyn SupportedKxGroup { + match self { + Self::X25519 => ring::kx_group::X25519, + Self::Secp256r1 => ring::kx_group::SECP256R1, + Self::Secp384r1 => ring::kx_group::SECP384R1, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum Tls12CipherSuite { + EcdheEcdsaAes128Gcm, + EcdheEcdsaAes256Gcm, + EcdheEcdsaChacha20, + EcdheRsaAes128Gcm, + EcdheRsaAes256Gcm, + EcdheRsaChacha20, +} + +impl Tls12CipherSuite { + fn from_openssl_name(name: &str) -> Option { + match name { + "ECDHE-ECDSA-AES128-GCM-SHA256" => Some(Self::EcdheEcdsaAes128Gcm), + "ECDHE-ECDSA-AES256-GCM-SHA384" => Some(Self::EcdheEcdsaAes256Gcm), + "ECDHE-ECDSA-CHACHA20-POLY1305" => Some(Self::EcdheEcdsaChacha20), + "ECDHE-RSA-AES128-GCM-SHA256" => Some(Self::EcdheRsaAes128Gcm), + "ECDHE-RSA-AES256-GCM-SHA384" => Some(Self::EcdheRsaAes256Gcm), + "ECDHE-RSA-CHACHA20-POLY1305" => Some(Self::EcdheRsaChacha20), + _ => None, + } + } + + fn suite(self) -> CipherSuite { + match self { + Self::EcdheEcdsaAes128Gcm => CipherSuite::TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + Self::EcdheEcdsaAes256Gcm => CipherSuite::TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + Self::EcdheEcdsaChacha20 => CipherSuite::TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + Self::EcdheRsaAes128Gcm => CipherSuite::TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + Self::EcdheRsaAes256Gcm => CipherSuite::TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + Self::EcdheRsaChacha20 => CipherSuite::TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash, thiserror::Error)] +pub enum Unsupported { + #[error( + "ssl_ecdh_curve {0:?} is not supported: rustls with ring only offers X25519, prime256v1 and secp384r1, so the default key exchange groups are used" + )] + EcdhCurve(String), + #[error( + "ssl_security_level {0:?} is not supported: rustls has one fixed security level, comparable to OpenSSL level 2, so legacy servers that need a lower level cannot be reached" + )] + SecurityLevel(String), + #[error( + "ssl_security_level entry {0:?} is not supported: rustls only offers ECDHE AEAD cipher suites, so the entry is ignored" + )] + CipherToken(String), +} + +pub(crate) struct CipherSelection { + pub(crate) tls12_cipher_suites: Option>, + pub(crate) unsupported: Vec, +} + +enum CipherToken { + Suite(Tls12CipherSuite), + EverySuite, + Ordering, + Unsupported(Unsupported), +} + +fn cipher_token(token: &str) -> CipherToken { + if let Some(suite) = Tls12CipherSuite::from_openssl_name(token) { + return CipherToken::Suite(suite); + } + match token { + "DEFAULT" | "ALL" | "HIGH" => CipherToken::EverySuite, + "@STRENGTH" | "@SECLEVEL=2" => CipherToken::Ordering, + level if level.starts_with("@SECLEVEL=") => { + CipherToken::Unsupported(Unsupported::SecurityLevel(level.to_owned())) + } + other => CipherToken::Unsupported(Unsupported::CipherToken(other.to_owned())), + } +} + +pub(crate) fn parse_cipher_string(value: &str) -> CipherSelection { + let tokens: Vec = tokenize(value) + .iter() + .map(|token| cipher_token(token)) + .collect(); + let every_suite = tokens + .iter() + .any(|token| matches!(token, CipherToken::EverySuite)); + let mut suites: Vec = tokens + .iter() + .filter_map(|token| match token { + CipherToken::Suite(suite) => Some(*suite), + _ => None, + }) + .collect(); + suites.sort_unstable(); + suites.dedup(); + CipherSelection { + tls12_cipher_suites: (!every_suite && !suites.is_empty()).then_some(suites), + unsupported: tokens + .into_iter() + .filter_map(|token| match token { + CipherToken::Unsupported(unsupported) => Some(unsupported), + _ => None, + }) + .collect(), + } +} + +fn tokenize(value: &str) -> Vec { + value + .split([':', ',', ' ']) + .flat_map(|entry| match entry.split_once('@') { + Some((name, command)) => vec![name.to_owned(), format!("@{command}")], + None => vec![entry.to_owned()], + }) + .filter(|token| !token.is_empty()) + .collect() +} + +pub fn client_config(config: &HttpClientConfig) -> Result { + let base = ring::default_provider(); + let provider = Arc::new(CryptoProvider { + kx_groups: config + .key_exchange_group + .map_or_else(|| base.kx_groups.clone(), |group| vec![group.supported()]), + cipher_suites: base + .cipher_suites + .iter() + .copied() + .filter(|suite| { + suite.tls13().is_some() + || config + .tls12_cipher_suites + .as_ref() + .is_none_or(|allowed| allowed.iter().any(|a| a.suite() == suite.suite())) + }) + .collect(), + ..base + }); + let builder = ClientConfig::builder_with_provider(Arc::clone(&provider)) + .with_safe_default_protocol_versions() + .map_err(|error| Error::Client(error.to_string()))?; + let verified = match &config.verify { + Verify::Disabled => builder + .dangerous() + .with_custom_certificate_verifier(Arc::new(NoVerification(provider))), + Verify::BuiltInRoots => builder.with_root_certificates(built_in_roots()), + Verify::CaBundle(path) => builder.with_root_certificates(bundle_roots(path)?), + }; + let mut tls = match &config.client_certificate { + None => verified.with_no_client_auth(), + Some(path) => { + let (chain, key) = identity(path)?; + verified + .with_client_auth_cert(chain, key) + .map_err(|error| invalid_pem(path, error))? + } + }; + tls.alpn_protocols = if config.http2 { + vec![b"h2".to_vec(), b"http/1.1".to_vec()] + } else { + vec![b"http/1.1".to_vec()] + }; + Ok(tls) +} + +fn built_in_roots() -> RootCertStore { + let mut store = RootCertStore::empty(); + store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + store +} + +fn bundle_roots(path: &Path) -> Result { + let certificates = certificates(path)?; + if certificates.is_empty() { + return Err(invalid_pem(path, "no certificates found")); + } + let mut store = RootCertStore::empty(); + for certificate in certificates { + store + .add(certificate) + .map_err(|error| invalid_pem(path, error))?; + } + Ok(store) +} + +fn identity(path: &Path) -> Result<(Vec>, PrivateKeyDer<'static>), Error> { + let chain = certificates(path)?; + if chain.is_empty() { + return Err(invalid_pem(path, "no certificates found")); + } + let key = + PrivateKeyDer::from_pem_slice(&read(path)?).map_err(|error| invalid_pem(path, error))?; + Ok((chain, key)) +} + +fn certificates(path: &Path) -> Result>, Error> { + CertificateDer::pem_slice_iter(&read(path)?) + .collect::>() + .map_err(|error| invalid_pem(path, error)) +} + +fn read(path: &Path) -> Result, Error> { + std::fs::read(path).map_err(|error| Error::Read { + path: path.to_path_buf(), + message: error.to_string(), + }) +} + +fn invalid_pem(path: &Path, message: impl fmt::Display) -> Error { + Error::InvalidPem { + path: path.to_path_buf(), + message: message.to_string(), + } +} + +#[derive(Debug)] +struct NoVerification(Arc); + +impl ServerCertVerifier for NoVerification { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + _now: UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + _message: &[u8], + _cert: &CertificateDer<'_>, + _dss: &DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn verify_tls13_signature( + &self, + _message: &[u8], + _cert: &CertificateDer<'_>, + _dss: &DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn supported_verify_schemes(&self) -> Vec { + self.0.signature_verification_algorithms.supported_schemes() + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use rustls::NamedGroup; + + use super::*; + use crate::HttpSettings; + + fn config(settings: HttpSettings) -> HttpClientConfig { + HttpClientConfig::resolve(&settings).config + } + + fn offered_groups(tls: &ClientConfig) -> Vec { + tls.crypto_provider() + .kx_groups + .iter() + .map(|group| group.name()) + .collect() + } + + fn offered_tls12_suites(tls: &ClientConfig) -> Vec { + tls.crypto_provider() + .cipher_suites + .iter() + .filter(|suite| suite.tls13().is_none()) + .map(|suite| suite.suite()) + .collect() + } + + #[rstest] + #[case("X25519", NamedGroup::X25519)] + #[case("prime256v1", NamedGroup::secp256r1)] + #[case("secp384r1", NamedGroup::secp384r1)] + fn ecdh_curve_is_the_only_key_exchange_group_offered( + #[case] curve: &str, + #[case] expected: NamedGroup, + ) { + let tls = client_config(&config(HttpSettings { + ssl_ecdh_curve: Some(curve.into()), + ..HttpSettings::default() + })) + .unwrap(); + assert_eq!(offered_groups(&tls), [expected]); + } + + #[test] + fn default_settings_offer_every_group_and_suite_of_the_provider() { + let tls = client_config(&config(HttpSettings::default())).unwrap(); + let provider = ring::default_provider(); + assert_eq!(offered_groups(&tls).len(), provider.kx_groups.len()); + assert_eq!( + tls.crypto_provider().cipher_suites.len(), + provider.cipher_suites.len() + ); + } + + #[test] + fn named_suites_are_the_only_tls12_suites_offered_and_tls13_stays() { + let tls = client_config(&config(HttpSettings { + ssl_security_level: Some("ECDHE-RSA-AES256-GCM-SHA384".into()), + ..HttpSettings::default() + })) + .unwrap(); + assert_eq!( + offered_tls12_suites(&tls), + [CipherSuite::TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384] + ); + assert!( + tls.crypto_provider() + .cipher_suites + .iter() + .any(|suite| suite.tls13().is_some()) + ); + } + + #[rstest] + #[case(true, &[b"h2".as_slice(), b"http/1.1".as_slice()])] + #[case(false, &[b"http/1.1".as_slice()])] + fn alpn_offers_h2_only_when_http2_is_on(#[case] http2: bool, #[case] expected: &[&[u8]]) { + let tls = client_config(&config(HttpSettings { + http2, + ..HttpSettings::default() + })) + .unwrap(); + assert_eq!(tls.alpn_protocols, expected); + } + + #[test] + fn client_certificate_without_a_private_key_is_an_invalid_pem_error() { + let path = std::env::temp_dir().join(format!( + "litellm-http-cert-without-key-{}.pem", + std::process::id() + )); + std::fs::write( + &path, + b"-----BEGIN CERTIFICATE-----\nAA==\n-----END CERTIFICATE-----\n", + ) + .unwrap(); + let result = client_config(&HttpClientConfig { + client_certificate: Some(path.clone()), + ..config(HttpSettings::default()) + }) + .map(drop); + std::fs::remove_file(&path).unwrap(); + assert!(matches!( + result, + Err(Error::InvalidPem { path: reported, .. }) if reported == path + )); + } +} diff --git a/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs b/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs index 876fa0aae87..58dc03eea2d 100644 --- a/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs +++ b/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs @@ -16,7 +16,7 @@ use crate::{ }, custom_httpx::{ http_handler::{HeaderPolicy, execute_http_request, with_headers}, - media::MediaFetcher, + media::{MediaFetcher, UrlPolicy}, transport, }, }; @@ -41,12 +41,13 @@ impl OcrClient { pub fn new( pool: &HttpClientPool, config: &HttpClientConfig, + url_policy: UrlPolicy, vertex_auth: VertexAuth, ) -> Result { Ok(Self { provider_http: pool.client(config, ClientVariant::Provider)?, polling_http: pool.client(config, ClientVariant::NoRedirect)?, - document_fetcher: MediaFetcher::new(pool, config)?, + document_fetcher: MediaFetcher::new(pool, config, url_policy)?, vertex_auth, }) } diff --git a/litellm-rust/crates/llms/src/custom_httpx/media.rs b/litellm-rust/crates/llms/src/custom_httpx/media.rs index a1c4fe68734..02d152d3ef1 100644 --- a/litellm-rust/crates/llms/src/custom_httpx/media.rs +++ b/litellm-rust/crates/llms/src/custom_httpx/media.rs @@ -7,7 +7,7 @@ use std::{ time::Duration, }; -use litellm_http::{ClientVariant, HttpClientConfig, HttpClientPool}; +use litellm_http::{ClientVariant, EnvironmentProxies, HttpClientConfig, HttpClientPool}; use reqwest::{ Url, dns::{Addrs, Name, Resolve, Resolving}, @@ -35,10 +35,45 @@ pub enum Error { Transport(#[from] crate::custom_httpx::transport::Error), } +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct UrlPolicy { + pub validate: bool, + pub allowed_hosts: Vec, +} + +impl Default for UrlPolicy { + fn default() -> Self { + Self { + validate: true, + allowed_hosts: Vec::new(), + } + } +} + +impl UrlPolicy { + fn allows(&self, host: &str, port: u16) -> bool { + let host = normalize_host(host); + let with_port = format!("{host}:{port}"); + self.allowed_hosts + .iter() + .map(|entry| normalize_host(entry)) + .any(|entry| entry == host || entry == with_port) + } +} + +fn normalize_host(host: &str) -> String { + host.to_ascii_lowercase().trim_end_matches('.').to_owned() +} + +type ProxyMatch = Arc bool + Send + Sync>; + #[derive(Clone)] pub struct MediaFetcher { - client: reqwest::Client, + pinned: reqwest::Client, + unpinned: reqwest::Client, + uses_proxy: ProxyMatch, address_resolver: Arc, + url_policy: UrlPolicy, allow_private_network: bool, } @@ -65,19 +100,36 @@ impl MediaFetcher { pub fn new( pool: &HttpClientPool, config: &HttpClientConfig, + url_policy: UrlPolicy, ) -> Result { - Self::with_address_resolver(pool, config, Arc::new(SystemAddressResolver)) + let uses_proxy: ProxyMatch = if config.trust_proxy_env { + let proxies = EnvironmentProxies::from_environment(); + Arc::new(move |url| proxies.apply_to(url)) + } else { + Arc::new(|_| false) + }; + Self::with_resolution( + pool, + config, + url_policy, + Arc::new(SystemAddressResolver), + uses_proxy, + ) } - fn with_address_resolver( + fn with_resolution( pool: &HttpClientPool, config: &HttpClientConfig, + url_policy: UrlPolicy, address_resolver: Arc, + uses_proxy: ProxyMatch, ) -> Result { - let client = pool.client(config, ClientVariant::Media)?; Ok(Self { - client, + pinned: pool.client(config, ClientVariant::Media)?, + unpinned: pool.client(config, ClientVariant::UnpinnedMedia)?, + uses_proxy, address_resolver, + url_policy, allow_private_network: false, }) } @@ -85,8 +137,11 @@ impl MediaFetcher { #[cfg(any(test, feature = "test-support"))] pub fn for_test(client: reqwest::Client) -> Self { Self { - client, + pinned: client.clone(), + unpinned: client, + uses_proxy: Arc::new(|_| false), address_resolver: Arc::new(AllowPrivateResolver), + url_policy: UrlPolicy::default(), allow_private_network: true, } } @@ -107,9 +162,9 @@ impl MediaFetcher { ) -> Result { let mut redirects_followed = 0; loop { - self.validate_url(&url).await?; let mut response = self - .client + .client_for(&url) + .await? .get(url.clone()) .send() .await @@ -156,7 +211,10 @@ impl MediaFetcher { } } - async fn validate_url(&self, url: &Url) -> Result<(), Error> { + async fn client_for(&self, url: &Url) -> Result<&reqwest::Client, Error> { + if !self.url_policy.validate { + return Ok(&self.unpinned); + } if !matches!(url.scheme(), "http" | "https") || !url.username().is_empty() || url.password().is_some() @@ -165,12 +223,28 @@ impl MediaFetcher { } let host = url.host_str().ok_or(Error::BlockedUrl)?; if self.allow_private_network { - return Ok(()); - } - if let Ok(ip) = host.parse::() { - return (!is_blocked_ip(ip)).then_some(()).ok_or(Error::BlockedUrl); + return Ok(&self.pinned); } let port = url.port_or_known_default().ok_or(Error::BlockedUrl)?; + if self.url_policy.allows(host, port) { + return Ok(&self.unpinned); + } + self.validate_host(host, port).await?; + Ok(if (self.uses_proxy)(url) { + &self.unpinned + } else { + &self.pinned + }) + } + + async fn validate_host(&self, host: &str, port: u16) -> Result<(), Error> { + if let Ok(ip) = host + .trim_start_matches('[') + .trim_end_matches(']') + .parse::() + { + return (!is_blocked_ip(ip)).then_some(()).ok_or(Error::BlockedUrl); + } let addresses = self .address_resolver .resolve(host, port) @@ -360,14 +434,35 @@ mod tests { address: SocketAddr, blocked_hosts: HashSet<&'static str>, ) -> MediaFetcher { - MediaFetcher::with_address_resolver( - &HttpClientPool::new(Arc::new(LoopbackDnsResolver(address))), - &HttpClientConfig::resolve(&HttpSettings::default()).unwrap(), + fetcher(address, blocked_hosts, UrlPolicy::default(), false) + } + + fn fetcher( + pinned_address: SocketAddr, + blocked_hosts: HashSet<&'static str>, + url_policy: UrlPolicy, + uses_proxy: bool, + ) -> MediaFetcher { + let direct = HttpClientConfig { + trust_proxy_env: false, + ..HttpClientConfig::resolve(&HttpSettings::default()).config + }; + MediaFetcher::with_resolution( + &HttpClientPool::new(Arc::new(LoopbackDnsResolver(pinned_address))), + &direct, + url_policy, Arc::new(TestAddressResolver { blocked_hosts }), + Arc::new(move |_| uses_proxy), ) .expect("test fetcher builds") } + const UNROUTABLE: SocketAddr = + SocketAddr::new(IpAddr::V4(std::net::Ipv4Addr::new(192, 0, 2, 1)), 9); + + const OK_RESPONSE: &[u8] = + b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok"; + fn policy(max_bytes: u64, max_redirects: usize) -> DownloadPolicy { DownloadPolicy { timeout: Duration::from_secs(1), @@ -541,14 +636,88 @@ mod tests { async fn rejects_url_credentials_before_network_access() { let fetcher = MediaFetcher::new( &HttpClientPool::new(Arc::new(PublicDnsResolver)), - &HttpClientConfig::resolve(&HttpSettings::default()).expect("default settings resolve"), + &HttpClientConfig::resolve(&HttpSettings::default()).config, + UrlPolicy::default(), ) .expect("media fetcher builds"); let url = Url::parse("https://user:password@8.8.8.8/document").expect("credentialed URL parses"); assert!(matches!( - fetcher.validate_url(&url).await, + fetcher.fetch(url, policy(1, 0)).await, Err(Error::BlockedUrl) )); } + + #[tokio::test] + async fn allowlisted_private_host_is_fetched_without_the_pinned_resolver() { + let (url, server, _) = serve_named("localhost", vec![OK_RESPONSE]).await; + let port = url.port().expect("test URL has a port"); + let allowed = UrlPolicy { + validate: true, + allowed_hosts: vec![format!("LOCALHOST:{port}")], + }; + let media = fetcher(UNROUTABLE, HashSet::from(["localhost"]), allowed, false) + .fetch(url, policy(2, 0)) + .await + .expect("allowlisted host downloads"); + server.await.expect("server completes"); + assert_eq!(media.bytes, b"ok"); + } + + #[tokio::test] + async fn allowlist_entry_for_another_port_does_not_open_the_host() { + let (url, _server, _) = serve_named("localhost", vec![OK_RESPONSE]).await; + let other_port = UrlPolicy { + validate: true, + allowed_hosts: vec!["localhost:1".into()], + }; + let result = fetcher(UNROUTABLE, HashSet::from(["localhost"]), other_port, false) + .fetch(url, policy(2, 0)) + .await; + assert!(matches!(result, Err(Error::BlockedUrl))); + } + + #[tokio::test] + async fn validation_off_fetches_private_hosts_and_follows_redirects() { + let (url, server, _) = serve_named( + "localhost", + vec![ + b"HTTP/1.1 302 Found\r\nLocation: /moved\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + OK_RESPONSE, + ], + ) + .await; + let off = UrlPolicy { + validate: false, + allowed_hosts: Vec::new(), + }; + let media = fetcher(UNROUTABLE, HashSet::from(["localhost"]), off, false) + .fetch(url, policy(2, 1)) + .await + .expect("unvalidated download succeeds"); + let requests = server.await.expect("server completes"); + assert_eq!(media.bytes, b"ok"); + assert!(requests[1].starts_with("GET /moved ")); + } + + #[tokio::test] + async fn proxied_urls_skip_the_pinned_resolver_but_keep_the_address_check() { + let (url, server, _) = serve_named("localhost", vec![OK_RESPONSE]).await; + let media = fetcher(UNROUTABLE, HashSet::new(), UrlPolicy::default(), true) + .fetch(url.clone(), policy(2, 0)) + .await + .expect("public host behind a proxy downloads"); + server.await.expect("server completes"); + assert_eq!(media.bytes, b"ok"); + + let blocked = fetcher( + UNROUTABLE, + HashSet::from(["localhost"]), + UrlPolicy::default(), + true, + ) + .fetch(url, policy(2, 0)) + .await; + assert!(matches!(blocked, Err(Error::BlockedUrl))); + } } diff --git a/litellm-rust/crates/python-bridge/src/http.rs b/litellm-rust/crates/python-bridge/src/http.rs index c5952c53132..118f4669b62 100644 --- a/litellm-rust/crates/python-bridge/src/http.rs +++ b/litellm-rust/crates/python-bridge/src/http.rs @@ -1,10 +1,11 @@ use std::{ + collections::HashSet, path::{Path, PathBuf}, - sync::{Arc, LazyLock}, + sync::{Arc, LazyLock, Mutex, PoisonError}, }; -use litellm_http::{HttpClientConfig, HttpClientPool, HttpSettings, SslVerify}; -use litellm_llms::custom_httpx::media::PublicDnsResolver; +use litellm_http::{HttpClientConfig, HttpClientPool, HttpSettings, SslVerify, Unsupported}; +use litellm_llms::custom_httpx::media::{PublicDnsResolver, UrlPolicy}; use pyo3::{prelude::*, types::PyDict}; use crate::{errors::RustBridgeDeclined, python_settings::PythonSettings}; @@ -12,6 +13,8 @@ use crate::{errors::RustBridgeDeclined, python_settings::PythonSettings}; static POOL: LazyLock = LazyLock::new(|| HttpClientPool::new(Arc::new(PublicDnsResolver))); +static REPORTED_UNSUPPORTED: LazyLock>> = LazyLock::new(Mutex::default); + pub(crate) fn pool() -> &'static HttpClientPool { &POOL } @@ -21,22 +24,48 @@ pub(crate) fn call_config( kwargs: &Bound<'_, PyDict>, asynchronous: bool, ) -> PyResult { - decline_live_client(kwargs)?; - decline_custom_url_policy(&PythonSettings::UrlPolicy.read(py)?)?; let configured = settings(&PythonSettings::Http.read(py)?)? .with_environment(&|name| std::env::var(name).ok()); let settings = for_call(configured, call_ssl_verify(kwargs)?, asynchronous) .without_missing_files(&|path: &Path| path.exists()); - HttpClientConfig::resolve(&settings) - .map_err(|error| RustBridgeDeclined::new_err(error.to_string())) + let resolution = HttpClientConfig::resolve(&settings); + for unsupported in unreported(&REPORTED_UNSUPPORTED, resolution.unsupported) { + PythonSettings::warn(py, &unsupported.to_string())?; + } + Ok(resolution.config) +} + +fn unreported( + reported: &Mutex>, + unsupported: Vec, +) -> Vec { + let mut reported = reported.lock().unwrap_or_else(PoisonError::into_inner); + unsupported + .into_iter() + .filter(|unsupported| reported.insert(unsupported.clone())) + .collect() +} + +pub(crate) fn url_policy(py: Python<'_>) -> PyResult { + let policy: PythonUrlPolicy = + PythonSettings::UrlPolicy + .read(py)? + .extract() + .map_err(|error: PyErr| { + RustBridgeDeclined::new_err(format!( + "litellm URL policy cannot be used by the Rust route: {error}" + )) + })?; + Ok(UrlPolicy { + validate: policy.user_url_validation, + allowed_hosts: policy.user_url_allowed_hosts, + }) } fn call_ssl_verify(kwargs: &Bound<'_, PyDict>) -> PyResult> { - kwargs + Ok(kwargs .get_item("ssl_verify")? - .filter(|value| !value.is_none()) - .map(|value| ssl_verify(&value, "the ssl_verify argument")) - .transpose() + .and_then(|value| ssl_verify(&value))) } fn for_call( @@ -51,35 +80,12 @@ fn for_call( } } -fn decline_live_client(kwargs: &Bound<'_, PyDict>) -> PyResult<()> { - if kwargs - .get_item("client")? - .is_some_and(|value| !value.is_none()) - { - return Err(RustBridgeDeclined::new_err( - "client is a live Python HTTP client and cannot be used by the Rust route", - )); - } - Ok(()) -} - #[derive(FromPyObject)] struct PythonUrlPolicy { user_url_validation: bool, user_url_allowed_hosts: Vec, } -fn decline_custom_url_policy(value: &Bound<'_, PyAny>) -> PyResult<()> { - match value.extract::() { - Ok(policy) if policy.user_url_validation && policy.user_url_allowed_hosts.is_empty() => { - Ok(()) - } - Ok(_) | Err(_) => Err(RustBridgeDeclined::new_err( - "litellm.user_url_validation / user_url_allowed_hosts are applied by the Python route", - )), - } -} - #[derive(FromPyObject)] struct PythonHttpSettings<'py> { ssl_verify: Bound<'py, PyAny>, @@ -101,7 +107,7 @@ fn settings(value: &Bound<'_, PyAny>) -> PyResult { )) })?; Ok(HttpSettings { - ssl_verify: Some(ssl_verify(&python.ssl_verify, "litellm.ssl_verify")?), + ssl_verify: ssl_verify(&python.ssl_verify), ssl_certificate: python.ssl_certificate.map(PathBuf::from), ssl_security_level: python.ssl_security_level, ssl_ecdh_curve: python.ssl_ecdh_curve, @@ -115,20 +121,18 @@ fn settings(value: &Bound<'_, PyAny>) -> PyResult { }) } -fn ssl_verify(value: &Bound<'_, PyAny>, source: &str) -> PyResult { +fn ssl_verify(value: &Bound<'_, PyAny>) -> Option { if let Ok(enabled) = value.extract::() { - return Ok(if enabled { + return Some(if enabled { SslVerify::Enabled } else { SslVerify::Disabled }); } - if let Ok(path) = value.extract::() { - return Ok(SslVerify::parse(&path)); - } - Err(RustBridgeDeclined::new_err(format!( - "{source} is a live Python object and cannot be used by the Rust route" - ))) + value + .extract::() + .ok() + .map(|path| SslVerify::parse(&path)) } #[cfg(test)] @@ -247,50 +251,30 @@ user_agent='litellm/9.9.9', Python::initialize(); Python::attach(|py| { let settings = settings(&python_settings(py, overrides)).unwrap(); - let config = HttpClientConfig::resolve(&settings).unwrap(); + let config = HttpClientConfig::resolve(&settings).config; assert_eq!(config.verify, expected); }); } #[test] - fn ssl_context_global_declines_instead_of_being_dropped() { + fn ssl_context_global_is_ignored_so_environment_and_defaults_apply() { Python::initialize(); Python::attach(|py| { - let error = settings(&python_settings(py, "ssl_verify=object()")).unwrap_err(); - assert!(error.is_instance_of::(py)); - assert!(error.value(py).to_string().contains("litellm.ssl_verify")); + let settings = settings(&python_settings(py, "ssl_verify=object()")).unwrap(); + assert_eq!(settings.ssl_verify, None); }); } - fn url_policy<'py>(py: Python<'py>, fields: &str) -> Bound<'py, PyAny> { - let source = std::ffi::CString::new(format!( - "import types\npolicy = types.SimpleNamespace({fields})" - )) - .unwrap(); - let locals = PyDict::new(py); - py.run(&source, Some(&locals), Some(&locals)).unwrap(); - locals.get_item("policy").unwrap().unwrap() - } - #[test] - fn default_url_policy_stays_on_the_rust_route() { - Python::initialize(); - Python::attach(|py| { - let policy = url_policy(py, "user_url_validation=True, user_url_allowed_hosts=[]"); - decline_custom_url_policy(&policy).unwrap(); - }); - } - - #[rstest] - #[case::validation_off("user_url_validation=False, user_url_allowed_hosts=[]")] - #[case::allowlist("user_url_validation=True, user_url_allowed_hosts=['docs.internal']")] - #[case::mistyped("user_url_validation=True, user_url_allowed_hosts=None")] - fn custom_url_policy_declines_so_python_applies_it(#[case] fields: &str) { - Python::initialize(); - Python::attach(|py| { - let error = decline_custom_url_policy(&url_policy(py, fields)).unwrap_err(); - assert!(error.is_instance_of::(py)); - }); + fn unsupported_settings_are_reported_once_per_process() { + let reported = Mutex::default(); + let curve = Unsupported::EcdhCurve("secp521r1".into()); + let level = Unsupported::SecurityLevel("@SECLEVEL=1".into()); + assert_eq!( + unreported(&reported, vec![curve.clone(), level.clone()]), + [curve.clone(), level] + ); + assert_eq!(unreported(&reported, vec![curve]), []); } #[test] @@ -333,15 +317,19 @@ user_agent='litellm/9.9.9', } #[test] - fn live_ssl_context_argument_declines() { + fn live_ssl_context_argument_is_ignored_so_the_configured_value_applies() { Python::initialize(); Python::attach(|py| { let kwargs = PyDict::new(py); kwargs .set_item("ssl_verify", py.eval(c"object()", None, None).unwrap()) .unwrap(); - let error = call_ssl_verify(&kwargs).unwrap_err(); - assert!(error.is_instance_of::(py)); + let configured = HttpSettings { + ssl_verify: Some(SslVerify::Disabled), + ..HttpSettings::default() + }; + let settings = for_call(configured.clone(), call_ssl_verify(&kwargs).unwrap(), true); + assert_eq!(settings, configured); }); } @@ -357,33 +345,7 @@ user_agent='litellm/9.9.9', ..HttpSettings::default() }; let settings = for_call(opted_out, None, asynchronous); - let config = HttpClientConfig::resolve(&settings).unwrap(); + let config = HttpClientConfig::resolve(&settings).config; assert_eq!(config.trust_proxy_env, expected); } - - #[test] - fn live_python_client_declines_before_dispatch() { - Python::initialize(); - Python::attach(|py| { - let kwargs = PyDict::new(py); - kwargs - .set_item("client", py.eval(c"object()", None, None).unwrap()) - .unwrap(); - let error = decline_live_client(&kwargs).unwrap_err(); - assert!(error.is_instance_of::(py)); - }); - } - - #[rstest] - #[case::absent_client("{}")] - #[case::none_client("{'client': None}")] - #[case::proxy_shared_session("{'shared_session': object()}")] - fn calls_without_a_python_client_stay_on_the_rust_route(#[case] kwargs: &str) { - Python::initialize(); - Python::attach(|py| { - let source = std::ffi::CString::new(kwargs).unwrap(); - let kwargs = py.eval(&source, None, None).unwrap(); - decline_live_client(kwargs.cast::().unwrap()).unwrap(); - }); - } } diff --git a/litellm-rust/crates/python-bridge/src/python_settings.rs b/litellm-rust/crates/python-bridge/src/python_settings.rs index b7855566850..79921d67452 100644 --- a/litellm-rust/crates/python-bridge/src/python_settings.rs +++ b/litellm-rust/crates/python-bridge/src/python_settings.rs @@ -22,6 +22,11 @@ impl PythonSettings { pub(crate) fn read(self, py: Python<'_>) -> PyResult> { py.import(MODULE)?.getattr(self.name())?.call0() } + + pub(crate) fn warn(py: Python<'_>, message: &str) -> PyResult<()> { + py.import(MODULE)?.getattr("warn")?.call1((message,))?; + Ok(()) + } } #[cfg(test)] diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs index 174d0ff18c8..f9d7024c824 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs @@ -38,8 +38,13 @@ fn run_ocr( asynchronous: bool, ) -> PyResult> { let config = http::call_config(py, &kwargs, asynchronous)?; - let client = OcrClient::new(http::pool(), &config, VERTEX_AUTH.clone()) - .map_err(|error| RustBridgeDeclined::new_err(error.to_string()))?; + let client = OcrClient::new( + http::pool(), + &config, + http::url_policy(py)?, + VERTEX_AUTH.clone(), + ) + .map_err(|error| RustBridgeDeclined::new_err(error.to_string()))?; run_legacy_call( py, if asynchronous { ASYNC_SURFACE } else { SURFACE }, diff --git a/litellm/rust_bridge/settings.py b/litellm/rust_bridge/settings.py index bccfd01ec73..e170f93b198 100644 --- a/litellm/rust_bridge/settings.py +++ b/litellm/rust_bridge/settings.py @@ -24,6 +24,12 @@ class UrlPolicy: user_url_allowed_hosts: Sequence[str] +def warn(message: str) -> None: + from litellm._logging import verbose_logger + + verbose_logger.warning("%s", message) + + def url_policy() -> UrlPolicy: import litellm diff --git a/tests/test_litellm/rust_bridge/test_settings.py b/tests/test_litellm/rust_bridge/test_settings.py index 7e7b1c6743b..f75145c2b2c 100644 --- a/tests/test_litellm/rust_bridge/test_settings.py +++ b/tests/test_litellm/rust_bridge/test_settings.py @@ -1,4 +1,5 @@ import dataclasses +import logging from pathlib import Path from typing import Final @@ -65,3 +66,10 @@ def test_http_settings_ignores_environment_overrides(monkeypatch: pytest.MonkeyP assert result.user_agent == default_user_agent() assert result.ssl_verify is True + + +def test_warn_reaches_the_litellm_logger(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + settings.warn("ssl_ecdh_curve 'secp521r1' is not supported") + + assert [record.getMessage() for record in caplog.records] == ["ssl_ecdh_curve 'secp521r1' is not supported"] From ffbfe7205fa10c1f56b2205583728bf614a95419 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Fri, 18 Sep 2026 19:40:16 -0700 Subject: [PATCH 13/15] refactor(rust): parse TLS settings through FromStr, From and TryFrom KeyExchangeGroup and Tls12CipherSuite parse with FromStr and fail with Unsupported, so a setting rustls cannot honor is a typed error instead of a missing value. The cipher string conversions cannot fail and use From. The rustls ClientConfig is built with TryFrom<&HttpClientConfig>, and the built-in root store is constructed in one expression --- litellm-rust/crates/http/src/config.rs | 8 +- litellm-rust/crates/http/src/lib.rs | 2 +- litellm-rust/crates/http/src/tls.rs | 208 +++++++++++++------------ 3 files changed, 113 insertions(+), 105 deletions(-) diff --git a/litellm-rust/crates/http/src/config.rs b/litellm-rust/crates/http/src/config.rs index ebe557788ca..30216405fc8 100644 --- a/litellm-rust/crates/http/src/config.rs +++ b/litellm-rust/crates/http/src/config.rs @@ -7,7 +7,7 @@ use std::{ use crate::{ error::Error, settings::{HttpSettings, SslVerify, TcpKeepalive}, - tls::{self, KeyExchangeGroup, Tls12CipherSuite, Unsupported}, + tls::{CipherSelection, KeyExchangeGroup, Tls12CipherSuite, Unsupported}, }; #[derive(Clone, Debug, PartialEq, Eq, Hash)] @@ -43,7 +43,7 @@ impl HttpClientConfig { let (key_exchange_group, unsupported_curve) = match settings .ssl_ecdh_curve .as_deref() - .map(KeyExchangeGroup::from_openssl_name) + .map(str::parse::) { None => (None, None), Some(Ok(group)) => (Some(group), None), @@ -52,7 +52,7 @@ impl HttpClientConfig { let ciphers = settings .ssl_security_level .as_deref() - .map(tls::parse_cipher_string); + .map(CipherSelection::from); let verify = match &settings.ssl_verify { Some(SslVerify::Disabled) => Verify::Disabled, Some(SslVerify::CaBundle(path)) => Verify::CaBundle(path.clone()), @@ -91,7 +91,7 @@ impl HttpClientConfig { pub fn client_builder(&self) -> Result { let base = reqwest::Client::builder() - .use_preconfigured_tls(tls::client_config(self)?) + .use_preconfigured_tls(rustls::ClientConfig::try_from(self)?) .connect_timeout(self.connect_timeout) .pool_idle_timeout(self.pool_idle_timeout); let with_keepalive = match self.tcp_keepalive { diff --git a/litellm-rust/crates/http/src/lib.rs b/litellm-rust/crates/http/src/lib.rs index 45f370d3a9c..e222d0e3f50 100644 --- a/litellm-rust/crates/http/src/lib.rs +++ b/litellm-rust/crates/http/src/lib.rs @@ -10,4 +10,4 @@ pub use error::Error; pub use pool::{ClientVariant, HttpClientPool}; pub use proxy::EnvironmentProxies; pub use settings::{HttpSettings, SslVerify, TcpKeepalive}; -pub use tls::{KeyExchangeGroup, Tls12CipherSuite, Unsupported, client_config}; +pub use tls::{KeyExchangeGroup, Tls12CipherSuite, Unsupported}; diff --git a/litellm-rust/crates/http/src/tls.rs b/litellm-rust/crates/http/src/tls.rs index 605200c0be6..49405b97366 100644 --- a/litellm-rust/crates/http/src/tls.rs +++ b/litellm-rust/crates/http/src/tls.rs @@ -1,4 +1,4 @@ -use std::{fmt, path::Path, sync::Arc}; +use std::{fmt, path::Path, str::FromStr, sync::Arc}; use rustls::{ CipherSuite, ClientConfig, DigitallySignedStruct, RootCertStore, SignatureScheme, @@ -19,8 +19,10 @@ pub enum KeyExchangeGroup { Secp384r1, } -impl KeyExchangeGroup { - pub(crate) fn from_openssl_name(name: &str) -> Result { +impl FromStr for KeyExchangeGroup { + type Err = Unsupported; + + fn from_str(name: &str) -> Result { match name.trim().to_ascii_lowercase().as_str() { "x25519" => Ok(Self::X25519), "prime256v1" | "secp256r1" | "p-256" => Ok(Self::Secp256r1), @@ -28,7 +30,9 @@ impl KeyExchangeGroup { _ => Err(Unsupported::EcdhCurve(name.to_owned())), } } +} +impl KeyExchangeGroup { fn supported(self) -> &'static dyn SupportedKxGroup { match self { Self::X25519 => ring::kx_group::X25519, @@ -48,19 +52,23 @@ pub enum Tls12CipherSuite { EcdheRsaChacha20, } -impl Tls12CipherSuite { - fn from_openssl_name(name: &str) -> Option { +impl FromStr for Tls12CipherSuite { + type Err = Unsupported; + + fn from_str(name: &str) -> Result { match name { - "ECDHE-ECDSA-AES128-GCM-SHA256" => Some(Self::EcdheEcdsaAes128Gcm), - "ECDHE-ECDSA-AES256-GCM-SHA384" => Some(Self::EcdheEcdsaAes256Gcm), - "ECDHE-ECDSA-CHACHA20-POLY1305" => Some(Self::EcdheEcdsaChacha20), - "ECDHE-RSA-AES128-GCM-SHA256" => Some(Self::EcdheRsaAes128Gcm), - "ECDHE-RSA-AES256-GCM-SHA384" => Some(Self::EcdheRsaAes256Gcm), - "ECDHE-RSA-CHACHA20-POLY1305" => Some(Self::EcdheRsaChacha20), - _ => None, + "ECDHE-ECDSA-AES128-GCM-SHA256" => Ok(Self::EcdheEcdsaAes128Gcm), + "ECDHE-ECDSA-AES256-GCM-SHA384" => Ok(Self::EcdheEcdsaAes256Gcm), + "ECDHE-ECDSA-CHACHA20-POLY1305" => Ok(Self::EcdheEcdsaChacha20), + "ECDHE-RSA-AES128-GCM-SHA256" => Ok(Self::EcdheRsaAes128Gcm), + "ECDHE-RSA-AES256-GCM-SHA384" => Ok(Self::EcdheRsaAes256Gcm), + "ECDHE-RSA-CHACHA20-POLY1305" => Ok(Self::EcdheRsaChacha20), + _ => Err(Unsupported::CipherToken(name.to_owned())), } } +} +impl Tls12CipherSuite { fn suite(self) -> CipherSuite { match self { Self::EcdheEcdsaAes128Gcm => CipherSuite::TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, @@ -101,46 +109,47 @@ enum CipherToken { Unsupported(Unsupported), } -fn cipher_token(token: &str) -> CipherToken { - if let Some(suite) = Tls12CipherSuite::from_openssl_name(token) { - return CipherToken::Suite(suite); - } - match token { - "DEFAULT" | "ALL" | "HIGH" => CipherToken::EverySuite, - "@STRENGTH" | "@SECLEVEL=2" => CipherToken::Ordering, - level if level.starts_with("@SECLEVEL=") => { - CipherToken::Unsupported(Unsupported::SecurityLevel(level.to_owned())) +impl From<&str> for CipherToken { + fn from(token: &str) -> Self { + match token { + "DEFAULT" | "ALL" | "HIGH" => Self::EverySuite, + "@STRENGTH" | "@SECLEVEL=2" => Self::Ordering, + level if level.starts_with("@SECLEVEL=") => { + Self::Unsupported(Unsupported::SecurityLevel(level.to_owned())) + } + name => name.parse().map_or_else(Self::Unsupported, Self::Suite), } - other => CipherToken::Unsupported(Unsupported::CipherToken(other.to_owned())), } } -pub(crate) fn parse_cipher_string(value: &str) -> CipherSelection { - let tokens: Vec = tokenize(value) - .iter() - .map(|token| cipher_token(token)) - .collect(); - let every_suite = tokens - .iter() - .any(|token| matches!(token, CipherToken::EverySuite)); - let mut suites: Vec = tokens - .iter() - .filter_map(|token| match token { - CipherToken::Suite(suite) => Some(*suite), - _ => None, - }) - .collect(); - suites.sort_unstable(); - suites.dedup(); - CipherSelection { - tls12_cipher_suites: (!every_suite && !suites.is_empty()).then_some(suites), - unsupported: tokens - .into_iter() +impl From<&str> for CipherSelection { + fn from(value: &str) -> Self { + let tokens: Vec = tokenize(value) + .iter() + .map(|token| CipherToken::from(token.as_str())) + .collect(); + let every_suite = tokens + .iter() + .any(|token| matches!(token, CipherToken::EverySuite)); + let mut suites: Vec = tokens + .iter() .filter_map(|token| match token { - CipherToken::Unsupported(unsupported) => Some(unsupported), + CipherToken::Suite(suite) => Some(*suite), _ => None, }) - .collect(), + .collect(); + suites.sort_unstable(); + suites.dedup(); + CipherSelection { + tls12_cipher_suites: (!every_suite && !suites.is_empty()).then_some(suites), + unsupported: tokens + .into_iter() + .filter_map(|token| match token { + CipherToken::Unsupported(unsupported) => Some(unsupported), + _ => None, + }) + .collect(), + } } } @@ -155,57 +164,56 @@ fn tokenize(value: &str) -> Vec { .collect() } -pub fn client_config(config: &HttpClientConfig) -> Result { - let base = ring::default_provider(); - let provider = Arc::new(CryptoProvider { - kx_groups: config - .key_exchange_group - .map_or_else(|| base.kx_groups.clone(), |group| vec![group.supported()]), - cipher_suites: base - .cipher_suites - .iter() - .copied() - .filter(|suite| { - suite.tls13().is_some() - || config - .tls12_cipher_suites - .as_ref() - .is_none_or(|allowed| allowed.iter().any(|a| a.suite() == suite.suite())) - }) - .collect(), - ..base - }); - let builder = ClientConfig::builder_with_provider(Arc::clone(&provider)) - .with_safe_default_protocol_versions() - .map_err(|error| Error::Client(error.to_string()))?; - let verified = match &config.verify { - Verify::Disabled => builder - .dangerous() - .with_custom_certificate_verifier(Arc::new(NoVerification(provider))), - Verify::BuiltInRoots => builder.with_root_certificates(built_in_roots()), - Verify::CaBundle(path) => builder.with_root_certificates(bundle_roots(path)?), - }; - let mut tls = match &config.client_certificate { - None => verified.with_no_client_auth(), - Some(path) => { - let (chain, key) = identity(path)?; - verified - .with_client_auth_cert(chain, key) - .map_err(|error| invalid_pem(path, error))? - } - }; - tls.alpn_protocols = if config.http2 { - vec![b"h2".to_vec(), b"http/1.1".to_vec()] - } else { - vec![b"http/1.1".to_vec()] - }; - Ok(tls) -} +impl TryFrom<&HttpClientConfig> for ClientConfig { + type Error = Error; -fn built_in_roots() -> RootCertStore { - let mut store = RootCertStore::empty(); - store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); - store + fn try_from(config: &HttpClientConfig) -> Result { + let base = ring::default_provider(); + let provider = Arc::new(CryptoProvider { + kx_groups: config + .key_exchange_group + .map_or_else(|| base.kx_groups.clone(), |group| vec![group.supported()]), + cipher_suites: base + .cipher_suites + .iter() + .copied() + .filter(|suite| { + suite.tls13().is_some() + || config.tls12_cipher_suites.as_ref().is_none_or(|allowed| { + allowed.iter().any(|a| a.suite() == suite.suite()) + }) + }) + .collect(), + ..base + }); + let builder = ClientConfig::builder_with_provider(Arc::clone(&provider)) + .with_safe_default_protocol_versions() + .map_err(|error| Error::Client(error.to_string()))?; + let verified = match &config.verify { + Verify::Disabled => builder + .dangerous() + .with_custom_certificate_verifier(Arc::new(NoVerification(provider))), + Verify::BuiltInRoots => builder.with_root_certificates(RootCertStore { + roots: webpki_roots::TLS_SERVER_ROOTS.to_vec(), + }), + Verify::CaBundle(path) => builder.with_root_certificates(bundle_roots(path)?), + }; + let mut tls = match &config.client_certificate { + None => verified.with_no_client_auth(), + Some(path) => { + let (chain, key) = identity(path)?; + verified + .with_client_auth_cert(chain, key) + .map_err(|error| invalid_pem(path, error))? + } + }; + tls.alpn_protocols = if config.http2 { + vec![b"h2".to_vec(), b"http/1.1".to_vec()] + } else { + vec![b"http/1.1".to_vec()] + }; + Ok(tls) + } } fn bundle_roots(path: &Path) -> Result { @@ -327,7 +335,7 @@ mod tests { #[case] curve: &str, #[case] expected: NamedGroup, ) { - let tls = client_config(&config(HttpSettings { + let tls = ClientConfig::try_from(&config(HttpSettings { ssl_ecdh_curve: Some(curve.into()), ..HttpSettings::default() })) @@ -337,7 +345,7 @@ mod tests { #[test] fn default_settings_offer_every_group_and_suite_of_the_provider() { - let tls = client_config(&config(HttpSettings::default())).unwrap(); + let tls = ClientConfig::try_from(&config(HttpSettings::default())).unwrap(); let provider = ring::default_provider(); assert_eq!(offered_groups(&tls).len(), provider.kx_groups.len()); assert_eq!( @@ -348,7 +356,7 @@ mod tests { #[test] fn named_suites_are_the_only_tls12_suites_offered_and_tls13_stays() { - let tls = client_config(&config(HttpSettings { + let tls = ClientConfig::try_from(&config(HttpSettings { ssl_security_level: Some("ECDHE-RSA-AES256-GCM-SHA384".into()), ..HttpSettings::default() })) @@ -369,7 +377,7 @@ mod tests { #[case(true, &[b"h2".as_slice(), b"http/1.1".as_slice()])] #[case(false, &[b"http/1.1".as_slice()])] fn alpn_offers_h2_only_when_http2_is_on(#[case] http2: bool, #[case] expected: &[&[u8]]) { - let tls = client_config(&config(HttpSettings { + let tls = ClientConfig::try_from(&config(HttpSettings { http2, ..HttpSettings::default() })) @@ -388,7 +396,7 @@ mod tests { b"-----BEGIN CERTIFICATE-----\nAA==\n-----END CERTIFICATE-----\n", ) .unwrap(); - let result = client_config(&HttpClientConfig { + let result = ClientConfig::try_from(&HttpClientConfig { client_certificate: Some(path.clone()), ..config(HttpSettings::default()) }) From 80dbb2a28a57660bd4c57a55ee4abb90d10a6d2f Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Fri, 18 Sep 2026 19:47:10 -0700 Subject: [PATCH 14/15] refactor(rust): resolve the http client config through From and TryFrom HttpClientConfig::resolve becomes From<&HttpSettings> for Resolution and client_builder becomes TryFrom<&HttpClientConfig> for reqwest::ClientBuilder, matching the rustls conversion. The verify decision moves into From<&HttpSettings> for Verify, and the proxy environment rule moves next to its flags as HttpSettings::trusts_proxy_env. The curve and cipher results are read with transpose and a default selection, which removes the tuple destructuring --- litellm-rust/crates/core/tests/ocr.rs | 4 +- litellm-rust/crates/http/AGENTS.md | 1 + litellm-rust/crates/http/src/config.rs | 112 +++++++++--------- litellm-rust/crates/http/src/pool.rs | 8 +- litellm-rust/crates/http/src/settings.rs | 4 + litellm-rust/crates/http/src/tls.rs | 5 +- .../crates/llms/src/custom_httpx/media.rs | 6 +- litellm-rust/crates/python-bridge/src/http.rs | 10 +- 8 files changed, 78 insertions(+), 72 deletions(-) create mode 100644 litellm-rust/crates/http/AGENTS.md diff --git a/litellm-rust/crates/core/tests/ocr.rs b/litellm-rust/crates/core/tests/ocr.rs index a6b26bd8a27..e7a8fc0abc1 100644 --- a/litellm-rust/crates/core/tests/ocr.rs +++ b/litellm-rust/crates/core/tests/ocr.rs @@ -6,7 +6,7 @@ use litellm_host::{ host::{Host, HostOp, HostResult}, machine::{HostFailure, Machine, MachineStep}, }; -use litellm_http::{HttpClientConfig, HttpClientPool, HttpSettings}; +use litellm_http::{HttpClientPool, HttpSettings, Resolution}; use litellm_llms::{ base_llm::ocr::{ error::Error as OcrError, @@ -184,7 +184,7 @@ async fn ocr_client_uses_the_injected_http_pool_configuration() { }; let client = OcrClient::new( &HttpClientPool::new(Arc::new(PublicDnsResolver)), - &HttpClientConfig::resolve(&settings).config, + &Resolution::from(&settings).config, UrlPolicy::default(), VertexAuth::default(), ) diff --git a/litellm-rust/crates/http/AGENTS.md b/litellm-rust/crates/http/AGENTS.md new file mode 100644 index 00000000000..08fa34bd799 --- /dev/null +++ b/litellm-rust/crates/http/AGENTS.md @@ -0,0 +1 @@ +- https://github.com/BerriAI/litellm-docs/blob/main/docs/guides/security_settings.md diff --git a/litellm-rust/crates/http/src/config.rs b/litellm-rust/crates/http/src/config.rs index 30216405fc8..a6cc08c210d 100644 --- a/litellm-rust/crates/http/src/config.rs +++ b/litellm-rust/crates/http/src/config.rs @@ -38,84 +38,80 @@ pub struct Resolution { pub unsupported: Vec, } -impl HttpClientConfig { - pub fn resolve(settings: &HttpSettings) -> Resolution { - let (key_exchange_group, unsupported_curve) = match settings - .ssl_ecdh_curve - .as_deref() - .map(str::parse::) - { - None => (None, None), - Some(Ok(group)) => (Some(group), None), - Some(Err(unsupported)) => (None, Some(unsupported)), - }; - let ciphers = settings - .ssl_security_level - .as_deref() - .map(CipherSelection::from); - let verify = match &settings.ssl_verify { - Some(SslVerify::Disabled) => Verify::Disabled, - Some(SslVerify::CaBundle(path)) => Verify::CaBundle(path.clone()), +impl From<&HttpSettings> for Verify { + fn from(settings: &HttpSettings) -> Self { + match &settings.ssl_verify { + Some(SslVerify::Disabled) => Self::Disabled, + Some(SslVerify::CaBundle(path)) => Self::CaBundle(path.clone()), Some(SslVerify::Enabled) | None => settings .ssl_cert_file .clone() - .map_or(Verify::BuiltInRoots, Verify::CaBundle), - }; - let (tls12_cipher_suites, unsupported_ciphers) = ciphers - .map_or((None, Vec::new()), |ciphers| { - (ciphers.tls12_cipher_suites, ciphers.unsupported) - }); - Resolution { - config: Self { - verify, + .map_or(Self::BuiltInRoots, Self::CaBundle), + } + } +} + +impl From<&HttpSettings> for Resolution { + fn from(settings: &HttpSettings) -> Self { + let curve = settings + .ssl_ecdh_curve + .as_deref() + .map(str::parse::) + .transpose(); + let ciphers = settings + .ssl_security_level + .as_deref() + .map(CipherSelection::from) + .unwrap_or_default(); + Self { + config: HttpClientConfig { + verify: Verify::from(settings), client_certificate: settings.ssl_certificate.clone(), - key_exchange_group, - tls12_cipher_suites, + key_exchange_group: curve.clone().ok().flatten(), + tls12_cipher_suites: ciphers.tls12_cipher_suites, force_ipv4: settings.force_ipv4, http2: settings.http2, user_agent: settings.user_agent.clone(), - trust_proxy_env: !settings.ignore_proxy_env - || settings.trust_proxy_env - || settings.http2 - || settings.httpx_transport, + trust_proxy_env: settings.trusts_proxy_env(), connect_timeout: settings.connect_timeout, tcp_keepalive: settings.tcp_keepalive, pool_idle_timeout: settings.pool_idle_timeout, }, - unsupported: unsupported_curve - .into_iter() - .chain(unsupported_ciphers) - .collect(), + unsupported: curve.err().into_iter().chain(ciphers.unsupported).collect(), } } +} - pub fn client_builder(&self) -> Result { +impl TryFrom<&HttpClientConfig> for reqwest::ClientBuilder { + type Error = Error; + + fn try_from(config: &HttpClientConfig) -> Result { let base = reqwest::Client::builder() - .use_preconfigured_tls(rustls::ClientConfig::try_from(self)?) - .connect_timeout(self.connect_timeout) - .pool_idle_timeout(self.pool_idle_timeout); - let with_keepalive = match self.tcp_keepalive { + .use_preconfigured_tls(rustls::ClientConfig::try_from(config)?) + .connect_timeout(config.connect_timeout) + .pool_idle_timeout(config.pool_idle_timeout); + let with_keepalive = match config.tcp_keepalive { None => base, Some(keepalive) => base .tcp_keepalive(keepalive.idle) .tcp_keepalive_interval(keepalive.interval) .tcp_keepalive_retries(keepalive.retries), }; - let with_address = if self.force_ipv4 { + let with_address = if config.force_ipv4 { with_keepalive.local_address(IpAddr::V4(Ipv4Addr::UNSPECIFIED)) } else { with_keepalive }; - let with_protocol = if self.http2 { + let with_protocol = if config.http2 { with_address } else { with_address.http1_only() }; - let with_agent = match &self.user_agent { + let with_agent = match &config.user_agent { Some(agent) => with_protocol.user_agent(agent), None => with_protocol, }; - Ok(if self.trust_proxy_env { + Ok(if config.trust_proxy_env { with_agent } else { with_agent.no_proxy() @@ -161,7 +157,7 @@ mod tests { #[case] settings: HttpSettings, #[case] expected: Verify, ) { - let config = HttpClientConfig::resolve(&settings).config; + let config = Resolution::from(&settings).config; assert_eq!(config.verify, expected); } @@ -172,7 +168,7 @@ mod tests { ..HttpSettings::default() } .with_environment(&|name: &str| (name == "SSL_VERIFY").then(|| "true".to_string())); - let config = HttpClientConfig::resolve(&settings).config; + let config = Resolution::from(&settings).config; assert_eq!(config.verify, Verify::BuiltInRoots); } @@ -188,7 +184,7 @@ mod tests { ssl_ecdh_curve: Some(curve.into()), ..HttpSettings::default() }; - let resolution = HttpClientConfig::resolve(&settings); + let resolution = Resolution::from(&settings); assert_eq!(resolution.config.key_exchange_group, expected); assert_eq!(resolution.unsupported, []); } @@ -199,7 +195,7 @@ mod tests { ssl_ecdh_curve: Some("secp521r1".into()), ..HttpSettings::default() }; - let resolution = HttpClientConfig::resolve(&settings); + let resolution = Resolution::from(&settings); assert_eq!(resolution.config.key_exchange_group, None); assert_eq!( resolution.unsupported, @@ -213,7 +209,7 @@ mod tests { ssl_security_level: Some("DEFAULT@SECLEVEL=1".into()), ..HttpSettings::default() }; - let resolution = HttpClientConfig::resolve(&settings); + let resolution = Resolution::from(&settings); assert_eq!(resolution.config.tls12_cipher_suites, None); assert_eq!( resolution.unsupported, @@ -230,7 +226,7 @@ mod tests { ), ..HttpSettings::default() }; - let resolution = HttpClientConfig::resolve(&settings); + let resolution = Resolution::from(&settings); assert_eq!( resolution.config.tls12_cipher_suites, Some(vec![ @@ -265,7 +261,7 @@ mod tests { pool_idle_timeout: Duration::from_secs(45), ..HttpSettings::default() }; - let config = HttpClientConfig::resolve(&settings).config; + let config = Resolution::from(&settings).config; assert_eq!( config, HttpClientConfig { @@ -303,7 +299,7 @@ mod tests { #[case] settings: HttpSettings, #[case] expected: bool, ) { - let config = HttpClientConfig::resolve(&settings).config; + let config = Resolution::from(&settings).config; assert_eq!(config.trust_proxy_env, expected); } @@ -312,10 +308,10 @@ mod tests { let path = std::env::temp_dir().join("litellm-http-missing-bundle.pem"); let config = HttpClientConfig { verify: Verify::CaBundle(path.clone()), - ..HttpClientConfig::resolve(&HttpSettings::default()).config + ..Resolution::from(&HttpSettings::default()).config }; assert!(matches!( - config.client_builder(), + reqwest::ClientBuilder::try_from(&config), Err(Error::Read { path: reported, .. }) if reported == path )); } @@ -327,9 +323,9 @@ mod tests { std::fs::write(&path, b"not a certificate").unwrap(); let config = HttpClientConfig { verify: Verify::CaBundle(path.clone()), - ..HttpClientConfig::resolve(&HttpSettings::default()).config + ..Resolution::from(&HttpSettings::default()).config }; - let result = config.client_builder().map(drop); + let result = reqwest::ClientBuilder::try_from(&config).map(drop); std::fs::remove_file(&path).unwrap(); assert!(matches!( result, diff --git a/litellm-rust/crates/http/src/pool.rs b/litellm-rust/crates/http/src/pool.rs index e6e0de9bc5f..330d6de29e8 100644 --- a/litellm-rust/crates/http/src/pool.rs +++ b/litellm-rust/crates/http/src/pool.rs @@ -67,7 +67,9 @@ impl HttpClientPool { { return Ok(pooled.client.clone()); } - let client = self.apply(variant, key.0.client_builder()?).build()?; + let client = self + .apply(variant, reqwest::ClientBuilder::try_from(&key.0)?) + .build()?; self.lock().insert( key, PooledClient { @@ -114,7 +116,7 @@ mod tests { }; use super::*; - use crate::{HttpSettings, Verify}; + use crate::{HttpSettings, Resolution, Verify}; struct FixedResolver(SocketAddr); @@ -132,7 +134,7 @@ mod tests { fn config(user_agent: &str) -> HttpClientConfig { HttpClientConfig { user_agent: Some(user_agent.into()), - ..HttpClientConfig::resolve(&HttpSettings::default()).config + ..Resolution::from(&HttpSettings::default()).config } } diff --git a/litellm-rust/crates/http/src/settings.rs b/litellm-rust/crates/http/src/settings.rs index be2f4f42fb4..6d7bf934e4b 100644 --- a/litellm-rust/crates/http/src/settings.rs +++ b/litellm-rust/crates/http/src/settings.rs @@ -111,6 +111,10 @@ impl HttpSettings { } } + pub fn trusts_proxy_env(&self) -> bool { + !self.ignore_proxy_env || self.trust_proxy_env || self.http2 || self.httpx_transport + } + pub fn without_missing_files(self, exists: &dyn Fn(&Path) -> bool) -> Self { Self { ssl_verify: match self.ssl_verify { diff --git a/litellm-rust/crates/http/src/tls.rs b/litellm-rust/crates/http/src/tls.rs index 49405b97366..aaae2b659e3 100644 --- a/litellm-rust/crates/http/src/tls.rs +++ b/litellm-rust/crates/http/src/tls.rs @@ -97,6 +97,7 @@ pub enum Unsupported { CipherToken(String), } +#[derive(Default)] pub(crate) struct CipherSelection { pub(crate) tls12_cipher_suites: Option>, pub(crate) unsupported: Vec, @@ -304,10 +305,10 @@ mod tests { use rustls::NamedGroup; use super::*; - use crate::HttpSettings; + use crate::{HttpSettings, Resolution}; fn config(settings: HttpSettings) -> HttpClientConfig { - HttpClientConfig::resolve(&settings).config + Resolution::from(&settings).config } fn offered_groups(tls: &ClientConfig) -> Vec { diff --git a/litellm-rust/crates/llms/src/custom_httpx/media.rs b/litellm-rust/crates/llms/src/custom_httpx/media.rs index 02d152d3ef1..572e7f12e54 100644 --- a/litellm-rust/crates/llms/src/custom_httpx/media.rs +++ b/litellm-rust/crates/llms/src/custom_httpx/media.rs @@ -350,7 +350,7 @@ impl Resolve for PublicDnsResolver { mod tests { use std::collections::HashSet; - use litellm_http::HttpSettings; + use litellm_http::{HttpSettings, Resolution}; use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, net::TcpListener, @@ -445,7 +445,7 @@ mod tests { ) -> MediaFetcher { let direct = HttpClientConfig { trust_proxy_env: false, - ..HttpClientConfig::resolve(&HttpSettings::default()).config + ..Resolution::from(&HttpSettings::default()).config }; MediaFetcher::with_resolution( &HttpClientPool::new(Arc::new(LoopbackDnsResolver(pinned_address))), @@ -636,7 +636,7 @@ mod tests { async fn rejects_url_credentials_before_network_access() { let fetcher = MediaFetcher::new( &HttpClientPool::new(Arc::new(PublicDnsResolver)), - &HttpClientConfig::resolve(&HttpSettings::default()).config, + &Resolution::from(&HttpSettings::default()).config, UrlPolicy::default(), ) .expect("media fetcher builds"); diff --git a/litellm-rust/crates/python-bridge/src/http.rs b/litellm-rust/crates/python-bridge/src/http.rs index 118f4669b62..385f78be9fa 100644 --- a/litellm-rust/crates/python-bridge/src/http.rs +++ b/litellm-rust/crates/python-bridge/src/http.rs @@ -4,7 +4,9 @@ use std::{ sync::{Arc, LazyLock, Mutex, PoisonError}, }; -use litellm_http::{HttpClientConfig, HttpClientPool, HttpSettings, SslVerify, Unsupported}; +use litellm_http::{ + HttpClientConfig, HttpClientPool, HttpSettings, Resolution, SslVerify, Unsupported, +}; use litellm_llms::custom_httpx::media::{PublicDnsResolver, UrlPolicy}; use pyo3::{prelude::*, types::PyDict}; @@ -28,7 +30,7 @@ pub(crate) fn call_config( .with_environment(&|name| std::env::var(name).ok()); let settings = for_call(configured, call_ssl_verify(kwargs)?, asynchronous) .without_missing_files(&|path: &Path| path.exists()); - let resolution = HttpClientConfig::resolve(&settings); + let resolution = Resolution::from(&settings); for unsupported in unreported(&REPORTED_UNSUPPORTED, resolution.unsupported) { PythonSettings::warn(py, &unsupported.to_string())?; } @@ -251,7 +253,7 @@ user_agent='litellm/9.9.9', Python::initialize(); Python::attach(|py| { let settings = settings(&python_settings(py, overrides)).unwrap(); - let config = HttpClientConfig::resolve(&settings).config; + let config = Resolution::from(&settings).config; assert_eq!(config.verify, expected); }); } @@ -345,7 +347,7 @@ user_agent='litellm/9.9.9', ..HttpSettings::default() }; let settings = for_call(opted_out, None, asynchronous); - let config = HttpClientConfig::resolve(&settings).config; + let config = Resolution::from(&settings).config; assert_eq!(config.trust_proxy_env, expected); } } From bae4f22d3a201bf6c3d84c6017b44851dc5d07f0 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Fri, 18 Sep 2026 19:53:14 -0700 Subject: [PATCH 15/15] refactor(rust): merge http settings from per-source layers Each source (per-call kwargs, environment variables, the Python module) now builds an HttpSettingsLayer, and HttpSettings::from_layers merges them with explicit precedence. The aiohttp and httpx proxy-env rule is resolved once in the merge, so HttpSettings carries a single trust_proxy_env flag --- litellm-rust/crates/http/src/config.rs | 41 +-- litellm-rust/crates/http/src/lib.rs | 2 +- litellm-rust/crates/http/src/settings.rs | 286 +++++++++++++----- litellm-rust/crates/python-bridge/src/http.rs | 137 ++++----- 4 files changed, 278 insertions(+), 188 deletions(-) diff --git a/litellm-rust/crates/http/src/config.rs b/litellm-rust/crates/http/src/config.rs index a6cc08c210d..10f28b44eec 100644 --- a/litellm-rust/crates/http/src/config.rs +++ b/litellm-rust/crates/http/src/config.rs @@ -72,7 +72,7 @@ impl From<&HttpSettings> for Resolution { force_ipv4: settings.force_ipv4, http2: settings.http2, user_agent: settings.user_agent.clone(), - trust_proxy_env: settings.trusts_proxy_env(), + trust_proxy_env: settings.trust_proxy_env, connect_timeout: settings.connect_timeout, tcp_keepalive: settings.tcp_keepalive, pool_idle_timeout: settings.pool_idle_timeout, @@ -125,17 +125,12 @@ mod tests { use super::*; - fn no_env(_: &str) -> Option { - None - } - fn settings(ssl_verify: Option, ssl_cert_file: Option<&str>) -> HttpSettings { HttpSettings { ssl_verify, ssl_cert_file: ssl_cert_file.map(PathBuf::from), ..HttpSettings::default() } - .with_environment(&no_env) } #[rstest] @@ -161,17 +156,6 @@ mod tests { assert_eq!(config.verify, expected); } - #[test] - fn ssl_verify_environment_variable_beats_the_configured_setting() { - let settings = HttpSettings { - ssl_verify: Some(SslVerify::Disabled), - ..HttpSettings::default() - } - .with_environment(&|name: &str| (name == "SSL_VERIFY").then(|| "true".to_string())); - let config = Resolution::from(&settings).config; - assert_eq!(config.verify, Verify::BuiltInRoots); - } - #[rstest] #[case::x25519("X25519", Some(KeyExchangeGroup::X25519))] #[case::openssl_p256("prime256v1", Some(KeyExchangeGroup::Secp256r1))] @@ -280,29 +264,6 @@ mod tests { ); } - #[rstest] - #[case::aiohttp_default(HttpSettings::default(), true)] - #[case::aiohttp_opted_out(HttpSettings { ignore_proxy_env: true, ..HttpSettings::default() }, false)] - #[case::session_trust_env_beats_opt_out( - HttpSettings { ignore_proxy_env: true, trust_proxy_env: true, ..HttpSettings::default() }, - true - )] - #[case::http2_uses_httpx( - HttpSettings { ignore_proxy_env: true, http2: true, ..HttpSettings::default() }, - true - )] - #[case::aiohttp_disabled( - HttpSettings { ignore_proxy_env: true, httpx_transport: true, ..HttpSettings::default() }, - true - )] - fn environment_proxies_apply_unless_the_aiohttp_transport_opts_out( - #[case] settings: HttpSettings, - #[case] expected: bool, - ) { - let config = Resolution::from(&settings).config; - assert_eq!(config.trust_proxy_env, expected); - } - #[test] fn missing_ca_bundle_is_a_read_error() { let path = std::env::temp_dir().join("litellm-http-missing-bundle.pem"); diff --git a/litellm-rust/crates/http/src/lib.rs b/litellm-rust/crates/http/src/lib.rs index e222d0e3f50..ddbc3b63b08 100644 --- a/litellm-rust/crates/http/src/lib.rs +++ b/litellm-rust/crates/http/src/lib.rs @@ -9,5 +9,5 @@ pub use config::{HttpClientConfig, Resolution, Verify}; pub use error::Error; pub use pool::{ClientVariant, HttpClientPool}; pub use proxy::EnvironmentProxies; -pub use settings::{HttpSettings, SslVerify, TcpKeepalive}; +pub use settings::{HttpSettings, HttpSettingsLayer, SslVerify, TcpKeepalive}; pub use tls::{KeyExchangeGroup, Tls12CipherSuite, Unsupported}; diff --git a/litellm-rust/crates/http/src/settings.rs b/litellm-rust/crates/http/src/settings.rs index 6d7bf934e4b..8ac7ef92568 100644 --- a/litellm-rust/crates/http/src/settings.rs +++ b/litellm-rust/crates/http/src/settings.rs @@ -27,6 +27,79 @@ pub struct TcpKeepalive { pub retries: u32, } +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct HttpSettingsLayer { + pub ssl_verify: Option, + pub ssl_cert_file: Option, + pub ssl_certificate: Option, + pub ssl_security_level: Option, + pub ssl_ecdh_curve: Option, + pub force_ipv4: Option, + pub http2: Option, + pub aiohttp_trust_env: Option, + pub disable_aiohttp_trust_env: Option, + pub disable_aiohttp_transport: Option, + pub user_agent: Option, + pub tcp_keepalive: Option, + pub pool_idle_timeout: Option, +} + +impl HttpSettingsLayer { + pub fn from_environment(env: &(dyn Fn(&str) -> Option + Sync)) -> Self { + let enabled = |name: &str| { + env(name) + .is_some_and(|value| value.trim().eq_ignore_ascii_case("true")) + .then_some(true) + }; + let number = |name: &str| env(name).and_then(|value| value.trim().parse::().ok()); + let seconds = |name: &str, default: u32| { + Duration::from_secs(u64::from(number(name).unwrap_or(default))) + }; + Self { + ssl_verify: env("SSL_VERIFY").map(|value| SslVerify::parse(&value)), + ssl_cert_file: env("SSL_CERT_FILE").map(PathBuf::from), + ssl_certificate: env("SSL_CERTIFICATE").map(PathBuf::from), + ssl_security_level: env("SSL_SECURITY_LEVEL"), + ssl_ecdh_curve: env("SSL_ECDH_CURVE"), + force_ipv4: None, + http2: enabled("LITELLM_HTTP2"), + aiohttp_trust_env: enabled("AIOHTTP_TRUST_ENV"), + disable_aiohttp_trust_env: enabled("DISABLE_AIOHTTP_TRUST_ENV"), + disable_aiohttp_transport: enabled("DISABLE_AIOHTTP_TRANSPORT"), + user_agent: env("LITELLM_USER_AGENT"), + tcp_keepalive: enabled("AIOHTTP_SO_KEEPALIVE").map(|_| TcpKeepalive { + idle: seconds("AIOHTTP_TCP_KEEPIDLE", 60), + interval: seconds("AIOHTTP_TCP_KEEPINTVL", 30), + retries: number("AIOHTTP_TCP_KEEPCNT").unwrap_or(5), + }), + pool_idle_timeout: number("AIOHTTP_KEEPALIVE_TIMEOUT") + .map(|timeout| Duration::from_secs(u64::from(timeout))), + } + } + + fn or(self, lower: Self) -> Self { + Self { + ssl_verify: self.ssl_verify.or(lower.ssl_verify), + ssl_cert_file: self.ssl_cert_file.or(lower.ssl_cert_file), + ssl_certificate: self.ssl_certificate.or(lower.ssl_certificate), + ssl_security_level: self.ssl_security_level.or(lower.ssl_security_level), + ssl_ecdh_curve: self.ssl_ecdh_curve.or(lower.ssl_ecdh_curve), + force_ipv4: self.force_ipv4.or(lower.force_ipv4), + http2: self.http2.or(lower.http2), + aiohttp_trust_env: self.aiohttp_trust_env.or(lower.aiohttp_trust_env), + disable_aiohttp_trust_env: self + .disable_aiohttp_trust_env + .or(lower.disable_aiohttp_trust_env), + disable_aiohttp_transport: self + .disable_aiohttp_transport + .or(lower.disable_aiohttp_transport), + user_agent: self.user_agent.or(lower.user_agent), + tcp_keepalive: self.tcp_keepalive.or(lower.tcp_keepalive), + pool_idle_timeout: self.pool_idle_timeout.or(lower.pool_idle_timeout), + } + } +} + #[derive(Clone, Debug, PartialEq, Eq)] pub struct HttpSettings { pub ssl_verify: Option, @@ -36,10 +109,8 @@ pub struct HttpSettings { pub ssl_ecdh_curve: Option, pub force_ipv4: bool, pub http2: bool, - pub httpx_transport: bool, pub user_agent: Option, pub trust_proxy_env: bool, - pub ignore_proxy_env: bool, pub connect_timeout: Duration, pub tcp_keepalive: Option, pub pool_idle_timeout: Duration, @@ -55,10 +126,8 @@ impl Default for HttpSettings { ssl_ecdh_curve: None, force_ipv4: false, http2: false, - httpx_transport: false, user_agent: None, - trust_proxy_env: false, - ignore_proxy_env: false, + trust_proxy_env: true, connect_timeout: Duration::from_secs(10), tcp_keepalive: None, pool_idle_timeout: Duration::from_secs(120), @@ -67,54 +136,38 @@ impl Default for HttpSettings { } impl HttpSettings { - pub fn with_environment(self, env: &(dyn Fn(&str) -> Option + Sync)) -> Self { - let enabled = - |name: &str| env(name).is_some_and(|value| value.trim().eq_ignore_ascii_case("true")); - let number = |name: &str| env(name).and_then(|value| value.trim().parse::().ok()); - let seconds = |name: &str, default: u32| { - Duration::from_secs(u64::from(number(name).unwrap_or(default))) - }; + pub fn from_layers( + highest_precedence_first: impl IntoIterator, + ) -> Self { + let merged = highest_precedence_first + .into_iter() + .reduce(HttpSettingsLayer::or) + .unwrap_or_default(); + let defaults = Self::default(); + let http2 = merged.http2.unwrap_or(defaults.http2); Self { - ssl_verify: env("SSL_VERIFY") - .map(|value| SslVerify::parse(&value)) - .or(self.ssl_verify), - ssl_cert_file: env("SSL_CERT_FILE") - .map(PathBuf::from) - .or(self.ssl_cert_file), - ssl_certificate: env("SSL_CERTIFICATE") - .map(PathBuf::from) - .or(self.ssl_certificate) + ssl_verify: merged.ssl_verify, + ssl_cert_file: merged.ssl_cert_file, + ssl_certificate: merged + .ssl_certificate .filter(|path| !path.as_os_str().is_empty()), - ssl_security_level: env("SSL_SECURITY_LEVEL") - .or(self.ssl_security_level) - .filter(|level| !level.is_empty()), - ssl_ecdh_curve: env("SSL_ECDH_CURVE") - .or(self.ssl_ecdh_curve) - .filter(|curve| !curve.is_empty()), - http2: self.http2 || enabled("LITELLM_HTTP2"), - httpx_transport: self.httpx_transport || enabled("DISABLE_AIOHTTP_TRANSPORT"), - user_agent: env("LITELLM_USER_AGENT").or(self.user_agent), - trust_proxy_env: self.trust_proxy_env || enabled("AIOHTTP_TRUST_ENV"), - ignore_proxy_env: self.ignore_proxy_env || enabled("DISABLE_AIOHTTP_TRUST_ENV"), - tcp_keepalive: enabled("AIOHTTP_SO_KEEPALIVE") - .then(|| TcpKeepalive { - idle: seconds("AIOHTTP_TCP_KEEPIDLE", 60), - interval: seconds("AIOHTTP_TCP_KEEPINTVL", 30), - retries: number("AIOHTTP_TCP_KEEPCNT").unwrap_or(5), - }) - .or(self.tcp_keepalive), - pool_idle_timeout: number("AIOHTTP_KEEPALIVE_TIMEOUT") - .map_or(self.pool_idle_timeout, |timeout| { - Duration::from_secs(u64::from(timeout)) - }), - ..self + ssl_security_level: merged.ssl_security_level.filter(|level| !level.is_empty()), + ssl_ecdh_curve: merged.ssl_ecdh_curve.filter(|curve| !curve.is_empty()), + force_ipv4: merged.force_ipv4.unwrap_or(defaults.force_ipv4), + http2, + user_agent: merged.user_agent, + trust_proxy_env: !merged.disable_aiohttp_trust_env.unwrap_or(false) + || merged.aiohttp_trust_env.unwrap_or(false) + || merged.disable_aiohttp_transport.unwrap_or(false) + || http2, + tcp_keepalive: merged.tcp_keepalive, + pool_idle_timeout: merged + .pool_idle_timeout + .unwrap_or(defaults.pool_idle_timeout), + ..defaults } } - pub fn trusts_proxy_env(&self) -> bool { - !self.ignore_proxy_env || self.trust_proxy_env || self.http2 || self.httpx_transport - } - pub fn without_missing_files(self, exists: &dyn Fn(&Path) -> bool) -> Self { Self { ssl_verify: match self.ssl_verify { @@ -161,15 +214,15 @@ mod tests { } #[test] - fn environment_overrides_configured_ssl_values() { - let settings = HttpSettings { + fn higher_layers_override_lower_ones() { + let configured = HttpSettingsLayer { ssl_verify: Some(SslVerify::Enabled), ssl_certificate: Some("/configured/client.pem".into()), ssl_security_level: Some("configured".into()), user_agent: Some("configured/1".into()), - ..HttpSettings::default() - } - .with_environment(&env_of(&[ + ..HttpSettingsLayer::default() + }; + let environment = HttpSettingsLayer::from_environment(&env_of(&[ ("SSL_VERIFY", "false"), ("SSL_CERT_FILE", "/env/roots.pem"), ("SSL_CERTIFICATE", "/env/client.pem"), @@ -177,6 +230,7 @@ mod tests { ("SSL_ECDH_CURVE", "X25519"), ("LITELLM_USER_AGENT", "env/2"), ])); + let settings = HttpSettings::from_layers([environment, configured]); assert_eq!(settings.ssl_verify, Some(SslVerify::Disabled)); assert_eq!(settings.ssl_cert_file, Some("/env/roots.pem".into())); assert_eq!(settings.ssl_certificate, Some("/env/client.pem".into())); @@ -189,30 +243,60 @@ mod tests { } #[test] - fn missing_environment_keeps_configured_values() { - let configured = HttpSettings { - ssl_verify: Some(SslVerify::CaBundle("/configured/roots.pem".into())), - http2: true, - trust_proxy_env: true, - user_agent: Some("configured/1".into()), - ..HttpSettings::default() + fn an_explicit_false_in_a_higher_layer_beats_a_lower_true() { + let higher = HttpSettingsLayer { + http2: Some(false), + force_ipv4: Some(false), + ..HttpSettingsLayer::default() }; - assert_eq!(configured.clone().with_environment(&no_env), configured); + let lower = HttpSettingsLayer { + http2: Some(true), + force_ipv4: Some(true), + ..HttpSettingsLayer::default() + }; + let settings = HttpSettings::from_layers([higher, lower]); + assert!(!settings.http2); + assert!(!settings.force_ipv4); + } + + #[test] + fn an_empty_environment_is_an_empty_layer_so_lower_layers_and_defaults_apply() { + assert_eq!( + HttpSettingsLayer::from_environment(&no_env), + HttpSettingsLayer::default() + ); + let configured = HttpSettingsLayer { + ssl_verify: Some(SslVerify::CaBundle("/configured/roots.pem".into())), + http2: Some(true), + user_agent: Some("configured/1".into()), + ..HttpSettingsLayer::default() + }; + assert_eq!( + HttpSettings::from_layers([HttpSettingsLayer::default(), configured]), + HttpSettings { + ssl_verify: Some(SslVerify::CaBundle("/configured/roots.pem".into())), + http2: true, + user_agent: Some("configured/1".into()), + ..HttpSettings::default() + } + ); + assert_eq!(HttpSettings::from_layers([]), HttpSettings::default()); } #[test] fn empty_environment_values_clear_the_setting_like_python_truthiness() { - let settings = HttpSettings { + let configured = HttpSettingsLayer { ssl_certificate: Some("/configured/client.pem".into()), ssl_security_level: Some("configured".into()), ssl_ecdh_curve: Some("X25519".into()), - ..HttpSettings::default() - } - .with_environment(&env_of(&[ + ..HttpSettingsLayer::default() + }; + let environment = HttpSettingsLayer::from_environment(&env_of(&[ ("SSL_CERTIFICATE", ""), ("SSL_SECURITY_LEVEL", ""), ("SSL_ECDH_CURVE", ""), ])); + let settings = HttpSettings::from_layers([environment, configured]); assert_eq!(settings.ssl_certificate, None); assert_eq!(settings.ssl_security_level, None); assert_eq!(settings.ssl_ecdh_curve, None); @@ -220,11 +304,11 @@ mod tests { #[test] fn socket_keepalive_follows_the_aiohttp_variables_with_python_defaults() { - let tuned = HttpSettings::default().with_environment(&env_of(&[ + let tuned = HttpSettings::from_layers([HttpSettingsLayer::from_environment(&env_of(&[ ("AIOHTTP_SO_KEEPALIVE", "True"), ("AIOHTTP_TCP_KEEPIDLE", "45"), ("AIOHTTP_KEEPALIVE_TIMEOUT", "30"), - ])); + ]))]); assert_eq!( tuned.tcp_keepalive, Some(TcpKeepalive { @@ -238,12 +322,53 @@ mod tests { #[test] fn socket_keepalive_stays_off_unless_enabled() { - let settings = - HttpSettings::default().with_environment(&env_of(&[("AIOHTTP_TCP_KEEPIDLE", "45")])); + let settings = HttpSettings::from_layers([HttpSettingsLayer::from_environment(&env_of( + &[("AIOHTTP_TCP_KEEPIDLE", "45")], + ))]); assert_eq!(settings.tcp_keepalive, None); assert_eq!(settings.pool_idle_timeout, Duration::from_secs(120)); } + fn proxy_flags( + aiohttp_trust_env: bool, + disable_aiohttp_trust_env: bool, + disable_aiohttp_transport: bool, + http2: bool, + ) -> HttpSettingsLayer { + HttpSettingsLayer { + aiohttp_trust_env: Some(aiohttp_trust_env), + disable_aiohttp_trust_env: Some(disable_aiohttp_trust_env), + disable_aiohttp_transport: Some(disable_aiohttp_transport), + http2: Some(http2), + ..HttpSettingsLayer::default() + } + } + + #[rstest] + #[case::aiohttp_default(proxy_flags(false, false, false, false), true)] + #[case::aiohttp_opted_out(proxy_flags(false, true, false, false), false)] + #[case::session_trust_env_beats_opt_out(proxy_flags(true, true, false, false), true)] + #[case::http2_uses_httpx(proxy_flags(false, true, false, true), true)] + #[case::aiohttp_disabled(proxy_flags(false, true, true, false), true)] + fn environment_proxies_apply_unless_the_aiohttp_transport_opts_out( + #[case] layer: HttpSettingsLayer, + #[case] expected: bool, + ) { + assert_eq!(HttpSettings::from_layers([layer]).trust_proxy_env, expected); + } + + #[test] + fn a_proxy_opt_out_in_one_source_still_yields_to_trust_env_from_another() { + let environment = + HttpSettingsLayer::from_environment(&env_of(&[("DISABLE_AIOHTTP_TRUST_ENV", "true")])); + let configured = HttpSettingsLayer { + aiohttp_trust_env: Some(true), + ..HttpSettingsLayer::default() + }; + assert!(!HttpSettings::from_layers([environment.clone()]).trust_proxy_env); + assert!(HttpSettings::from_layers([environment, configured]).trust_proxy_env); + } + #[test] fn missing_files_fall_back_to_default_verification() { let settings = HttpSettings { @@ -267,11 +392,14 @@ mod tests { } #[rstest] - #[case("true", true)] - #[case("True", true)] - #[case("false", false)] - #[case("1", false)] - fn boolean_switches_only_turn_on_for_true(#[case] value: &'static str, #[case] expected: bool) { + #[case("true", Some(true))] + #[case("True", Some(true))] + #[case("false", None)] + #[case("1", None)] + fn boolean_switches_only_turn_on_for_true( + #[case] value: &'static str, + #[case] expected: Option, + ) { let env = move |name: &str| match name { "LITELLM_HTTP2" | "AIOHTTP_TRUST_ENV" @@ -279,10 +407,10 @@ mod tests { | "DISABLE_AIOHTTP_TRUST_ENV" => Some(value.to_string()), _ => None, }; - let settings = HttpSettings::default().with_environment(&env); - assert_eq!(settings.http2, expected); - assert_eq!(settings.httpx_transport, expected); - assert_eq!(settings.trust_proxy_env, expected); - assert_eq!(settings.ignore_proxy_env, expected); + let layer = HttpSettingsLayer::from_environment(&env); + assert_eq!(layer.http2, expected); + assert_eq!(layer.aiohttp_trust_env, expected); + assert_eq!(layer.disable_aiohttp_transport, expected); + assert_eq!(layer.disable_aiohttp_trust_env, expected); } } diff --git a/litellm-rust/crates/python-bridge/src/http.rs b/litellm-rust/crates/python-bridge/src/http.rs index 385f78be9fa..d174dccaa56 100644 --- a/litellm-rust/crates/python-bridge/src/http.rs +++ b/litellm-rust/crates/python-bridge/src/http.rs @@ -5,7 +5,8 @@ use std::{ }; use litellm_http::{ - HttpClientConfig, HttpClientPool, HttpSettings, Resolution, SslVerify, Unsupported, + HttpClientConfig, HttpClientPool, HttpSettings, HttpSettingsLayer, Resolution, SslVerify, + Unsupported, }; use litellm_llms::custom_httpx::media::{PublicDnsResolver, UrlPolicy}; use pyo3::{prelude::*, types::PyDict}; @@ -26,10 +27,12 @@ pub(crate) fn call_config( kwargs: &Bound<'_, PyDict>, asynchronous: bool, ) -> PyResult { - let configured = settings(&PythonSettings::Http.read(py)?)? - .with_environment(&|name| std::env::var(name).ok()); - let settings = for_call(configured, call_ssl_verify(kwargs)?, asynchronous) - .without_missing_files(&|path: &Path| path.exists()); + let settings = HttpSettings::from_layers([ + for_call(call_ssl_verify(kwargs)?, asynchronous), + HttpSettingsLayer::from_environment(&|name| std::env::var(name).ok()), + configured(&PythonSettings::Http.read(py)?)?, + ]) + .without_missing_files(&|path: &Path| path.exists()); let resolution = Resolution::from(&settings); for unsupported in unreported(&REPORTED_UNSUPPORTED, resolution.unsupported) { PythonSettings::warn(py, &unsupported.to_string())?; @@ -70,15 +73,11 @@ fn call_ssl_verify(kwargs: &Bound<'_, PyDict>) -> PyResult> { .and_then(|value| ssl_verify(&value))) } -fn for_call( - configured: HttpSettings, - call_ssl_verify: Option, - asynchronous: bool, -) -> HttpSettings { - HttpSettings { - ssl_verify: call_ssl_verify.or(configured.ssl_verify), - httpx_transport: configured.httpx_transport || !asynchronous, - ..configured +fn for_call(call_ssl_verify: Option, asynchronous: bool) -> HttpSettingsLayer { + HttpSettingsLayer { + ssl_verify: call_ssl_verify, + disable_aiohttp_transport: (!asynchronous).then_some(true), + ..HttpSettingsLayer::default() } } @@ -102,24 +101,24 @@ struct PythonHttpSettings<'py> { user_agent: String, } -fn settings(value: &Bound<'_, PyAny>) -> PyResult { +fn configured(value: &Bound<'_, PyAny>) -> PyResult { let python: PythonHttpSettings = value.extract().map_err(|error: PyErr| { RustBridgeDeclined::new_err(format!( "litellm HTTP settings cannot be used by the Rust route: {error}" )) })?; - Ok(HttpSettings { + Ok(HttpSettingsLayer { ssl_verify: ssl_verify(&python.ssl_verify), ssl_certificate: python.ssl_certificate.map(PathBuf::from), ssl_security_level: python.ssl_security_level, ssl_ecdh_curve: python.ssl_ecdh_curve, - force_ipv4: python.force_ipv4, - http2: python.http2, - httpx_transport: python.disable_aiohttp_transport, + force_ipv4: Some(python.force_ipv4), + http2: Some(python.http2), + aiohttp_trust_env: Some(python.aiohttp_trust_env), + disable_aiohttp_trust_env: Some(python.disable_aiohttp_trust_env), + disable_aiohttp_transport: Some(python.disable_aiohttp_transport), user_agent: Some(python.user_agent), - trust_proxy_env: python.aiohttp_trust_env, - ignore_proxy_env: python.disable_aiohttp_trust_env, - ..HttpSettings::default() + ..HttpSettingsLayer::default() }) } @@ -174,12 +173,12 @@ settings = types.SimpleNamespace(**{{name: defaults[name] for name in json.loads } #[test] - fn default_python_settings_produce_default_settings_with_verification_on() { + fn default_python_settings_resolve_to_default_settings_with_verification_on() { Python::initialize(); Python::attach(|py| { - let settings = settings(&python_settings(py, "")).unwrap(); + let layer = configured(&python_settings(py, "")).unwrap(); assert_eq!( - settings, + HttpSettings::from_layers([layer]), HttpSettings { ssl_verify: Some(SslVerify::Enabled), user_agent: Some("litellm/test".into()), @@ -190,10 +189,10 @@ settings = types.SimpleNamespace(**{{name: defaults[name] for name in json.loads } #[test] - fn python_settings_flow_into_settings() { + fn python_settings_flow_into_the_configured_layer() { Python::initialize(); Python::attach(|py| { - let settings = settings(&python_settings( + let layer = configured(&python_settings( py, " ssl_verify='/etc/ssl/corp.pem', @@ -210,19 +209,19 @@ user_agent='litellm/9.9.9', )) .unwrap(); assert_eq!( - settings, - HttpSettings { + layer, + HttpSettingsLayer { ssl_verify: Some(SslVerify::CaBundle("/etc/ssl/corp.pem".into())), ssl_certificate: Some("/etc/ssl/client.pem".into()), ssl_security_level: Some("2".into()), ssl_ecdh_curve: Some("X25519".into()), - force_ipv4: true, - http2: true, - httpx_transport: true, + force_ipv4: Some(true), + http2: Some(true), + aiohttp_trust_env: Some(true), + disable_aiohttp_trust_env: Some(true), + disable_aiohttp_transport: Some(true), user_agent: Some("litellm/9.9.9".into()), - trust_proxy_env: true, - ignore_proxy_env: true, - ..HttpSettings::default() + ..HttpSettingsLayer::default() } ); }); @@ -232,11 +231,12 @@ user_agent='litellm/9.9.9', fn user_agent_environment_variable_beats_the_python_default() { Python::initialize(); Python::attach(|py| { - let settings = settings(&python_settings(py, "")) - .unwrap() - .with_environment(&|name| { + let settings = HttpSettings::from_layers([ + HttpSettingsLayer::from_environment(&|name| { (name == "LITELLM_USER_AGENT").then(|| "operator/1".to_string()) - }); + }), + configured(&python_settings(py, "")).unwrap(), + ]); assert_eq!(settings.user_agent.as_deref(), Some("operator/1")); }); } @@ -252,8 +252,8 @@ user_agent='litellm/9.9.9', ) { Python::initialize(); Python::attach(|py| { - let settings = settings(&python_settings(py, overrides)).unwrap(); - let config = Resolution::from(&settings).config; + let layer = configured(&python_settings(py, overrides)).unwrap(); + let config = Resolution::from(&HttpSettings::from_layers([layer])).config; assert_eq!(config.verify, expected); }); } @@ -262,8 +262,8 @@ user_agent='litellm/9.9.9', fn ssl_context_global_is_ignored_so_environment_and_defaults_apply() { Python::initialize(); Python::attach(|py| { - let settings = settings(&python_settings(py, "ssl_verify=object()")).unwrap(); - assert_eq!(settings.ssl_verify, None); + let layer = configured(&python_settings(py, "ssl_verify=object()")).unwrap(); + assert_eq!(layer.ssl_verify, None); }); } @@ -283,22 +283,27 @@ user_agent='litellm/9.9.9', fn mistyped_python_settings_decline_instead_of_raising() { Python::initialize(); Python::attach(|py| { - let error = settings(&python_settings(py, "force_ipv4='yes'")).unwrap_err(); + let error = configured(&python_settings(py, "force_ipv4='yes'")).unwrap_err(); assert!(error.is_instance_of::(py)); }); } + fn configured_ssl_verify(ssl_verify: SslVerify) -> HttpSettingsLayer { + HttpSettingsLayer { + ssl_verify: Some(ssl_verify), + ..HttpSettingsLayer::default() + } + } + #[test] - fn call_ssl_verify_beats_the_configured_and_environment_value() { + fn call_ssl_verify_beats_the_configured_value() { Python::initialize(); Python::attach(|py| { let kwargs = PyDict::new(py); kwargs.set_item("ssl_verify", false).unwrap(); - let configured = HttpSettings { - ssl_verify: Some(SslVerify::Enabled), - ..HttpSettings::default() - }; - let settings = for_call(configured, call_ssl_verify(&kwargs).unwrap(), true); + let call = for_call(call_ssl_verify(&kwargs).unwrap(), true); + let settings = + HttpSettings::from_layers([call, configured_ssl_verify(SslVerify::Enabled)]); assert_eq!(settings.ssl_verify, Some(SslVerify::Disabled)); }); } @@ -309,12 +314,10 @@ user_agent='litellm/9.9.9', Python::attach(|py| { let kwargs = PyDict::new(py); kwargs.set_item("ssl_verify", py.None()).unwrap(); - let configured = HttpSettings { - ssl_verify: Some(SslVerify::Disabled), - ..HttpSettings::default() - }; - let settings = for_call(configured.clone(), call_ssl_verify(&kwargs).unwrap(), true); - assert_eq!(settings, configured); + let call = for_call(call_ssl_verify(&kwargs).unwrap(), true); + let settings = + HttpSettings::from_layers([call, configured_ssl_verify(SslVerify::Disabled)]); + assert_eq!(settings.ssl_verify, Some(SslVerify::Disabled)); }); } @@ -326,12 +329,10 @@ user_agent='litellm/9.9.9', kwargs .set_item("ssl_verify", py.eval(c"object()", None, None).unwrap()) .unwrap(); - let configured = HttpSettings { - ssl_verify: Some(SslVerify::Disabled), - ..HttpSettings::default() - }; - let settings = for_call(configured.clone(), call_ssl_verify(&kwargs).unwrap(), true); - assert_eq!(settings, configured); + let call = for_call(call_ssl_verify(&kwargs).unwrap(), true); + let settings = + HttpSettings::from_layers([call, configured_ssl_verify(SslVerify::Disabled)]); + assert_eq!(settings.ssl_verify, Some(SslVerify::Disabled)); }); } @@ -342,12 +343,12 @@ user_agent='litellm/9.9.9', #[case] asynchronous: bool, #[case] expected: bool, ) { - let opted_out = HttpSettings { - ignore_proxy_env: true, - ..HttpSettings::default() + let opted_out = HttpSettingsLayer { + disable_aiohttp_trust_env: Some(true), + disable_aiohttp_transport: Some(false), + ..HttpSettingsLayer::default() }; - let settings = for_call(opted_out, None, asynchronous); - let config = Resolution::from(&settings).config; - assert_eq!(config.trust_proxy_env, expected); + let settings = HttpSettings::from_layers([for_call(None, asynchronous), opted_out]); + assert_eq!(settings.trust_proxy_env, expected); } }