From 1a581626301c8a09a2cd29578c5175802b1ebb5a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2026 18:31:15 -0700 Subject: [PATCH] refactor(http): hand out an owned Client and route all providers through the pool (#43245) * refactor(messages): take the provider client from the injected HTTP pool The messages route kept its own process-wide reqwest client, so it ignored ssl_verify, CA bundles, client certs, proxies and every other setting that litellm-http resolves. The machine now takes the HttpClientPool and the call's HttpClientConfig, as OCR does, and the bridge passes its shared pool. Co-Authored-By: Claude Opus 5.5 * refactor(http): hand out an owned Client and move chat, audio and OIDC onto the pool HttpClientPool now returns litellm_http::Client, a newtype only crates/http can build, so every provider client carries the resolved TLS, proxy and timeout settings. Chat completions and audio transcription drop their process-wide reqwest clients and take the pool and call config like messages; their 600s ceiling moves to the request. OidcResolver takes its client instead of building one, and the bridge hands it the pooled one. Co-Authored-By: Claude Opus 5.5 * refactor(secrets): build Google, Azure and CyberArk manager clients from the pool The native secret managers built bare reqwest clients, so they ignored the host's TLS and proxy settings. load_native_manager now takes the pool and the host config and hands each manager a pooled client. CyberArk's CYBERARK_SSL_VERIFY and CYBERARK_CLIENT_CERT/KEY become an override on the host config instead of a hand-built client. To express a certificate and key in separate files, HttpClientConfig::client_certificate is now a ClientIdentity that is either one PEM or a split pair. Co-Authored-By: Claude Opus 5.5 * chore(clippy): only crates/http may build a reqwest client Fence reqwest::Client, ClientBuilder and the TLS builder methods with disallowed-types and disallowed-methods so new code takes a litellm_http::Client from the pool. crates/http is exempt as the one place clients are built, and testkit as a dev-only installer. Tests move to litellm_http::Client::plain_for_test or a pooled client. Co-Authored-By: Claude Opus 5.5 * fix(secrets-cyberark): keep verifying certificates when the host disables it Python hands CyberArk its own ssl_verify, which wins over the global setting, so CYBERARK_SSL_VERIFY unset or true still verifies even when the host sets ssl_verify false. The pooled client copied the host's Disabled and would send the API key unverified; fall back to the built-in roots instead. Co-Authored-By: Claude Opus 5.5 * fix(python-bridge): treat a missing litellm package as no host HTTP settings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Yujong Lee Co-authored-by: Claude Opus 5.5 Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/Cargo.lock | 10 +++ litellm-rust/clippy.toml | 12 ++++ litellm-rust/crates/auth-aws/Cargo.toml | 1 + litellm-rust/crates/auth-aws/src/aws.rs | 2 +- .../crates/cache-azure-blob/Cargo.toml | 2 + .../crates/cache-azure-blob/src/cache.rs | 2 +- .../crates/cache-azure-blob/src/transport.rs | 2 +- .../cache-azure-blob/tests/transport.rs | 2 +- litellm-rust/crates/cache-gcs/Cargo.toml | 2 + litellm-rust/crates/cache-gcs/src/cache.rs | 2 +- litellm-rust/crates/cache-gcs/tests/cache.rs | 2 +- .../crates/cache-gcs/tests/support/mod.rs | 2 +- .../crates/cache-qdrant-semantic/Cargo.toml | 2 + .../cache-qdrant-semantic/src/embedder.rs | 2 +- .../cache-qdrant-semantic/tests/embedder.rs | 22 +++++-- litellm-rust/crates/cache-s3/Cargo.toml | 2 + litellm-rust/crates/cache-s3/src/cache.rs | 7 ++- litellm-rust/crates/cache-s3/src/transport.rs | 2 +- .../crates/cache-s3/tests/support/mod.rs | 7 ++- litellm-rust/crates/core/Cargo.toml | 1 + .../core/src/audio_transcription/client.rs | 13 ---- .../core/src/audio_transcription/handler.rs | 20 ++++-- .../core/src/audio_transcription/mod.rs | 13 ++-- .../core/src/chat_completions/client.rs | 14 ----- .../core/src/chat_completions/handler.rs | 20 ++++-- .../crates/core/src/chat_completions/mod.rs | 8 ++- litellm-rust/crates/core/src/constants.rs | 6 -- .../crates/core/src/messages/client.rs | 14 ----- .../crates/core/src/messages/error.rs | 2 + .../crates/core/src/messages/handler.rs | 12 ++-- litellm-rust/crates/core/src/messages/mod.rs | 19 ++++-- .../crates/core/src/messages/route.rs | 15 ++++- litellm-rust/crates/core/src/ocr/prepare.rs | 5 +- .../crates/core/tests/audio_transcription.rs | 20 +++--- .../crates/core/tests/chat_completions.rs | 21 ++++--- .../crates/core/tests/messages/host.rs | 2 +- .../crates/core/tests/messages/main.rs | 11 +++- .../crates/core/tests/messages/response.rs | 35 +++++++---- .../crates/core/tests/messages/stream.rs | 2 +- litellm-rust/crates/core/tests/ocr/main.rs | 7 +-- litellm-rust/crates/core/tests/ocr/mistral.rs | 13 ++-- litellm-rust/crates/core/tests/support/mod.rs | 13 +++- litellm-rust/crates/http/Cargo.toml | 2 + litellm-rust/crates/http/src/client.rs | 38 +++++++++++ litellm-rust/crates/http/src/config.rs | 12 +++- litellm-rust/crates/http/src/lib.rs | 10 ++- litellm-rust/crates/http/src/media.rs | 25 +++----- litellm-rust/crates/http/src/outbound.rs | 2 +- litellm-rust/crates/http/src/pool.rs | 14 +++-- litellm-rust/crates/http/src/tls.rs | 58 ++++++++++++----- litellm-rust/crates/llms/Cargo.toml | 1 + .../document_intelligence/transformation.rs | 4 +- .../crates/llms/src/base_llm/ocr/document.rs | 25 +++++--- .../crates/llms/src/base_llm/ocr/handler.rs | 27 ++++---- .../llms/src/reducto/ocr/transformation.rs | 5 +- litellm-rust/crates/llms/tests/ocr_handler.rs | 2 +- litellm-rust/crates/python-bridge/Cargo.toml | 1 + .../python-bridge/src/cache/activation.rs | 2 +- .../crates/python-bridge/src/cache/config.rs | 2 +- .../crates/python-bridge/src/cache/handle.rs | 2 +- .../crates/python-bridge/src/cache/mod.rs | 10 --- .../crates/python-bridge/src/cache/native.rs | 8 +-- litellm-rust/crates/python-bridge/src/http.rs | 19 ++++-- .../python-bridge/src/python_settings.rs | 12 +++- .../src/routes/audio_transcription.rs | 34 ++++++---- .../src/routes/chat_completions.rs | 42 +++++++++---- .../python-bridge/src/routes/messages/mod.rs | 5 +- .../python-bridge/src/secrets/callback.rs | 2 +- .../crates/python-bridge/src/secrets/mod.rs | 7 ++- .../python-bridge/src/secrets/resolved.rs | 29 ++++++--- .../python-bridge/src/secrets/runtime.rs | 15 ++++- litellm-rust/crates/secrets-azure/Cargo.toml | 2 + .../crates/secrets-azure/src/key_vault.rs | 11 ++-- .../crates/secrets-azure/tests/key_vault.rs | 19 +++--- .../crates/secrets-azure/tests/live.rs | 2 +- .../crates/secrets-cyberark/Cargo.toml | 2 + .../crates/secrets-cyberark/src/error.rs | 2 + .../secrets-cyberark/src/secret_manager.rs | 7 ++- .../src/secret_manager/client.rs | 63 +++++++++++++++---- .../secrets-cyberark/tests/secret_manager.rs | 2 + .../tests/secret_manager/configuration.rs | 26 ++++---- .../tests/secret_manager/support.rs | 14 ++++- .../tests/secret_manager/writes.rs | 6 +- litellm-rust/crates/secrets-google/Cargo.toml | 2 + .../secrets-google/src/secret_manager.rs | 7 ++- .../secrets-google/tests/secret_manager.rs | 16 +++-- litellm-rust/crates/secrets/Cargo.toml | 2 + litellm-rust/crates/secrets/src/error.rs | 2 + litellm-rust/crates/secrets/src/native.rs | 33 ++++++---- litellm-rust/crates/secrets/src/oidc.rs | 34 +++++----- litellm-rust/crates/secrets/src/resolver.rs | 15 +---- litellm-rust/crates/secrets/src/source.rs | 5 +- litellm-rust/crates/secrets/tests/aws.rs | 6 +- litellm-rust/crates/secrets/tests/azure.rs | 6 +- .../secrets/tests/common_read_contract.rs | 8 +-- litellm-rust/crates/secrets/tests/cyberark.rs | 2 +- litellm-rust/crates/secrets/tests/google.rs | 4 +- .../crates/secrets/tests/hashicorp.rs | 4 +- litellm-rust/crates/secrets/tests/oidc.rs | 36 ++++++----- .../crates/secrets/tests/resolution.rs | 14 ++--- litellm-rust/crates/secrets/tests/source.rs | 4 +- litellm-rust/crates/testkit/src/lib.rs | 6 ++ 102 files changed, 745 insertions(+), 403 deletions(-) delete mode 100644 litellm-rust/crates/core/src/audio_transcription/client.rs delete mode 100644 litellm-rust/crates/core/src/chat_completions/client.rs delete mode 100644 litellm-rust/crates/core/src/messages/client.rs create mode 100644 litellm-rust/crates/http/src/client.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index ffcc6a5496b..5bb65b6b05f 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -2911,6 +2911,7 @@ dependencies = [ "litellm-cache", "litellm-cache-response", "litellm-cache-testing", + "litellm-http", "reqwest 0.12.28", "rstest", "serde_json", @@ -2944,6 +2945,7 @@ dependencies = [ "litellm-auth-types", "litellm-cache", "litellm-cache-testing", + "litellm-http", "percent-encoding", "reqwest 0.12.28", "rstest", @@ -2970,6 +2972,7 @@ dependencies = [ "futures-util", "litellm-cache", "litellm-cache-testing", + "litellm-http", "qdrant-client", "reqwest 0.12.28", "rstest", @@ -3043,6 +3046,7 @@ dependencies = [ "litellm-auth-aws", "litellm-cache", "litellm-cache-testing", + "litellm-http", "reqwest 0.12.28", "rstest", "serde_json", @@ -3207,11 +3211,13 @@ dependencies = [ "http 1.4.2", "hyper-util", "litellm-core-utils", + "rcgen", "reqwest 0.12.28", "rstest", "rustls 0.23.42", "serde", "serde_json", + "tempfile", "thiserror 2.0.19", "tokio", "veil", @@ -3345,6 +3351,7 @@ dependencies = [ "google-cloud-auth", "google-cloud-kms-v1", "litellm-core-utils", + "litellm-http", "litellm-python-compat", "litellm-secrets-aws", "litellm-secrets-azure", @@ -3392,6 +3399,7 @@ dependencies = [ "litellm-auth-azure", "litellm-auth-types", "litellm-core-utils", + "litellm-http", "litellm-secrets-types", "percent-encoding", "reqwest 0.12.28", @@ -3411,6 +3419,7 @@ version = "0.1.0" dependencies = [ "base64 0.22.1", "litellm-core-utils", + "litellm-http", "litellm-secrets-types", "litellm-tracing", "moka", @@ -3439,6 +3448,7 @@ dependencies = [ "litellm-auth-gcp", "litellm-auth-types", "litellm-core-utils", + "litellm-http", "litellm-secrets-types", "moka", "percent-encoding", diff --git a/litellm-rust/clippy.toml b/litellm-rust/clippy.toml index f7e3293069b..0e2ff770d27 100644 --- a/litellm-rust/clippy.toml +++ b/litellm-rust/clippy.toml @@ -7,4 +7,16 @@ disallowed-methods = [ { path = "pyo3_async_runtimes::tokio::local_future_into_py", reason = "use litellm_host_python::run_async / run_async_value" }, { path = "pyo3_async_runtimes::tokio::run", reason = "use litellm_host_python::run_sync / run_sync_value" }, { path = "pyo3_async_runtimes::tokio::run_until_complete", reason = "use litellm_host_python::run_sync / run_sync_value" }, + { path = "reqwest::Client::new", reason = "take litellm_http::Client from HttpClientPool" }, + { path = "reqwest::Client::builder", reason = "HttpClientConfig owns client construction" }, + { path = "reqwest::ClientBuilder::danger_accept_invalid_certs", reason = "set HttpClientConfig::verify instead" }, + { path = "reqwest::ClientBuilder::identity", reason = "set HttpClientConfig::client_certificate instead" }, + { path = "reqwest::ClientBuilder::use_preconfigured_tls", reason = "HttpClientConfig owns the TLS configuration" }, +] + +# Every outbound client comes from litellm_http::HttpClientPool so it honors the host's TLS, +# proxy and timeout settings. Only crates/http builds one. +disallowed-types = [ + { path = "reqwest::Client", reason = "take litellm_http::Client from HttpClientPool; only crates/http builds one" }, + { path = "reqwest::ClientBuilder", reason = "HttpClientConfig owns client construction" }, ] diff --git a/litellm-rust/crates/auth-aws/Cargo.toml b/litellm-rust/crates/auth-aws/Cargo.toml index 1a35af48574..9592f278d94 100644 --- a/litellm-rust/crates/auth-aws/Cargo.toml +++ b/litellm-rust/crates/auth-aws/Cargo.toml @@ -22,5 +22,6 @@ aws-types = "1.4.0" aws-smithy-runtime-api = "1.13.0" [dev-dependencies] +litellm-http = { workspace = true, features = ["test-support"] } reqwest.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/auth-aws/src/aws.rs b/litellm-rust/crates/auth-aws/src/aws.rs index cb9195ffeb6..409ff78867f 100644 --- a/litellm-rust/crates/auth-aws/src/aws.rs +++ b/litellm-rust/crates/auth-aws/src/aws.rs @@ -962,7 +962,7 @@ mod tests { &no_env, ) .await?; - let client = reqwest::Client::new(); + let client = litellm_http::Client::plain_for_test(); let mut failures = Vec::new(); for region in ["us-west-2", "us-east-1"] { diff --git a/litellm-rust/crates/cache-azure-blob/Cargo.toml b/litellm-rust/crates/cache-azure-blob/Cargo.toml index baa1b0f5482..5bdfa16ef53 100644 --- a/litellm-rust/crates/cache-azure-blob/Cargo.toml +++ b/litellm-rust/crates/cache-azure-blob/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-http.workspace = true litellm-auth-azure.workspace = true litellm-auth-types.workspace = true litellm-cache.workspace = true @@ -19,6 +20,7 @@ tokio.workspace = true url.workspace = true [dev-dependencies] +litellm-http = { workspace = true, features = ["test-support"] } litellm-cache-response.workspace = true litellm-cache-testing.workspace = true rstest.workspace = true diff --git a/litellm-rust/crates/cache-azure-blob/src/cache.rs b/litellm-rust/crates/cache-azure-blob/src/cache.rs index 489b08d485e..c5c1fdd8ab9 100644 --- a/litellm-rust/crates/cache-azure-blob/src/cache.rs +++ b/litellm-rust/crates/cache-azure-blob/src/cache.rs @@ -31,7 +31,7 @@ impl AzureBlobCache { pub async fn connect( account_url: &str, container: &str, - http: reqwest::Client, + http: litellm_http::Client, codec: C, runtime: Handle, ) -> Result { diff --git a/litellm-rust/crates/cache-azure-blob/src/transport.rs b/litellm-rust/crates/cache-azure-blob/src/transport.rs index ed038b8d69d..3914b92365c 100644 --- a/litellm-rust/crates/cache-azure-blob/src/transport.rs +++ b/litellm-rust/crates/cache-azure-blob/src/transport.rs @@ -8,7 +8,7 @@ use azure_core::{ use futures_util::TryStreamExt; #[derive(Debug)] -pub struct ReqwestTransport(pub reqwest::Client); +pub struct ReqwestTransport(pub litellm_http::Client); #[async_trait::async_trait] impl HttpClient for ReqwestTransport { diff --git a/litellm-rust/crates/cache-azure-blob/tests/transport.rs b/litellm-rust/crates/cache-azure-blob/tests/transport.rs index cd1e10aa3d8..c8b14cd8543 100644 --- a/litellm-rust/crates/cache-azure-blob/tests/transport.rs +++ b/litellm-rust/crates/cache-azure-blob/tests/transport.rs @@ -31,7 +31,7 @@ async fn connect(server: &MockServer) -> AzureBlobCache> { None, ClientOptions { transport: Some(Transport::new(Arc::new(ReqwestTransport( - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), )))), ..ClientOptions::default() }, diff --git a/litellm-rust/crates/cache-gcs/Cargo.toml b/litellm-rust/crates/cache-gcs/Cargo.toml index da0acf554f9..1a06683e615 100644 --- a/litellm-rust/crates/cache-gcs/Cargo.toml +++ b/litellm-rust/crates/cache-gcs/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-http.workspace = true futures-util.workspace = true litellm-auth-gcp.workspace = true litellm-auth-types.workspace = true @@ -15,6 +16,7 @@ reqwest.workspace = true tokio.workspace = true [dev-dependencies] +litellm-http = { workspace = true, features = ["test-support"] } litellm-cache-testing.workspace = true rstest.workspace = true serde_json.workspace = true diff --git a/litellm-rust/crates/cache-gcs/src/cache.rs b/litellm-rust/crates/cache-gcs/src/cache.rs index a8a7fbc9a7b..bad81573cb4 100644 --- a/litellm-rust/crates/cache-gcs/src/cache.rs +++ b/litellm-rust/crates/cache-gcs/src/cache.rs @@ -5,8 +5,8 @@ use litellm_cache::{ BaseCache, BatchCache, BatchEntry, CacheCodec, DisconnectCache, Error, ExactCacheContext, FlushCache, }; +use litellm_http::Client; use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_encode}; -use reqwest::Client; use crate::{GcpTokenSource, TokenSource}; diff --git a/litellm-rust/crates/cache-gcs/tests/cache.rs b/litellm-rust/crates/cache-gcs/tests/cache.rs index cdce6a00bdd..12bb5344570 100644 --- a/litellm-rust/crates/cache-gcs/tests/cache.rs +++ b/litellm-rust/crates/cache-gcs/tests/cache.rs @@ -96,7 +96,7 @@ async fn cache_exposes_its_configuration(#[future(awt)] server: MockServer) { path_service_account: Some("/secrets/sa.json".into()), ..support::config(&server, Some("folder")) }, - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), litellm_cache::JsonCodec::::new(), ); assert_eq!(cache.bucket_name(), "bucket"); diff --git a/litellm-rust/crates/cache-gcs/tests/support/mod.rs b/litellm-rust/crates/cache-gcs/tests/support/mod.rs index 6097f0ee1bd..beb9aa39d9c 100644 --- a/litellm-rust/crates/cache-gcs/tests/support/mod.rs +++ b/litellm-rust/crates/cache-gcs/tests/support/mod.rs @@ -29,7 +29,7 @@ pub fn cache_with_token( ) -> JsonGcsCache { GcsCache::with_token_source( config(server, gcs_path), - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), JsonCodec::new(), token, ) diff --git a/litellm-rust/crates/cache-qdrant-semantic/Cargo.toml b/litellm-rust/crates/cache-qdrant-semantic/Cargo.toml index 950c2db7491..a44bef0a731 100644 --- a/litellm-rust/crates/cache-qdrant-semantic/Cargo.toml +++ b/litellm-rust/crates/cache-qdrant-semantic/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-http.workspace = true futures-util.workspace = true litellm-cache.workspace = true qdrant-client = { workspace = true, features = ["serde"] } @@ -17,6 +18,7 @@ tokio.workspace = true uuid.workspace = true [dev-dependencies] +litellm-http = { workspace = true, features = ["test-support"] } futures-executor = "0.3" litellm-cache-testing.workspace = true rstest.workspace = true diff --git a/litellm-rust/crates/cache-qdrant-semantic/src/embedder.rs b/litellm-rust/crates/cache-qdrant-semantic/src/embedder.rs index 340393600f2..b7fbcd9b02d 100644 --- a/litellm-rust/crates/cache-qdrant-semantic/src/embedder.rs +++ b/litellm-rust/crates/cache-qdrant-semantic/src/embedder.rs @@ -1,7 +1,7 @@ use std::time::Duration; use litellm_cache::{Error, semantic::Embedder}; -use reqwest::Client; +use litellm_http::Client; use serde_json::Value; pub struct OpenAiEmbedder { diff --git a/litellm-rust/crates/cache-qdrant-semantic/tests/embedder.rs b/litellm-rust/crates/cache-qdrant-semantic/tests/embedder.rs index de0fab0a66f..24e6e5eba3e 100644 --- a/litellm-rust/crates/cache-qdrant-semantic/tests/embedder.rs +++ b/litellm-rust/crates/cache-qdrant-semantic/tests/embedder.rs @@ -5,6 +5,10 @@ use std::{ use litellm_cache::{Error, semantic::Embedder}; use litellm_cache_qdrant_semantic::{OpenAiEmbedder, OpenAiEmbedderConfig}; +use litellm_http::{ + ClientVariant, HttpClientConfig, HttpClientPool, HttpSettings, Resolution, + media::PublicDnsResolver, +}; use rstest::rstest; use serde_json::{Value, json}; use tokio::{ @@ -104,7 +108,7 @@ fn config(base: String, timeout: Option) -> OpenAiEmbedderConfig { async fn posts_embeddings_request_and_parses_vector() { let server = TestHttpServer::response("200 OK", r#"{"data":[{"embedding":[0.1,0.2]}]}"#).await; let embedder = OpenAiEmbedder::new( - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), config( format!("{}/", server.base_url()), Some(Duration::from_secs(1)), @@ -156,14 +160,17 @@ async fn status_timeout_and_body_errors_are_unavailable( ) { let server = TestHttpServer::response_after(status, body, Duration::from_millis(delay_ms)).await; - let embedder = OpenAiEmbedder::new(reqwest::Client::new(), config(server.base_url(), timeout)); + let embedder = OpenAiEmbedder::new( + litellm_http::Client::plain_for_test(), + config(server.base_url(), timeout), + ); assert_eq!(embedder.async_embed("hello", None).await, expected); } #[rstest] fn sync_embedding_is_unsupported() { let embedder = OpenAiEmbedder::new( - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), config("http://127.0.0.1:9".to_owned(), None), ); assert_eq!( @@ -176,9 +183,12 @@ fn sync_embedding_is_unsupported() { #[tokio::test] async fn uses_the_injected_client() { let server = TestHttpServer::response("200 OK", r#"{"data":[{"embedding":[0.1,0.2]}]}"#).await; - let client = reqwest::Client::builder() - .user_agent("litellm-embedder-test") - .build() + let config_with_agent = HttpClientConfig { + user_agent: Some("litellm-embedder-test".into()), + ..Resolution::from(&HttpSettings::default()).config + }; + let client = HttpClientPool::new(Arc::new(PublicDnsResolver)) + .client(&config_with_agent, ClientVariant::Provider) .unwrap(); let embedder = OpenAiEmbedder::new(client, config(server.base_url(), None)); assert_eq!( diff --git a/litellm-rust/crates/cache-s3/Cargo.toml b/litellm-rust/crates/cache-s3/Cargo.toml index c8150180e7c..680f2da8215 100644 --- a/litellm-rust/crates/cache-s3/Cargo.toml +++ b/litellm-rust/crates/cache-s3/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-http.workspace = true litellm-cache.workspace = true litellm-auth-aws.workspace = true aws-sdk-s3 = { version = "1.146.1", default-features = false, features = ["rustls", "rt-tokio"] } @@ -19,6 +20,7 @@ reqwest.workspace = true tokio.workspace = true [dev-dependencies] +litellm-http = { workspace = true, features = ["test-support"] } litellm-cache-testing.workspace = true rstest.workspace = true wiremock = "0.6.5" diff --git a/litellm-rust/crates/cache-s3/src/cache.rs b/litellm-rust/crates/cache-s3/src/cache.rs index 91cd5e8ef54..ced948e80c6 100644 --- a/litellm-rust/crates/cache-s3/src/cache.rs +++ b/litellm-rust/crates/cache-s3/src/cache.rs @@ -42,7 +42,12 @@ pub struct S3Cache { } impl S3Cache { - pub fn new(config: S3CacheConfig, http: reqwest::Client, codec: C, runtime: Handle) -> Self { + pub fn new( + config: S3CacheConfig, + http: litellm_http::Client, + codec: C, + runtime: Handle, + ) -> Self { let endpoint_url: Option = config.endpoint.map(|endpoint| endpoint.url); let base = aws_sdk_s3::Config::builder() .behavior_version(BehaviorVersion::latest()) diff --git a/litellm-rust/crates/cache-s3/src/transport.rs b/litellm-rust/crates/cache-s3/src/transport.rs index 3e5ce578c31..eabc54ac9e2 100644 --- a/litellm-rust/crates/cache-s3/src/transport.rs +++ b/litellm-rust/crates/cache-s3/src/transport.rs @@ -9,7 +9,7 @@ use aws_smithy_runtime_api::client::{ use aws_smithy_types::body::SdkBody; #[derive(Clone, Debug)] -pub(crate) struct ReqwestHttpClient(pub(crate) reqwest::Client); +pub(crate) struct ReqwestHttpClient(pub(crate) litellm_http::Client); impl HttpClient for ReqwestHttpClient { fn http_connector( diff --git a/litellm-rust/crates/cache-s3/tests/support/mod.rs b/litellm-rust/crates/cache-s3/tests/support/mod.rs index 046b042c66a..b628c1df431 100644 --- a/litellm-rust/crates/cache-s3/tests/support/mod.rs +++ b/litellm-rust/crates/cache-s3/tests/support/mod.rs @@ -32,7 +32,12 @@ pub fn config(endpoint: &str) -> S3CacheConfig { } pub fn cache_with(config: S3CacheConfig, runtime: Handle) -> JsonS3Cache { - S3Cache::new(config, reqwest::Client::new(), JsonCodec::new(), runtime) + S3Cache::new( + config, + litellm_http::Client::plain_for_test(), + JsonCodec::new(), + runtime, + ) } pub fn cache(endpoint: &str) -> JsonS3Cache { diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index 12410c187e2..8700c8df308 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -36,6 +36,7 @@ url.workspace = true veil.workspace = true [dev-dependencies] +litellm-http = { workspace = true, features = ["test-support"] } litellm-auth-gcp.workspace = true litellm-llms = { workspace = true, features = ["test-support"] } rstest.workspace = true diff --git a/litellm-rust/crates/core/src/audio_transcription/client.rs b/litellm-rust/crates/core/src/audio_transcription/client.rs deleted file mode 100644 index 3cf131839b8..00000000000 --- a/litellm-rust/crates/core/src/audio_transcription/client.rs +++ /dev/null @@ -1,13 +0,0 @@ -use std::{sync::OnceLock, time::Duration}; - -use crate::constants::AUDIO_TRANSCRIPTION_TIMEOUT_SECS; - -pub(super) fn http_client() -> &'static reqwest::Client { - static CLIENT: OnceLock = OnceLock::new(); - CLIENT.get_or_init(|| { - reqwest::Client::builder() - .timeout(Duration::from_secs(AUDIO_TRANSCRIPTION_TIMEOUT_SECS)) - .build() - .unwrap_or_else(|_| reqwest::Client::new()) - }) -} diff --git a/litellm-rust/crates/core/src/audio_transcription/handler.rs b/litellm-rust/crates/core/src/audio_transcription/handler.rs index a1862f341a5..30900bc14c6 100644 --- a/litellm-rust/crates/core/src/audio_transcription/handler.rs +++ b/litellm-rust/crates/core/src/audio_transcription/handler.rs @@ -1,10 +1,16 @@ -use litellm_http::request::truncate_error_body; +use std::time::Duration; + +use litellm_http::{Client, request::truncate_error_body}; use serde_json::Value; -use super::{Error, client::http_client}; -use crate::audio_transcription::types::ProviderAudioTranscriptionRequest; +use super::Error; +use crate::{ + audio_transcription::types::ProviderAudioTranscriptionRequest, + constants::AUDIO_TRANSCRIPTION_TIMEOUT_SECS, +}; pub async fn execute_audio_transcription_provider_call( + http: &Client, request: ProviderAudioTranscriptionRequest, ) -> Result { let response = crate::outbound::outbound_request::( @@ -12,11 +18,15 @@ pub async fn execute_audio_transcription_provider_call( request.url.clone(), request.upstream_headers.clone(), &request.body, - request.timeout, + Some( + request + .timeout + .unwrap_or(Duration::from_secs(AUDIO_TRANSCRIPTION_TIMEOUT_SECS)), + ), &request.optional_params, ) .await? - .send(http_client()) + .send(http) .await .map_err(|error| { Error::Transport(litellm_http::transport::Error::Network(error.to_string())) diff --git a/litellm-rust/crates/core/src/audio_transcription/mod.rs b/litellm-rust/crates/core/src/audio_transcription/mod.rs index 801fd5e9673..dc75326d5c3 100644 --- a/litellm-rust/crates/core/src/audio_transcription/mod.rs +++ b/litellm-rust/crates/core/src/audio_transcription/mod.rs @@ -1,16 +1,21 @@ mod error; pub mod types; pub use error::Error; -mod client; mod handler; mod prepare; pub use handler::execute_audio_transcription_provider_call; +use litellm_http::{ClientVariant, HttpClientConfig, HttpClientPool}; pub use prepare::prepare_audio_transcription_provider_call; use serde_json::Value; use crate::audio_transcription::types::AudioTranscriptionRequest; -pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> Result { - execute_audio_transcription_provider_call(prepare_audio_transcription_provider_call(request)?) - .await +pub async fn audio_transcription( + pool: &HttpClientPool, + config: &HttpClientConfig, + request: AudioTranscriptionRequest<'_>, +) -> Result { + let request = prepare_audio_transcription_provider_call(request)?; + let http = pool.client(config, ClientVariant::Provider)?; + execute_audio_transcription_provider_call(&http, request).await } diff --git a/litellm-rust/crates/core/src/chat_completions/client.rs b/litellm-rust/crates/core/src/chat_completions/client.rs deleted file mode 100644 index d8ad6c49b7b..00000000000 --- a/litellm-rust/crates/core/src/chat_completions/client.rs +++ /dev/null @@ -1,14 +0,0 @@ -use std::{sync::OnceLock, time::Duration}; - -use crate::constants::{CHAT_COMPLETIONS_CONNECT_TIMEOUT_SECS, CHAT_COMPLETIONS_TIMEOUT_SECS}; - -pub(super) fn http_client() -> &'static reqwest::Client { - static CLIENT: OnceLock = OnceLock::new(); - CLIENT.get_or_init(|| { - reqwest::Client::builder() - .timeout(Duration::from_secs(CHAT_COMPLETIONS_TIMEOUT_SECS)) - .connect_timeout(Duration::from_secs(CHAT_COMPLETIONS_CONNECT_TIMEOUT_SECS)) - .build() - .unwrap_or_else(|_| reqwest::Client::new()) - }) -} diff --git a/litellm-rust/crates/core/src/chat_completions/handler.rs b/litellm-rust/crates/core/src/chat_completions/handler.rs index 2391ab83a60..f3404fcaa8a 100644 --- a/litellm-rust/crates/core/src/chat_completions/handler.rs +++ b/litellm-rust/crates/core/src/chat_completions/handler.rs @@ -1,20 +1,24 @@ -use litellm_http::{outbound::OutboundRequest, request::truncate_error_body}; +use std::time::Duration; + +use litellm_http::{Client, outbound::OutboundRequest, request::truncate_error_body}; use litellm_llms::base_llm::chat::transformation::ProviderChatResponseData; use litellm_types::utils::ChatCompletionsResponse; use serde_json::Value; -use super::{Error, client::http_client, prepare::prepare_provider_request}; -use crate::chat_completions::types::{ - ProviderChatCompletionsRequest, ResolvedChatCompletionsRequest, +use super::{Error, prepare::prepare_provider_request}; +use crate::{ + chat_completions::types::{ProviderChatCompletionsRequest, ResolvedChatCompletionsRequest}, + constants::CHAT_COMPLETIONS_TIMEOUT_SECS, }; pub(super) async fn execute_chat_completions_provider_call( + http: &Client, request: ResolvedChatCompletionsRequest<'_>, ) -> Result { let request = prepare_provider_request(request)?; let outbound = outbound_request(&request).await?; - let response = outbound.send(http_client()).await.map_err(|err| { + let response = outbound.send(http).await.map_err(|err| { // Failing to establish the connection means the request never went out, // so the host can still serve it. Everything else here, a timeout // above all, may have reached the provider and been answered. @@ -72,7 +76,11 @@ pub(super) async fn outbound_request( request.url.clone(), request.upstream_headers.clone(), &request.body, - request.timeout, + Some( + request + .timeout + .unwrap_or(Duration::from_secs(CHAT_COMPLETIONS_TIMEOUT_SECS)), + ), &request.optional_params, ) .await diff --git a/litellm-rust/crates/core/src/chat_completions/mod.rs b/litellm-rust/crates/core/src/chat_completions/mod.rs index 224c9d8cfed..be22aea5669 100644 --- a/litellm-rust/crates/core/src/chat_completions/mod.rs +++ b/litellm-rust/crates/core/src/chat_completions/mod.rs @@ -9,11 +9,11 @@ mod error; pub mod types; pub use error::Error; -mod client; mod common_utils; pub(crate) mod handler; mod prepare; use handler::execute_chat_completions_provider_call; +use litellm_http::{ClientVariant, HttpClientConfig, HttpClientPool}; use litellm_types::utils::ChatCompletionsResponse; use prepare::{parse_messages, resolve_provider_config, resolve_request}; use serde_json::{Map, Value}; @@ -21,9 +21,13 @@ use serde_json::{Map, Value}; use crate::chat_completions::types::ChatCompletionsRequest; pub async fn chat_completions( + pool: &HttpClientPool, + config: &HttpClientConfig, request: ChatCompletionsRequest<'_>, ) -> Result { - execute_chat_completions_provider_call(resolve_request(request)?).await + let request = resolve_request(request)?; + let http = pool.client(config, ClientVariant::Provider)?; + execute_chat_completions_provider_call(&http, request).await } /// Whether the core would accept this request, without resolving credentials or diff --git a/litellm-rust/crates/core/src/constants.rs b/litellm-rust/crates/core/src/constants.rs index 3d740e39677..455c3258799 100644 --- a/litellm-rust/crates/core/src/constants.rs +++ b/litellm-rust/crates/core/src/constants.rs @@ -5,9 +5,6 @@ pub const OPENAI_DEFAULT_API_BASE: &str = "https://api.openai.com"; /// timeout from the caller still overrides this on the request builder. pub(crate) const MESSAGES_TIMEOUT_SECS: u64 = 600; -/// Connect timeout for Anthropic Messages provider calls, in seconds. -pub(crate) const MESSAGES_CONNECT_TIMEOUT_SECS: u64 = 10; - /// Provider name used for Anthropic Messages when a deployment's provider model /// does not carry an explicit provider prefix. pub const ANTHROPIC_MESSAGES_PROVIDER: &str = "anthropic"; @@ -16,9 +13,6 @@ pub const ANTHROPIC_MESSAGES_PROVIDER: &str = "anthropic"; /// seconds. Mirrors the Python chat completions default. pub(crate) const CHAT_COMPLETIONS_TIMEOUT_SECS: u64 = 600; -/// Connect timeout for chat completions provider calls, in seconds. -pub(crate) const CHAT_COMPLETIONS_CONNECT_TIMEOUT_SECS: u64 = 10; - pub(crate) const AUDIO_TRANSCRIPTION_TIMEOUT_SECS: u64 = 600; /// `object` field every non-streaming chat completion response carries. diff --git a/litellm-rust/crates/core/src/messages/client.rs b/litellm-rust/crates/core/src/messages/client.rs deleted file mode 100644 index ca70b1b03eb..00000000000 --- a/litellm-rust/crates/core/src/messages/client.rs +++ /dev/null @@ -1,14 +0,0 @@ -use std::{sync::OnceLock, time::Duration}; - -use crate::constants::{MESSAGES_CONNECT_TIMEOUT_SECS, MESSAGES_TIMEOUT_SECS}; - -pub(super) fn http_client() -> &'static reqwest::Client { - static CLIENT: OnceLock = OnceLock::new(); - CLIENT.get_or_init(|| { - reqwest::Client::builder() - .timeout(Duration::from_secs(MESSAGES_TIMEOUT_SECS)) - .connect_timeout(Duration::from_secs(MESSAGES_CONNECT_TIMEOUT_SECS)) - .build() - .unwrap_or_else(|_| reqwest::Client::new()) - }) -} diff --git a/litellm-rust/crates/core/src/messages/error.rs b/litellm-rust/crates/core/src/messages/error.rs index 2a9723beb38..76f8813e330 100644 --- a/litellm-rust/crates/core/src/messages/error.rs +++ b/litellm-rust/crates/core/src/messages/error.rs @@ -17,6 +17,8 @@ pub enum Error { #[error(transparent)] Auth(#[from] litellm_auth::Error), #[error(transparent)] + Client(#[from] litellm_http::Error), + #[error(transparent)] Transport(#[from] litellm_http::transport::Error), #[error(transparent)] Headers(#[from] litellm_http::request::HeaderError), diff --git a/litellm-rust/crates/core/src/messages/handler.rs b/litellm-rust/crates/core/src/messages/handler.rs index de1a5f476ed..f90cb8cb454 100644 --- a/litellm-rust/crates/core/src/messages/handler.rs +++ b/litellm-rust/crates/core/src/messages/handler.rs @@ -5,13 +5,15 @@ use litellm_llms::base_llm::anthropic_messages::transformation::BaseAnthropicMes use litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse; use serde_json::Value; -use super::{Error, client::http_client, common_utils::truncate_error_body}; +use super::{Error, common_utils::truncate_error_body}; +use crate::constants::MESSAGES_TIMEOUT_SECS; pub(super) fn network(error: reqwest::Error) -> Error { Error::Transport(TransportError::Network(error.to_string())) } pub(super) async fn send( + http: &litellm_http::Client, url: &str, headers: &[(String, String)], body: &Value, @@ -20,13 +22,11 @@ pub(super) async fn send( let encoded = serde_json::to_vec(body) .map_err(|err| Error::InvalidRequest(format!("failed to encode messages body: {err}")))?; let builder = headers.iter().fold( - http_client().post(url).body(encoded), + http.post(url) + .body(encoded) + .timeout(timeout.unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS))), |builder, (key, value)| builder.header(key, value), ); - let builder = match timeout { - Some(duration) => builder.timeout(duration), - None => builder, - }; http_request(builder).await.map_err(network) } diff --git a/litellm-rust/crates/core/src/messages/mod.rs b/litellm-rust/crates/core/src/messages/mod.rs index 180eb08810e..5cb83b4e34d 100644 --- a/litellm-rust/crates/core/src/messages/mod.rs +++ b/litellm-rust/crates/core/src/messages/mod.rs @@ -7,13 +7,13 @@ mod error; pub mod types; pub use error::Error; -mod client; mod common_utils; mod handler; mod prepare; pub mod route; use std::sync::Arc; +use litellm_http::{ClientVariant, HttpClientConfig, HttpClientPool}; use litellm_secrets::source::EnvironmentSecrets; use litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse; use route::{LocalMessagesHost, MessagesCall, MessagesOutput, messages_machine}; @@ -21,7 +21,11 @@ use serde_json::Value; use crate::messages::types::MessagesRequest; -pub async fn messages(request: MessagesRequest<'_>) -> Result { +pub async fn messages( + pool: &HttpClientPool, + config: &HttpClientConfig, + request: MessagesRequest<'_>, +) -> Result { let Value::Object(body) = request.body else { return Err(Error::InvalidRequest( "messages body must be an object".into(), @@ -38,8 +42,15 @@ pub async fn messages(request: MessagesRequest<'_>) -> Result Ok(*message), MessagesOutput::Streamed => Err(Error::Unsupported( "streamed responses need a streaming host", diff --git a/litellm-rust/crates/core/src/messages/route.rs b/litellm-rust/crates/core/src/messages/route.rs index 40aff185e81..7f6589cdf3e 100644 --- a/litellm-rust/crates/core/src/messages/route.rs +++ b/litellm-rust/crates/core/src/messages/route.rs @@ -12,6 +12,7 @@ use litellm_host::{ machine::{CallMachine, HostChannel, MachineFault}, protocol::Protocol, }; +use litellm_http::{Client, ClientVariant, HttpClientConfig, HttpClientPool}; use litellm_secrets::source::SecretSource; use litellm_types::{ llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse, @@ -108,12 +109,20 @@ impl Host for LocalMessagesHost { } } -pub fn messages_machine(secrets: Arc) -> MessagesMachine { - CallMachine::new(move |host| Box::pin(execute(host, secrets.clone()))) +pub fn messages_machine( + pool: &HttpClientPool, + config: &HttpClientConfig, + secrets: Arc, +) -> Result { + let http = pool.client(config, ClientVariant::Provider)?; + Ok(CallMachine::new(move |host| { + Box::pin(execute(host, http.clone(), secrets.clone())) + })) } async fn execute( host: MessagesHost, + http: Client, secrets: Arc, ) -> Result { let call = host.project().await?; @@ -164,7 +173,7 @@ async fn execute( context, ) .await?; - let response = send(&wire.url, &wire.headers, &wire.body, request.timeout).await?; + let response = send(&http, &wire.url, &wire.headers, &wire.body, request.timeout).await?; if !response.status().is_success() { return Err(provider_error(response).await); } diff --git a/litellm-rust/crates/core/src/ocr/prepare.rs b/litellm-rust/crates/core/src/ocr/prepare.rs index 18961ec96fa..f13d6984763 100644 --- a/litellm-rust/crates/core/src/ocr/prepare.rs +++ b/litellm-rust/crates/core/src/ocr/prepare.rs @@ -110,7 +110,10 @@ mod tests { } fn client() -> OcrClient { - OcrClient::for_test(reqwest::Client::new(), reqwest::Client::new()) + OcrClient::for_test( + litellm_http::Client::plain_for_test(), + litellm_http::Client::no_redirect_for_test(), + ) } fn request(model: &str, base: &str, document: Value, options: Value) -> LiteLLMOcrRequest { diff --git a/litellm-rust/crates/core/tests/audio_transcription.rs b/litellm-rust/crates/core/tests/audio_transcription.rs index 196f085a6c3..612395fe63a 100644 --- a/litellm-rust/crates/core/tests/audio_transcription.rs +++ b/litellm-rust/crates/core/tests/audio_transcription.rs @@ -10,6 +10,10 @@ use support::*; const MODEL: &str = "mistral.voxtral-mini-3b-2507"; +async fn transcribe(request: AudioTranscriptionRequest<'_>) -> Result { + audio_transcription(&http_pool(), &http_config(), request).await +} + fn transcript_response(text: &str) -> ResponseTemplate { json_response(json!({"output": {"message": {"content": [{"text": text}]}}})) } @@ -47,7 +51,7 @@ async fn bedrock_converse_request_is_signed_for_the_requested_region( let upstream = upstream([transcript_response("hello")]).await; let base = upstream.uri(); - let response = audio_transcription(AudioTranscriptionRequest { + let response = transcribe(AudioTranscriptionRequest { api_base: Some(&base), optional_params: aws_params(region), ..request @@ -79,7 +83,7 @@ async fn the_provider_can_come_from_the_model_prefix(request: AudioTranscription let base = upstream.uri(); let model = format!("bedrock/{MODEL}"); - audio_transcription(AudioTranscriptionRequest { + transcribe(AudioTranscriptionRequest { model: &model, custom_llm_provider: None, api_base: Some(&base), @@ -110,7 +114,7 @@ async fn audio_and_transcription_params_reach_the_converse_body( ]) .collect(); - audio_transcription(AudioTranscriptionRequest { + transcribe(AudioTranscriptionRequest { audio: json!({"data": "AQI=", "format": format}), api_base: Some(&base), optional_params, @@ -142,7 +146,7 @@ async fn invalid_audio_is_rejected_before_sending( let upstream = upstream([transcript_response("hello")]).await; let base = upstream.uri(); - let error = audio_transcription(AudioTranscriptionRequest { + let error = transcribe(AudioTranscriptionRequest { audio, api_base: Some(&base), ..request @@ -174,7 +178,7 @@ async fn unsupported_providers_are_rejected_before_sending( #[case] provider: Option<&'static str>, #[case] reported: &str, ) { - let error = audio_transcription(AudioTranscriptionRequest { + let error = transcribe(AudioTranscriptionRequest { model, custom_llm_provider: provider, api_base: Some(UNREACHABLE_BASE), @@ -189,7 +193,7 @@ async fn unsupported_providers_are_rejected_before_sending( #[rstest] #[tokio::test] async fn a_non_string_extra_header_is_rejected(request: AudioTranscriptionRequest<'static>) { - let error = audio_transcription(AudioTranscriptionRequest { + let error = transcribe(AudioTranscriptionRequest { extra_headers: Some(Map::from_iter([("x-count".to_string(), json!(3))])), api_base: Some(UNREACHABLE_BASE), ..request @@ -212,7 +216,7 @@ async fn an_upstream_error_keeps_its_status_and_body( upstream([ResponseTemplate::new(status).set_body_string("upstream said no")]).await; let base = upstream.uri(); - let error = audio_transcription(AudioTranscriptionRequest { + let error = transcribe(AudioTranscriptionRequest { api_base: Some(&base), ..request }) @@ -239,7 +243,7 @@ async fn an_unreadable_success_body_is_an_invalid_response( let upstream = upstream([response]).await; let base = upstream.uri(); - let error = audio_transcription(AudioTranscriptionRequest { + let error = transcribe(AudioTranscriptionRequest { api_base: Some(&base), ..request }) diff --git a/litellm-rust/crates/core/tests/chat_completions.rs b/litellm-rust/crates/core/tests/chat_completions.rs index ae96509fe2e..d1f6cde19e8 100644 --- a/litellm-rust/crates/core/tests/chat_completions.rs +++ b/litellm-rust/crates/core/tests/chat_completions.rs @@ -4,6 +4,7 @@ use litellm_core::chat_completions::{ Error, chat_completions, chat_completions_decline_reason, types::ChatCompletionsRequest, }; use litellm_http::transport::Error as TransportError; +use litellm_types::utils::ChatCompletionsResponse; use rstest::{fixture, rstest}; use serde_json::{Map, Value, json}; use wiremock::ResponseTemplate; @@ -13,6 +14,10 @@ use support::*; const ANTHROPIC_MESSAGE: &str = r#"{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-5-20260101","content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":11,"output_tokens":4}}"#; +async fn complete(request: ChatCompletionsRequest<'_>) -> Result { + chat_completions(&http_pool(), &http_config(), request).await +} + fn object(value: Value) -> Map { let Value::Object(map) = value else { panic!("expected a json object, got {value}"); @@ -50,7 +55,7 @@ async fn anthropic_round_trip_translates_the_conversation_and_normalizes_the_res let upstream = upstream([anthropic_response(ANTHROPIC_MESSAGE)]).await; let base = upstream.uri(); - let response = chat_completions(ChatCompletionsRequest { + let response = complete(ChatCompletionsRequest { messages: json!([ {"role": "system", "content": "be terse"}, {"role": "user", "content": "hi"} @@ -90,7 +95,7 @@ async fn the_deployment_key_replaces_a_caller_supplied_x_api_key( let upstream = upstream([anthropic_response(ANTHROPIC_MESSAGE)]).await; let base = upstream.uri(); - chat_completions(ChatCompletionsRequest { + complete(ChatCompletionsRequest { api_base: Some(&base), extra_headers: Some(object( json!({"x-api-key": "caller-key", "x-trace": "kept"}), @@ -116,7 +121,7 @@ async fn bedrock_round_trip_is_signed_and_normalized(request: ChatCompletionsReq .await; let base = upstream.uri(); - let response = chat_completions(ChatCompletionsRequest { + let response = complete(ChatCompletionsRequest { model: "bedrock/anthropic.claude-sonnet-4-5", optional_params: object(json!({ "aws_access_key_id": "access-key", @@ -167,7 +172,7 @@ async fn a_response_it_cannot_normalize_is_reported_as_already_sent( let upstream = upstream([anthropic_response(body)]).await; let base = upstream.uri(); - let error = chat_completions(ChatCompletionsRequest { + let error = complete(ChatCompletionsRequest { api_base: Some(&base), ..request }) @@ -188,7 +193,7 @@ async fn an_upstream_error_status_keeps_its_code_and_body( let upstream = upstream([ResponseTemplate::new(status).set_body_string("slow down")]).await; let base = upstream.uri(); - let error = chat_completions(ChatCompletionsRequest { + let error = complete(ChatCompletionsRequest { api_base: Some(&base), ..request }) @@ -210,7 +215,7 @@ async fn an_upstream_error_status_keeps_its_code_and_body( async fn a_connection_that_is_never_established_declines_instead_of_failing( request: ChatCompletionsRequest<'static>, ) { - let error = chat_completions(ChatCompletionsRequest { + let error = complete(ChatCompletionsRequest { api_base: Some(UNREACHABLE_BASE), ..request }) @@ -232,7 +237,7 @@ async fn a_timeout_after_sending_is_not_a_pre_send_decline( upstream([anthropic_response(ANTHROPIC_MESSAGE).set_delay(Duration::from_secs(5))]).await; let base = upstream.uri(); - let error = chat_completions(ChatCompletionsRequest { + let error = complete(ChatCompletionsRequest { api_base: Some(&base), timeout: Some(Duration::from_millis(100)), ..request @@ -307,7 +312,7 @@ async fn a_declined_request_fails_the_call_before_sending( let upstream = upstream([anthropic_response(ANTHROPIC_MESSAGE)]).await; let base = upstream.uri(); - let error = chat_completions(ChatCompletionsRequest { + let error = complete(ChatCompletionsRequest { optional_params: object(json!({"stream": true})), api_base: Some(&base), ..request diff --git a/litellm-rust/crates/core/tests/messages/host.rs b/litellm-rust/crates/core/tests/messages/host.rs index ca2aece5ebd..844ada3e1ad 100644 --- a/litellm-rust/crates/core/tests/messages/host.rs +++ b/litellm-rust/crates/core/tests/messages/host.rs @@ -78,7 +78,7 @@ impl Host for RecordingHost { } async fn run_through(host: &RecordingHost) -> Result { - litellm_host::run::run(messages_machine(Arc::new(RecordingSecrets::empty())), host).await + litellm_host::run::run(machine(Arc::new(RecordingSecrets::empty())), host).await } fn authenticated(call: MessagesCall, api_base: String) -> MessagesCall { diff --git a/litellm-rust/crates/core/tests/messages/main.rs b/litellm-rust/crates/core/tests/messages/main.rs index 21ee678ced3..1ae822e5437 100644 --- a/litellm-rust/crates/core/tests/messages/main.rs +++ b/litellm-rust/crates/core/tests/messages/main.rs @@ -2,9 +2,11 @@ use std::{sync::Arc, time::Duration}; use litellm_core::messages::{ Error, - route::{LocalMessagesHost, MessagesCall, MessagesOutput, messages_machine}, + route::{LocalMessagesHost, MessagesCall, MessagesMachine, MessagesOutput, messages_machine}, types::MessagesShaping, }; +use litellm_http::{HttpSettings, Resolution}; +use litellm_secrets::source::SecretSource; use litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse; use rstest::fixture; use serde_json::{Map, Value, json}; @@ -75,11 +77,16 @@ fn headers<'a>(pairs: impl IntoIterator) -> Option) -> MessagesMachine { + messages_machine(&http_pool(), &http_config(), secrets) + .expect("default HTTP settings build a client") +} + async fn run_with( secrets: Arc, call: MessagesCall, ) -> Result { - litellm_host::run::run(messages_machine(secrets), &LocalMessagesHost::new(call)).await + litellm_host::run::run(machine(secrets), &LocalMessagesHost::new(call)).await } /// Runs the route with a secret source that knows nothing, so no environment leaks in. diff --git a/litellm-rust/crates/core/tests/messages/response.rs b/litellm-rust/crates/core/tests/messages/response.rs index 133b7d2b162..431dd4f4b93 100644 --- a/litellm-rust/crates/core/tests/messages/response.rs +++ b/litellm-rust/crates/core/tests/messages/response.rs @@ -190,29 +190,40 @@ fn facade_request(body: Value, api_base: &str) -> MessagesRequest<'_> { } #[tokio::test] -async fn the_facade_runs_the_route_in_process() { +async fn the_facade_sends_through_the_injected_http_pool_configuration() { let upstream = upstream([message_response()]).await; let base = upstream.uri(); + let settings = HttpSettings { + user_agent: Some("host-owned/1".into()), + ..HttpSettings::default() + }; - let message = messages(facade_request( - json!({"model": MODEL, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]}), - &base, - )) + let message = messages( + &http_pool(), + &Resolution::from(&settings).config, + facade_request( + json!({"model": MODEL, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]}), + &base, + ), + ) .await .expect("messages request succeeds"); assert_eq!(message.id, "msg_1"); - assert_eq!( - only_request(&upstream).await.header("x-api-key"), - Some("sk-ant") - ); + let sent = only_request(&upstream).await; + assert_eq!(sent.header("x-api-key"), Some("sk-ant")); + assert_eq!(sent.header("user-agent"), Some("host-owned/1")); } #[tokio::test] async fn the_facade_rejects_a_body_that_is_not_an_object() { - let error = messages(facade_request(json!([]), UNREACHABLE_BASE)) - .await - .expect_err("a non-object body is rejected"); + let error = messages( + &http_pool(), + &http_config(), + facade_request(json!([]), UNREACHABLE_BASE), + ) + .await + .expect_err("a non-object body is rejected"); assert_eq!( error, diff --git a/litellm-rust/crates/core/tests/messages/stream.rs b/litellm-rust/crates/core/tests/messages/stream.rs index c4be3127d66..4ca6e609052 100644 --- a/litellm-rust/crates/core/tests/messages/stream.rs +++ b/litellm-rust/crates/core/tests/messages/stream.rs @@ -87,7 +87,7 @@ fn sse_response() -> ResponseTemplate { } async fn stream_through(host: &RecordingStreamHost) -> Result { - litellm_host::run::run(messages_machine(Arc::new(RecordingSecrets::empty())), host).await + litellm_host::run::run(machine(Arc::new(RecordingSecrets::empty())), host).await } #[rstest] diff --git a/litellm-rust/crates/core/tests/ocr/main.rs b/litellm-rust/crates/core/tests/ocr/main.rs index 1a915389b20..e1f6b8cb5c1 100644 --- a/litellm-rust/crates/core/tests/ocr/main.rs +++ b/litellm-rust/crates/core/tests/ocr/main.rs @@ -4,6 +4,7 @@ use litellm_core::ocr::{ types::LiteLLMOcrRequest, wire::{OcrWireRequest, decode_request}, }; +use litellm_http::Client; use litellm_llms::base_llm::ocr::{ error::Error, handler::OcrClient, @@ -37,11 +38,7 @@ fn object(value: Value) -> Map { } fn ocr_client() -> OcrClient { - let document_http = reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("test document client builds"); - OcrClient::for_test(reqwest::Client::new(), document_http) + OcrClient::for_test(Client::plain_for_test(), Client::no_redirect_for_test()) } async fn perform(request: LiteLLMOcrRequest) -> Result { diff --git a/litellm-rust/crates/core/tests/ocr/mistral.rs b/litellm-rust/crates/core/tests/ocr/mistral.rs index f80e564b03f..4c3f1c5cc39 100644 --- a/litellm-rust/crates/core/tests/ocr/mistral.rs +++ b/litellm-rust/crates/core/tests/ocr/mistral.rs @@ -1,10 +1,7 @@ use std::sync::Arc; use litellm_auth_gcp::VertexAuth; -use litellm_http::{ - HttpClientPool, HttpSettings, Resolution, - media::{PublicDnsResolver, UrlPolicy}, -}; +use litellm_http::{HttpSettings, Resolution, media::UrlPolicy}; use litellm_llms::{ base_llm::ocr::{ settings::OcrSettings, @@ -192,12 +189,16 @@ async fn the_client_uses_the_injected_http_pool_configuration() { ..HttpSettings::default() }; let client = OcrClient::new( - &HttpClientPool::new(Arc::new(PublicDnsResolver)), + &http_pool(), &Resolution::from(&settings).config, UrlPolicy::default(), VertexAuth::default(), OcrSettings::default(), - Arc::new(litellm_secrets::source::EnvironmentSecrets::default()), + Arc::new( + litellm_secrets::source::EnvironmentSecrets::python_compatible( + litellm_http::Client::plain_for_test(), + ), + ), ) .unwrap(); diff --git a/litellm-rust/crates/core/tests/support/mod.rs b/litellm-rust/crates/core/tests/support/mod.rs index 4d2fe0232d0..1d9af236811 100644 --- a/litellm-rust/crates/core/tests/support/mod.rs +++ b/litellm-rust/crates/core/tests/support/mod.rs @@ -3,9 +3,12 @@ #![allow(dead_code)] // each test binary compiles this module on its own and uses a different subset -use std::sync::Mutex; +use std::sync::{Arc, Mutex}; use futures_util::future::BoxFuture; +use litellm_http::{ + HttpClientConfig, HttpClientPool, HttpSettings, Resolution, media::PublicDnsResolver, +}; use litellm_secrets::{SecretValue, source::SecretSource}; use serde_json::Value; use wiremock::{Mock, MockServer, Request, ResponseTemplate, matchers::any}; @@ -13,6 +16,14 @@ use wiremock::{Mock, MockServer, Request, ResponseTemplate, matchers::any}; /// A port nothing listens on, for calls that must fail before any request is sent. pub const UNREACHABLE_BASE: &str = "http://127.0.0.1:1"; +pub fn http_pool() -> HttpClientPool { + HttpClientPool::new(Arc::new(PublicDnsResolver)) +} + +pub fn http_config() -> HttpClientConfig { + Resolution::from(&HttpSettings::default()).config +} + /// Starts an upstream that answers its n-th request with the n-th response and 404s after. pub async fn upstream(responses: impl IntoIterator) -> MockServer { let server = MockServer::start().await; diff --git a/litellm-rust/crates/http/Cargo.toml b/litellm-rust/crates/http/Cargo.toml index cad5aa87e49..0cb2b15b768 100644 --- a/litellm-rust/crates/http/Cargo.toml +++ b/litellm-rust/crates/http/Cargo.toml @@ -22,5 +22,7 @@ veil.workspace = true webpki-roots.workspace = true [dev-dependencies] +rcgen = "0.14.10" +tempfile.workspace = true rstest.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/http/src/client.rs b/litellm-rust/crates/http/src/client.rs new file mode 100644 index 00000000000..1f7017d083b --- /dev/null +++ b/litellm-rust/crates/http/src/client.rs @@ -0,0 +1,38 @@ +use std::ops::Deref; + +#[derive(Clone, Debug)] +pub struct Client(reqwest::Client); + +impl Client { + pub(crate) fn new(client: reqwest::Client) -> Self { + Self(client) + } + + #[cfg(any(test, feature = "test-support"))] + pub fn for_test(client: reqwest::Client) -> Self { + Self(client) + } + + #[cfg(any(test, feature = "test-support"))] + pub fn plain_for_test() -> Self { + Self(reqwest::Client::new()) + } + + #[cfg(any(test, feature = "test-support"))] + pub fn no_redirect_for_test() -> Self { + Self( + reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("a client without TLS or proxy settings builds"), + ) + } +} + +impl Deref for Client { + type Target = reqwest::Client; + + fn deref(&self) -> &reqwest::Client { + &self.0 + } +} diff --git a/litellm-rust/crates/http/src/config.rs b/litellm-rust/crates/http/src/config.rs index cb0173369d5..2f36784bc70 100644 --- a/litellm-rust/crates/http/src/config.rs +++ b/litellm-rust/crates/http/src/config.rs @@ -18,10 +18,16 @@ pub enum Verify { BuiltInRoots, } +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum ClientIdentity { + Pem(PathBuf), + Split { certificate: PathBuf, key: PathBuf }, +} + #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct HttpClientConfig { pub verify: Verify, - pub client_certificate: Option, + pub client_certificate: Option, pub key_exchange_group: Option, pub tls12_cipher_suites: Option>, pub force_ipv4: bool, @@ -67,7 +73,7 @@ impl From<&HttpSettings> for Resolution { Self { config: HttpClientConfig { verify: Verify::from(settings), - client_certificate: settings.ssl_certificate.clone(), + client_certificate: settings.ssl_certificate.clone().map(ClientIdentity::Pem), key_exchange_group: curve.clone().ok().flatten(), tls12_cipher_suites: ciphers.tls12_cipher_suites, force_ipv4: settings.force_ipv4, @@ -276,7 +282,7 @@ mod tests { config, HttpClientConfig { verify: Verify::BuiltInRoots, - client_certificate: Some("/client.pem".into()), + client_certificate: Some(ClientIdentity::Pem("/client.pem".into())), key_exchange_group: None, tls12_cipher_suites: None, force_ipv4: true, diff --git a/litellm-rust/crates/http/src/lib.rs b/litellm-rust/crates/http/src/lib.rs index a1456208bb3..3e55a1843c8 100644 --- a/litellm-rust/crates/http/src/lib.rs +++ b/litellm-rust/crates/http/src/lib.rs @@ -1,3 +1,10 @@ +#![allow( + clippy::disallowed_types, + clippy::disallowed_methods, + reason = "this crate is the one place reqwest clients are built" +)] + +mod client; mod config; mod error; pub mod media; @@ -9,7 +16,8 @@ mod settings; mod tls; pub mod transport; -pub use config::{HttpClientConfig, Resolution, Verify}; +pub use client::Client; +pub use config::{ClientIdentity, HttpClientConfig, Resolution, Verify}; pub use error::{Error, TlsSource}; pub use pool::{ClientVariant, HttpClientPool}; pub use proxy::EnvironmentProxies; diff --git a/litellm-rust/crates/http/src/media.rs b/litellm-rust/crates/http/src/media.rs index 1b9159973ef..1dac68305b0 100644 --- a/litellm-rust/crates/http/src/media.rs +++ b/litellm-rust/crates/http/src/media.rs @@ -12,7 +12,7 @@ use reqwest::{ dns::{Addrs, Name, Resolve, Resolving}, }; -use crate::{ClientVariant, HttpClientConfig, HttpClientPool}; +use crate::{Client, ClientVariant, HttpClientConfig, HttpClientPool}; #[derive(Debug, thiserror::Error)] pub enum Error { @@ -93,8 +93,8 @@ type ProxyMatch = Arc bool + Send + Sync>; #[derive(Clone)] pub struct MediaFetcher { - pinned: reqwest::Client, - unpinned: reqwest::Client, + pinned: Client, + unpinned: Client, uses_proxy: ProxyMatch, address_resolver: Arc, url_policy: UrlPolicy, @@ -154,7 +154,7 @@ impl MediaFetcher { } #[cfg(any(test, feature = "test-support"))] - pub fn for_test(client: reqwest::Client) -> Self { + pub fn for_test(client: Client) -> Self { Self { pinned: client.clone(), unpinned: client, @@ -230,7 +230,7 @@ impl MediaFetcher { } } - async fn client_for(&self, url: &Url) -> Result<&reqwest::Client, Error> { + async fn client_for(&self, url: &Url) -> Result<&Client, Error> { if !self.url_policy.validate { return Ok(&self.unpinned); } @@ -520,10 +520,7 @@ mod tests { b"HTTP/1.1 200 OK\r\nContent-Type: application/pdf; charset=binary\r\nContent-Length: 3\r\nConnection: close\r\n\r\nabc", ) .await; - let client = reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("test client builds"); + let client = Client::no_redirect_for_test(); let media = MediaFetcher::for_test(client) .fetch(url, policy(3, 0)) .await @@ -539,10 +536,7 @@ mod tests { b"HTTP/1.1 200 OK\r\nContent-Type: application/pdf\r\nContent-Length: 3\r\nConnection: close\r\n\r\nabc", ) .await; - let client = reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("test client builds"); + let client = Client::no_redirect_for_test(); let error = MediaFetcher::for_test(client) .fetch(url, policy(2, 0)) .await @@ -557,10 +551,7 @@ mod tests { b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n2\r\nab\r\n2\r\ncd\r\n0\r\n\r\n", ) .await; - let client = reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("test client builds"); + let client = Client::no_redirect_for_test(); let error = MediaFetcher::for_test(client) .fetch(url, policy(3, 0)) .await diff --git a/litellm-rust/crates/http/src/outbound.rs b/litellm-rust/crates/http/src/outbound.rs index d100bdf624b..c2cfb00d79b 100644 --- a/litellm-rust/crates/http/src/outbound.rs +++ b/litellm-rust/crates/http/src/outbound.rs @@ -107,7 +107,7 @@ impl OutboundRequest { self.timeout } - pub async fn send(self, client: &reqwest::Client) -> Result { + pub async fn send(self, client: &crate::Client) -> Result { let builder = with_headers( client.post(&self.url).body(self.body), &self.headers, diff --git a/litellm-rust/crates/http/src/pool.rs b/litellm-rust/crates/http/src/pool.rs index ee47e5dc52a..1187c34f2d7 100644 --- a/litellm-rust/crates/http/src/pool.rs +++ b/litellm-rust/crates/http/src/pool.rs @@ -6,7 +6,7 @@ use std::{ use reqwest::dns::Resolve; -use crate::{config::HttpClientConfig, error::Error, proxy::EnvironmentProxies}; +use crate::{client::Client, config::HttpClientConfig, error::Error, proxy::EnvironmentProxies}; #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum ClientVariant { @@ -48,7 +48,7 @@ impl HttpClientPool { &self, config: &HttpClientConfig, variant: ClientVariant, - ) -> Result { + ) -> Result { let effective = match variant { ClientVariant::Media => HttpClientConfig { client_certificate: None, @@ -65,7 +65,7 @@ impl HttpClientPool { if let Some(pooled) = self.lock().get(&key) && pooled.built_at.elapsed() < self.ttl { - return Ok(pooled.client.clone()); + return Ok(Client::new(pooled.client.clone())); } let client = self .apply(variant, reqwest::ClientBuilder::try_from(&key.0)?) @@ -77,7 +77,7 @@ impl HttpClientPool { built_at: Instant::now(), }, ); - Ok(client) + Ok(Client::new(client)) } fn lock(&self) -> MutexGuard<'_, Clients> { @@ -116,7 +116,7 @@ mod tests { }; use super::*; - use crate::{HttpSettings, Resolution, Verify}; + use crate::{ClientIdentity, HttpSettings, Resolution, Verify}; struct FixedResolver(SocketAddr); @@ -288,7 +288,9 @@ mod tests { fn media_variant_never_loads_the_client_certificate() { let pool = pool(); let with_identity = HttpClientConfig { - client_certificate: Some(std::env::temp_dir().join("litellm-http-absent-client.pem")), + client_certificate: Some(ClientIdentity::Pem( + std::env::temp_dir().join("litellm-http-absent-client.pem"), + )), ..config("a") }; assert!( diff --git a/litellm-rust/crates/http/src/tls.rs b/litellm-rust/crates/http/src/tls.rs index e2e6d27cd54..c58076607e4 100644 --- a/litellm-rust/crates/http/src/tls.rs +++ b/litellm-rust/crates/http/src/tls.rs @@ -8,7 +8,7 @@ use rustls::{ }; use crate::{ - config::{HttpClientConfig, Verify}, + config::{ClientIdentity, HttpClientConfig, Verify}, error::{Error, TlsSource}, }; @@ -203,11 +203,15 @@ impl TryFrom<&HttpClientConfig> for ClientConfig { }; let mut tls = match &config.client_certificate { None => verified.with_no_client_auth(), - Some(path) => { - let (chain, key) = identity(path, TlsSource::ClientIdentity)?; + Some(identity) => { + let (certificate, key) = match identity { + ClientIdentity::Pem(path) => (path, path), + ClientIdentity::Split { certificate, key } => (certificate, key), + }; + let (chain, private_key) = client_identity(certificate, key)?; verified - .with_client_auth_cert(chain, key) - .map_err(|error| invalid_pem(path, TlsSource::ClientIdentity, error))? + .with_client_auth_cert(chain, private_key) + .map_err(|error| invalid_pem(key, TlsSource::ClientIdentity, error))? } }; tls.alpn_protocols = if config.http2 { @@ -233,17 +237,18 @@ fn bundle_roots(path: &Path, source: TlsSource) -> Result Ok(store) } -fn identity( - path: &Path, - source: TlsSource, +fn client_identity( + certificate: &Path, + key: &Path, ) -> Result<(Vec>, PrivateKeyDer<'static>), Error> { - let chain = certificates(path, source)?; + let source = TlsSource::ClientIdentity; + let chain = certificates(certificate, source)?; if chain.is_empty() { - return Err(invalid_pem(path, source, "no certificates found")); + return Err(invalid_pem(certificate, source, "no certificates found")); } - let key = PrivateKeyDer::from_pem_slice(&read(path, source)?) - .map_err(|error| invalid_pem(path, source, error))?; - Ok((chain, key)) + let private_key = PrivateKeyDer::from_pem_slice(&read(key, source)?) + .map_err(|error| invalid_pem(key, source, error))?; + Ok((chain, private_key)) } fn certificates(path: &Path, source: TlsSource) -> Result>, Error> { @@ -405,7 +410,7 @@ mod tests { ) .unwrap(); let result = ClientConfig::try_from(&HttpClientConfig { - client_certificate: Some(path.clone()), + client_certificate: Some(ClientIdentity::Pem(path.clone())), ..config(HttpSettings::default()) }) .map(drop); @@ -419,4 +424,29 @@ mod tests { }) if reported == path )); } + + #[test] + fn split_client_identity_reads_the_key_from_its_own_file() { + let identity = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let directory = tempfile::tempdir().unwrap(); + let certificate = directory.path().join("client.crt"); + let key = directory.path().join("client.key"); + std::fs::write(&certificate, identity.cert.pem()).unwrap(); + std::fs::write(&key, identity.signing_key.serialize_pem()).unwrap(); + + let split = ClientConfig::try_from(&HttpClientConfig { + client_certificate: Some(ClientIdentity::Split { + certificate: certificate.clone(), + key, + }), + ..config(HttpSettings::default()) + }); + let combined = ClientConfig::try_from(&HttpClientConfig { + client_certificate: Some(ClientIdentity::Pem(certificate)), + ..config(HttpSettings::default()) + }); + + assert!(split.unwrap().client_auth_cert_resolver.has_certs()); + assert!(combined.is_err()); + } } diff --git a/litellm-rust/crates/llms/Cargo.toml b/litellm-rust/crates/llms/Cargo.toml index ed15d9f7cdb..36ccd18f220 100644 --- a/litellm-rust/crates/llms/Cargo.toml +++ b/litellm-rust/crates/llms/Cargo.toml @@ -35,6 +35,7 @@ tokio = { workspace = true, features = ["sync"] } url.workspace = true [dev-dependencies] +litellm-http = { workspace = true, features = ["test-support"] } aws-smithy-eventstream = "=0.61.4" aws-smithy-types = "1.6.1" rstest.workspace = true diff --git a/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs index 51a2668310e..b4e9d01f867 100644 --- a/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs @@ -450,7 +450,7 @@ fn pixel_dimension(value: f64, scale: f64, field: &'static str) -> Result: Send + Sync { #[derive(Clone)] pub struct OcrClient { - provider_http: reqwest::Client, - polling_http: reqwest::Client, + provider_http: Client, + polling_http: Client, document_fetcher: MediaFetcher, vertex_auth: VertexAuth, settings: OcrSettings, @@ -60,11 +60,11 @@ impl OcrClient { }) } - pub fn provider_http(&self) -> &reqwest::Client { + pub fn provider_http(&self) -> &Client { &self.provider_http } - pub fn polling_http(&self) -> &reqwest::Client { + pub fn polling_http(&self) -> &Client { &self.polling_http } @@ -85,17 +85,18 @@ impl OcrClient { } #[cfg(any(test, feature = "test-support"))] - pub fn for_test(provider_http: reqwest::Client, document_http: reqwest::Client) -> Self { + pub fn for_test(provider_http: Client, no_redirect_http: Client) -> Self { Self { + secrets: Arc::new( + litellm_secrets::source::EnvironmentSecrets::python_compatible( + provider_http.clone(), + ), + ), provider_http, - polling_http: reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("test polling client builds"), - document_fetcher: MediaFetcher::for_test(document_http), + polling_http: no_redirect_http.clone(), + document_fetcher: MediaFetcher::for_test(no_redirect_http), vertex_auth: VertexAuth::default(), settings: OcrSettings::default(), - secrets: Arc::new(litellm_secrets::source::EnvironmentSecrets::default()), } } @@ -311,7 +312,7 @@ mod tests { let _connection = listener.accept().await.unwrap(); tokio::time::sleep(Duration::from_secs(1)).await; }); - let error = reqwest::Client::new() + let error = litellm_http::Client::plain_for_test() .get(format!("http://{address}")) .timeout(Duration::from_millis(10)) .send() diff --git a/litellm-rust/crates/llms/src/reducto/ocr/transformation.rs b/litellm-rust/crates/llms/src/reducto/ocr/transformation.rs index c3377536545..147056dab8d 100644 --- a/litellm-rust/crates/llms/src/reducto/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/reducto/ocr/transformation.rs @@ -564,7 +564,10 @@ mod tests { let params = ReductoParseV3Config .map_ocr_params(&overrides, "parse-v3") .unwrap(); - let client = OcrClient::for_test(reqwest::Client::new(), reqwest::Client::new()); + let client = OcrClient::for_test( + litellm_http::Client::plain_for_test(), + litellm_http::Client::no_redirect_for_test(), + ); let connection = OcrConnection::default(); let document = serde_json::from_value( json!({"type":"document_url","document_url":"reducto://ready.pdf"}), diff --git a/litellm-rust/crates/llms/tests/ocr_handler.rs b/litellm-rust/crates/llms/tests/ocr_handler.rs index 6e46e6f76d4..5ed3244087d 100644 --- a/litellm-rust/crates/llms/tests/ocr_handler.rs +++ b/litellm-rust/crates/llms/tests/ocr_handler.rs @@ -19,7 +19,7 @@ async fn read_bounded(response: String, limit: usize) -> Result().await; }); - let response = reqwest::Client::new() + let response = litellm_http::Client::plain_for_test() .get(format!("http://{address}")) .send() .await diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 057cad2f42e..d965223bd59 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -62,6 +62,7 @@ tokio = { workspace = true, features = ["rt", "sync"] } url.workspace = true [dev-dependencies] +litellm-http = { workspace = true, features = ["test-support"] } litellm-secrets-aws.workspace = true serde.workspace = true serde_with.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/cache/activation.rs b/litellm-rust/crates/python-bridge/src/cache/activation.rs index f77032c579d..58735679554 100644 --- a/litellm-rust/crates/python-bridge/src/cache/activation.rs +++ b/litellm-rust/crates/python-bridge/src/cache/activation.rs @@ -9,10 +9,10 @@ use super::{ cache_error, config::{CacheBackendConfig, NativeCacheConfig, UnsupportedCacheConfig}, embedder::PythonEmbedder, - host_client, native::NativeResponseCache, }; use crate::errors::RustBridgeDeclined; +use crate::http::host_client; fn declined(reason: UnsupportedCacheConfig) -> PyErr { RustBridgeDeclined::new_err(reason.message()) diff --git a/litellm-rust/crates/python-bridge/src/cache/config.rs b/litellm-rust/crates/python-bridge/src/cache/config.rs index 6e25f07efa1..e58902b07ee 100644 --- a/litellm-rust/crates/python-bridge/src/cache/config.rs +++ b/litellm-rust/crates/python-bridge/src/cache/config.rs @@ -1511,7 +1511,7 @@ mod tests { path_service_account: Some("credentials.json".into()), endpoint: litellm_cache_gcs::DEFAULT_ENDPOINT.into(), }, - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), Some("token".into()), ); let matching_config = NativeCacheConfig { diff --git a/litellm-rust/crates/python-bridge/src/cache/handle.rs b/litellm-rust/crates/python-bridge/src/cache/handle.rs index e14916b25c6..fcc8aa6218a 100644 --- a/litellm-rust/crates/python-bridge/src/cache/handle.rs +++ b/litellm-rust/crates/python-bridge/src/cache/handle.rs @@ -1,3 +1,4 @@ +use crate::http::host_client; use crate::logger::run_sync_value; use litellm_auth_aws::AwsAuthConfig; use litellm_cache_gcs::{DEFAULT_ENDPOINT, GcsConfig}; @@ -19,7 +20,6 @@ use super::{ config::{QdrantSemanticCacheConfig, project_redis_semantic}, embedder::PythonEmbedder, facade::FacadeGuard, - host_client, native::NativeResponseCache, request::duration, }; diff --git a/litellm-rust/crates/python-bridge/src/cache/mod.rs b/litellm-rust/crates/python-bridge/src/cache/mod.rs index ac1e00d5273..00b0c71684a 100644 --- a/litellm-rust/crates/python-bridge/src/cache/mod.rs +++ b/litellm-rust/crates/python-bridge/src/cache/mod.rs @@ -13,11 +13,9 @@ mod resolver; mod semantic; use litellm_cache::Error; -use litellm_http::ClientVariant; use pyo3::{ exceptions::{PyNotImplementedError, PyRuntimeError, PyValueError}, prelude::*, - types::PyDict, }; pub(crate) use self::{binding::ResolvedCache, handle::CacheTestHandle, resolver::CacheResolver}; @@ -29,11 +27,3 @@ fn cache_error(error: Error) -> PyErr { _ => PyRuntimeError::new_err(error.to_string()), } } - -/// The host's pooled HTTP client, configured from the proxy's HTTP settings. -fn host_client(py: Python<'_>, variant: ClientVariant) -> PyResult { - let http_config = crate::http::call_config(py, &PyDict::new(py), true)?; - crate::http::pool() - .client(&http_config, variant) - .map_err(crate::http::client_error) -} diff --git a/litellm-rust/crates/python-bridge/src/cache/native.rs b/litellm-rust/crates/python-bridge/src/cache/native.rs index 0e279046812..460136baa1f 100644 --- a/litellm-rust/crates/python-bridge/src/cache/native.rs +++ b/litellm-rust/crates/python-bridge/src/cache/native.rs @@ -92,7 +92,7 @@ impl NativeResponseCache { )) } - pub async fn s3(config: S3CacheConfig, http: reqwest::Client) -> Self { + pub async fn s3(config: S3CacheConfig, http: litellm_http::Client) -> Self { let runtime = tokio::runtime::Handle::current(); let backend = S3Cache::new(config, http, ResponseCacheCodec, runtime); let identity = BackendIdentity::S3 { @@ -112,7 +112,7 @@ impl NativeResponseCache { Ok(Self::exact(ResponseCache::new(Arc::new(backend)), identity)) } - pub fn gcs(config: GcsConfig, client: reqwest::Client, token: Option) -> Self { + pub fn gcs(config: GcsConfig, client: litellm_http::Client, token: Option) -> Self { let backend = match token { Some(token) => GcsCache::with_token_source( config, @@ -133,7 +133,7 @@ impl NativeResponseCache { pub async fn azure_blob( account_url: &str, container: &str, - http: reqwest::Client, + http: litellm_http::Client, ) -> Result { let backend = AzureBlobCache::connect( account_url, @@ -242,7 +242,7 @@ impl NativeResponseCache { pub async fn qdrant_semantic( config: QdrantSemanticCacheConfig, - client: reqwest::Client, + client: litellm_http::Client, runtime: tokio::runtime::Handle, ) -> Result { let qdrant = qdrant_client::Qdrant::from_url(&config.grpc_url) diff --git a/litellm-rust/crates/python-bridge/src/http.rs b/litellm-rust/crates/python-bridge/src/http.rs index 3dad3447f45..4d8f0fd7147 100644 --- a/litellm-rust/crates/python-bridge/src/http.rs +++ b/litellm-rust/crates/python-bridge/src/http.rs @@ -6,8 +6,8 @@ use std::{ use litellm_core_utils::settings::ProcessEnvironment; use litellm_http::{ - HttpClientConfig, HttpClientPool, HttpSettings, HttpSettingsLayer, Resolution, SslVerify, - TlsSource, Unsupported, + Client, ClientVariant, HttpClientConfig, HttpClientPool, HttpSettings, HttpSettingsLayer, + Resolution, SslVerify, TlsSource, Unsupported, media::{PublicDnsResolver, UrlPolicy}, }; use pyo3::{ @@ -97,7 +97,10 @@ pub(crate) fn call_config( let settings = HttpSettings::from_layers([ for_call(call_ssl_verify(kwargs)?, asynchronous), HttpSettingsLayer::from_environment(&ProcessEnvironment), - configured(&PythonSettings::Http.read(py)?)?, + match PythonSettings::Http.read_or_unset(py)? { + Some(snapshot) => configured(&snapshot)?, + None => HttpSettingsLayer::default(), + }, ]) .without_missing_files(&|path: &Path| path.exists()); let resolution = Resolution::from(&settings); @@ -107,6 +110,11 @@ pub(crate) fn call_config( Ok(resolution.config) } +pub(crate) fn host_client(py: Python<'_>, variant: ClientVariant) -> PyResult { + let config = call_config(py, &PyDict::new(py), true)?; + pool().client(&config, variant).map_err(client_error) +} + pub(crate) fn client_error(error: litellm_http::Error) -> PyErr { match error { litellm_http::Error::Read { @@ -143,7 +151,10 @@ fn unreported( } pub(crate) fn url_policy(py: Python<'_>) -> PyResult { - project_url_policy(&PythonSettings::UrlPolicy.read(py)?) + match PythonSettings::UrlPolicy.read_or_unset(py)? { + Some(snapshot) => project_url_policy(&snapshot), + None => Ok(UrlPolicy::default()), + } } fn project_url_policy(snapshot: &Snapshot<'_>) -> PyResult { diff --git a/litellm-rust/crates/python-bridge/src/python_settings.rs b/litellm-rust/crates/python-bridge/src/python_settings.rs index abf664b795d..f03e5fdce7f 100644 --- a/litellm-rust/crates/python-bridge/src/python_settings.rs +++ b/litellm-rust/crates/python-bridge/src/python_settings.rs @@ -1,4 +1,4 @@ -use pyo3::prelude::*; +use pyo3::{exceptions::PyModuleNotFoundError, prelude::*}; use crate::coercion::{FieldSpec, ProjectionError}; @@ -40,6 +40,16 @@ impl PythonSettings { Ok(Snapshot { group: self, value }) } + /// Reads the accessor, or `None` when the litellm package is not installed + /// (a bare extension module), meaning there are no configured values. + pub(crate) fn read_or_unset(self, py: Python<'_>) -> PyResult>> { + match self.read(py) { + Ok(snapshot) => Ok(Some(snapshot)), + Err(error) if error.is_instance_of::(py) => Ok(None), + Err(error) => Err(error), + } + } + #[cfg(test)] pub(crate) fn snapshot(self, value: Bound<'_, PyAny>) -> Snapshot<'_> { Snapshot { group: self, value } diff --git a/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs b/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs index dec4dcea21c..93d0e11d323 100644 --- a/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs +++ b/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs @@ -3,7 +3,8 @@ use litellm_core::audio_transcription::{ Error, audio_transcription as run_audio_transcription, types::AudioTranscriptionRequest, }; use litellm_host_python::from_py_argument; -use pyo3::prelude::*; +use litellm_http::HttpClientConfig; +use pyo3::{prelude::*, types::PyDict}; use serde_json::{Map, Value}; use crate::{ @@ -12,6 +13,7 @@ use crate::{ }; async fn execute( + config: HttpClientConfig, audio: Value, optional_params: Map, options: RouteOptions, @@ -24,16 +26,20 @@ async fn execute( extra_headers, timeout, } = options; - run_audio_transcription(AudioTranscriptionRequest { - model: &model, - audio, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - optional_params, - timeout, - }) + run_audio_transcription( + crate::http::pool(), + &config, + AudioTranscriptionRequest { + model: &model, + audio, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + optional_params, + timeout, + }, + ) .await } @@ -62,9 +68,10 @@ pub(crate) fn transcription( extra_headers, timeout: optional_timeout(timeout_seconds), }; + let config = crate::http::call_config(py, &PyDict::new(py), false)?; run_sync( py, - execute(audio, optional_params.unwrap_or_default(), options), + execute(config, audio, optional_params.unwrap_or_default(), options), audio_transcription_error_to_pyerr, ) } @@ -94,9 +101,10 @@ pub(crate) fn atranscription<'py>( extra_headers, timeout: optional_timeout(timeout_seconds), }; + let config = crate::http::call_config(py, &PyDict::new(py), true)?; run_async( py, - execute(audio, optional_params.unwrap_or_default(), options), + execute(config, audio, optional_params.unwrap_or_default(), options), audio_transcription_error_to_pyerr, ) } diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs index b96b12bfc43..6d7fad0d69c 100644 --- a/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs @@ -7,6 +7,7 @@ use litellm_core::chat_completions::{ types::ChatCompletionsRequest, }; use litellm_host_python::from_py_argument; +use litellm_http::HttpClientConfig; use litellm_types::utils::ChatCompletionsResponse; use pyo3::prelude::*; use serde_json::{Map, Value}; @@ -20,6 +21,7 @@ use crate::{ }; async fn execute( + config: HttpClientConfig, messages: Vec, optional_params: Map, options: RouteOptions, @@ -32,16 +34,20 @@ async fn execute( extra_headers, timeout, } = options; - run_chat_completions(ChatCompletionsRequest { - model: &model, - messages: Value::Array(messages), - optional_params, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - timeout, - }) + run_chat_completions( + crate::http::pool(), + &config, + ChatCompletionsRequest { + model: &model, + messages: Value::Array(messages), + optional_params, + api_key: api_key.as_deref(), + api_base: api_base.as_deref(), + custom_llm_provider: custom_llm_provider.as_deref(), + extra_headers, + timeout, + }, + ) .await } @@ -87,9 +93,15 @@ pub(crate) fn chat_completions( extra_headers, timeout: optional_timeout(timeout_seconds), }; + let config = crate::http::call_config(py, &PyDict::new(py), false)?; run_sync( py, - execute(messages, optional_params.unwrap_or_default(), options), + execute( + config, + messages, + optional_params.unwrap_or_default(), + options, + ), chat_completions_error_to_pyerr, ) } @@ -119,9 +131,15 @@ pub(crate) fn achat_completions<'py>( extra_headers, timeout: optional_timeout(timeout_seconds), }; + let config = crate::http::call_config(py, &PyDict::new(py), true)?; run_async( py, - execute(messages, optional_params.unwrap_or_default(), options), + execute( + config, + messages, + optional_params.unwrap_or_default(), + options, + ), chat_completions_error_to_pyerr, ) } diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs index 52cebb7c903..a59c9360c36 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs @@ -27,11 +27,14 @@ fn run_messages( asynchronous: bool, ) -> PyResult> { let secrets = crate::secrets::source(py)?; + let config = crate::http::call_config(py, &kwargs, asynchronous)?; + let machine = messages_machine(crate::http::pool(), &config, secrets) + .map_err(crate::http::client_error)?; run_legacy_call( py, SURFACE, PublicCall::capture(&request, &args, &kwargs)?, - crate::logger::LoggedMachine::new(messages_machine(secrets)), + crate::logger::LoggedMachine::new(machine), MessagesPythonHost::new(request.unbind()), crate::preflight::sdk_preflight, asynchronous, diff --git a/litellm-rust/crates/python-bridge/src/secrets/callback.rs b/litellm-rust/crates/python-bridge/src/secrets/callback.rs index 82bb4443f98..6ba60630b3a 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/callback.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/callback.rs @@ -210,7 +210,7 @@ handler.get_secret_from_manager = get_secret_from_manager KeyManagementSettings::default(), )), Arc::new(move |_: &str| fallback.map(str::to_owned)), - OidcResolver::default(), + OidcResolver::new(litellm_http::Client::plain_for_test()), ) .with_failure_policy(FailurePolicy::EnvironmentFallback); (resolver, locals, handler) diff --git a/litellm-rust/crates/python-bridge/src/secrets/mod.rs b/litellm-rust/crates/python-bridge/src/secrets/mod.rs index 439f9ddddd1..ed54c306397 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/mod.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/mod.rs @@ -27,7 +27,12 @@ const NATIVE: FieldSpec = FieldSpec::new("native", |field| field.schema_bo pub(crate) fn source(py: Python<'_>) -> PyResult> { if PythonSettings::SecretManager.read(py)?.read(&NATIVE)? { let context = litellm_host_python::PythonContext::capture(py)?; - return Ok(Arc::new(ResolvedSecrets::new(config::read(py)?, context))); + let client = crate::http::host_client(py, litellm_http::ClientVariant::Provider)?; + return Ok(Arc::new(ResolvedSecrets::new( + config::read(py)?, + context, + client, + ))); } Ok(Arc::new(PythonSecrets::new(py)?)) } diff --git a/litellm-rust/crates/python-bridge/src/secrets/resolved.rs b/litellm-rust/crates/python-bridge/src/secrets/resolved.rs index 40b187c99de..5a606ab1039 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/resolved.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/resolved.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use futures_util::future::BoxFuture; use litellm_core_utils::settings::ProcessEnvironment; use litellm_host_python::PythonContext; +use litellm_http::Client; use litellm_secrets::source::SecretSource; use litellm_secrets::{ Error, FailurePolicy, OidcResolver, SecretManagerState, SecretResolver, SecretValue, @@ -15,16 +16,20 @@ pub(crate) struct ResolvedSecrets { } impl ResolvedSecrets { - pub(crate) fn new(snapshot: SecretManagerSnapshot, context: PythonContext) -> Self { - Self::from_state(snapshot.into_state(context)) + pub(crate) fn new( + snapshot: SecretManagerSnapshot, + context: PythonContext, + client: Client, + ) -> Self { + Self::from_state(snapshot.into_state(context), client) } - fn from_state(state: Arc) -> Self { + fn from_state(state: Arc, client: Client) -> Self { Self { resolver: SecretResolver::new_python_compatible( state, Arc::new(ProcessEnvironment), - OidcResolver::default(), + OidcResolver::new(client), ) .with_failure_policy(FailurePolicy::EnvironmentFallback), } @@ -79,7 +84,7 @@ mod tests { } async fn resolve(state: Arc, name: &'static str) -> Option { - ResolvedSecrets::from_state(state) + ResolvedSecrets::from_state(state, litellm_http::Client::plain_for_test()) .resolve(&[name]) .await .unwrap() @@ -175,7 +180,10 @@ mod tests { .expect(1) .mount(&server) .await; - let source = ResolvedSecrets::from_state(state(&server, KeyManagementSettings::default())); + let source = ResolvedSecrets::from_state( + state(&server, KeyManagementSettings::default()), + litellm_http::Client::plain_for_test(), + ); let snapshot = source.resolve(&[declared]).await.unwrap(); assert_eq!(snapshot.get(undeclared), None); let result = source @@ -238,9 +246,12 @@ mod tests { #[tokio::test] async fn oidc_failures_are_not_converted_to_missing_secrets() { - let result = ResolvedSecrets::from_state(Arc::new(SecretManagerState::default())) - .resolve(&["oidc/"]) - .await; + let result = ResolvedSecrets::from_state( + Arc::new(SecretManagerState::default()), + litellm_http::Client::plain_for_test(), + ) + .resolve(&["oidc/"]) + .await; assert!(matches!(result, Err(litellm_secrets::Error::InvalidOidc))); } diff --git a/litellm-rust/crates/python-bridge/src/secrets/runtime.rs b/litellm-rust/crates/python-bridge/src/secrets/runtime.rs index 1a89130ee82..4d2e88115c8 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/runtime.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/runtime.rs @@ -10,6 +10,7 @@ use litellm_secrets_types::PythonSecretRead; use pyo3::{ exceptions::{PyAttributeError, PyRuntimeError, PyValueError}, prelude::*, + types::PyDict, }; #[derive(Clone, PartialEq)] @@ -44,10 +45,18 @@ impl NativeSecretManager { let system = configuration.system; let settings = configuration.settings.clone(); let enterprise_enabled = configuration.enterprise_enabled; + let http_config = crate::http::call_config(py, &PyDict::new(py), false)?; let backend = run_sync_value(py, async move { - load_native_manager(system, settings, environment, enterprise_enabled) - .await - .map_err(|error| PyValueError::new_err(error.to_string())) + load_native_manager( + crate::http::pool(), + &http_config, + system, + settings, + environment, + enterprise_enabled, + ) + .await + .map_err(|error| PyValueError::new_err(error.to_string())) })?; Ok(Self { backend, diff --git a/litellm-rust/crates/secrets-azure/Cargo.toml b/litellm-rust/crates/secrets-azure/Cargo.toml index 7e8a79f89ef..efdf681e2bc 100644 --- a/litellm-rust/crates/secrets-azure/Cargo.toml +++ b/litellm-rust/crates/secrets-azure/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-http.workspace = true tokio.workspace = true litellm-auth-azure.workspace = true litellm-auth-types.workspace = true @@ -18,6 +19,7 @@ veil.workspace = true percent-encoding = "2.3" [dev-dependencies] +litellm-http = { workspace = true, features = ["test-support"] } wiremock = "0.6.5" rstest.workspace = true serde_json.workspace = true diff --git a/litellm-rust/crates/secrets-azure/src/key_vault.rs b/litellm-rust/crates/secrets-azure/src/key_vault.rs index 13e59f8e4ac..095c451927c 100644 --- a/litellm-rust/crates/secrets-azure/src/key_vault.rs +++ b/litellm-rust/crates/secrets-azure/src/key_vault.rs @@ -19,7 +19,7 @@ const PATH_SEGMENT: &AsciiSet = &NON_ALPHANUMERIC #[derive(Clone)] pub struct AzureKeyVault { - client: reqwest::Client, + client: litellm_http::Client, vault: reqwest::Url, auth: Arc, inputs: Arc, @@ -33,7 +33,7 @@ struct SecretResponse { impl AzureKeyVault { pub fn with_client( - client: reqwest::Client, + client: litellm_http::Client, vault: reqwest::Url, environment: Arc, ) -> Result { @@ -57,7 +57,10 @@ impl AzureKeyVault { }) } - pub fn new(environment: Arc) -> Result { + pub fn new( + client: litellm_http::Client, + environment: Arc, + ) -> Result { let value = environment .get(AZURE_KEY_VAULT_URI) .ok_or(Error::MissingEnvironment(AZURE_KEY_VAULT_URI))?; @@ -65,7 +68,7 @@ impl AzureKeyVault { if vault.scheme() != "https" || vault.host_str().is_none() { return Err(Error::VaultUri); } - Self::with_client(reqwest::Client::new(), vault, environment) + Self::with_client(client, vault, environment) } pub fn scope(&self) -> &str { diff --git a/litellm-rust/crates/secrets-azure/tests/key_vault.rs b/litellm-rust/crates/secrets-azure/tests/key_vault.rs index a21149db345..fcbc46092e1 100644 --- a/litellm-rust/crates/secrets-azure/tests/key_vault.rs +++ b/litellm-rust/crates/secrets-azure/tests/key_vault.rs @@ -11,7 +11,7 @@ use wiremock::{ fn manager(server: &MockServer) -> AzureKeyVault { AzureKeyVault::with_client( - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), server.uri().parse().unwrap(), Arc::new(|name: &str| (name == "AZURE_AD_TOKEN").then(|| "fake".to_owned())), ) @@ -130,11 +130,14 @@ fn new_validates_vault_environment( #[case] uri: Option<&'static str>, #[case] missing_environment: bool, ) { - let result = AzureKeyVault::new(Arc::new(move |name: &str| { - (name == "AZURE_KEY_VAULT_URI") - .then(|| uri.map(str::to_owned)) - .flatten() - })); + let result = AzureKeyVault::new( + litellm_http::Client::plain_for_test(), + Arc::new(move |name: &str| { + (name == "AZURE_KEY_VAULT_URI") + .then(|| uri.map(str::to_owned)) + .flatten() + }), + ); if missing_environment { assert!(matches!( @@ -155,7 +158,7 @@ fn new_validates_vault_environment( #[case::local("http://localhost:8080", "https://localhost/.default")] fn derives_scope_from_vault_host(#[case] uri: &str, #[case] expected: &str) { let manager = AzureKeyVault::with_client( - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), uri.parse().unwrap(), Arc::new(|_: &str| None), ) @@ -184,7 +187,7 @@ async fn missing_credentials_do_not_request_vault() { fn manager_without_credentials(server: &MockServer) -> AzureKeyVault { AzureKeyVault::with_client( - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), server.uri().parse().unwrap(), Arc::new(|name: &str| { (name == "AZURE_CREDENTIAL").then(|| "ClientSecretCredential".to_owned()) diff --git a/litellm-rust/crates/secrets-azure/tests/live.rs b/litellm-rust/crates/secrets-azure/tests/live.rs index 18306382613..429cd3013f7 100644 --- a/litellm-rust/crates/secrets-azure/tests/live.rs +++ b/litellm-rust/crates/secrets-azure/tests/live.rs @@ -10,7 +10,7 @@ use rstest::rstest; #[ignore] async fn reads_a_real_secret() { let environment = Arc::new(ProcessEnvironment); - let manager = AzureKeyVault::new(environment).unwrap(); + let manager = AzureKeyVault::new(litellm_http::Client::plain_for_test(), environment).unwrap(); let name = std::env::var("AZURE_KEY_VAULT_LIVE_SECRET_NAME").unwrap(); let secret = manager.get_secret(&name).await.unwrap().unwrap(); assert!(matches!(&secret, Secret::String(_))); diff --git a/litellm-rust/crates/secrets-cyberark/Cargo.toml b/litellm-rust/crates/secrets-cyberark/Cargo.toml index 1c280171f4c..0a91c61ade9 100644 --- a/litellm-rust/crates/secrets-cyberark/Cargo.toml +++ b/litellm-rust/crates/secrets-cyberark/Cargo.toml @@ -8,6 +8,7 @@ repository.workspace = true [dependencies] litellm-secrets-types.workspace = true litellm-core-utils.workspace = true +litellm-http.workspace = true base64.workspace = true moka.workspace = true reqwest.workspace = true @@ -19,6 +20,7 @@ percent-encoding = "2.3" tokio = { workspace = true, features = ["sync"] } [dev-dependencies] +litellm-http = { workspace = true, features = ["test-support"] } rcgen = "0.14.10" rstest.workspace = true tempfile = "3.27.0" diff --git a/litellm-rust/crates/secrets-cyberark/src/error.rs b/litellm-rust/crates/secrets-cyberark/src/error.rs index 3dfeb95fe26..4f21647225b 100644 --- a/litellm-rust/crates/secrets-cyberark/src/error.rs +++ b/litellm-rust/crates/secrets-cyberark/src/error.rs @@ -18,6 +18,8 @@ pub enum Error { MissingCredentials, #[error("CyberArk client certificate could not be loaded")] ClientCertificate, + #[error("CyberArk Conjur HTTP client could not be built")] + Client(#[redact] Box), #[error("invalid refresh interval")] RefreshInterval, #[error("invalid CyberArk Conjur endpoint")] diff --git a/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs b/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs index 252a99c917f..37d547ff6c1 100644 --- a/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs +++ b/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs @@ -2,10 +2,13 @@ mod client; mod read; mod write; -use std::{fs, sync::Arc, time::Duration}; +use std::{sync::Arc, time::Duration}; use base64::{Engine, engine::general_purpose::STANDARD}; use litellm_core_utils::settings::Lookup; +use litellm_http::{ + Client, ClientIdentity, ClientVariant, HttpClientConfig, HttpClientPool, TlsSource, Verify, +}; use litellm_secrets_types::{ BaseSecretManager, CyberarkOperationContext, RotationError, SecretCache, SecretDeleter, SecretRotator, SecretValue, SecretWriteContext, SecretWriter, async_rotate_secret, @@ -37,7 +40,7 @@ const SECRET_NAME_SAFE: &AsciiSet = &NON_ALPHANUMERIC #[derive(Clone)] pub struct CyberArkSecretManager { - client: reqwest::Client, + client: Client, endpoint: reqwest::Url, account: String, username: String, diff --git a/litellm-rust/crates/secrets-cyberark/src/secret_manager/client.rs b/litellm-rust/crates/secrets-cyberark/src/secret_manager/client.rs index 1d99fe474d5..052d5570896 100644 --- a/litellm-rust/crates/secrets-cyberark/src/secret_manager/client.rs +++ b/litellm-rust/crates/secrets-cyberark/src/secret_manager/client.rs @@ -2,7 +2,7 @@ use super::*; impl CyberArkSecretManager { pub fn with_client( - client: reqwest::Client, + client: Client, endpoint: reqwest::Url, account: String, username: String, @@ -30,6 +30,8 @@ impl CyberArkSecretManager { } pub fn new( + pool: &HttpClientPool, + config: &HttpClientConfig, environment: Arc, enterprise_enabled: bool, ) -> Result { @@ -46,21 +48,34 @@ impl CyberArkSecretManager { .get(CYBERARK_SSL_VERIFY) .map(|value| !value.trim().eq_ignore_ascii_case("false")) .unwrap_or(true); - let mut builder = reqwest::Client::builder(); if !verify { litellm_tracing::warn!( "CyberArk SSL verification is disabled. This is insecure and should only be used for testing with self-signed certificates." ); - builder = builder.danger_accept_invalid_certs(true); } - if !cert.is_empty() && !key.is_empty() { - let certificate = fs::read(cert).map_err(|_| Error::ClientCertificate)?; - let private_key = fs::read(key).map_err(|_| Error::ClientCertificate)?; - let identity = reqwest::Identity::from_pem(&[certificate, private_key].concat()) - .map_err(|_| Error::ClientCertificate)?; - builder = builder.identity(identity); - } - let client = builder.build()?; + let config = HttpClientConfig { + verify: effective_verify(verify, &config.verify), + client_certificate: (!cert.is_empty() && !key.is_empty()).then(|| { + ClientIdentity::Split { + certificate: cert.into(), + key: key.into(), + } + }), + ..config.clone() + }; + let client = + pool.client(&config, ClientVariant::Provider) + .map_err(|error| match error { + litellm_http::Error::Read { + tls_source: TlsSource::ClientIdentity, + .. + } + | litellm_http::Error::InvalidPem { + tls_source: TlsSource::ClientIdentity, + .. + } => Error::ClientCertificate, + other => Error::Client(Box::new(other)), + })?; let endpoint = reqwest::Url::parse( &environment .get(CYBERARK_API_BASE) @@ -139,9 +154,35 @@ impl CyberArkSecretManager { } } +fn effective_verify(cyberark_verify: bool, host: &Verify) -> Verify { + match (cyberark_verify, host) { + (false, _) => Verify::Disabled, + (true, Verify::Disabled) => Verify::BuiltInRoots, + (true, host) => host.clone(), + } +} + fn normalize_endpoint(mut endpoint: reqwest::Url) -> reqwest::Url { if !endpoint.path().ends_with('/') { endpoint.set_path(&format!("{}/", endpoint.path())); } endpoint } + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use super::*; + + #[test] + fn cyberark_verification_does_not_follow_a_host_that_disabled_it() { + let bundle = Verify::CaBundle(PathBuf::from("/ca.pem")); + assert_eq!( + effective_verify(true, &Verify::Disabled), + Verify::BuiltInRoots + ); + assert_eq!(effective_verify(true, &bundle), bundle); + assert_eq!(effective_verify(false, &bundle), Verify::Disabled); + } +} diff --git a/litellm-rust/crates/secrets-cyberark/tests/secret_manager.rs b/litellm-rust/crates/secrets-cyberark/tests/secret_manager.rs index 2048e067b6e..783ba6ff67a 100644 --- a/litellm-rust/crates/secrets-cyberark/tests/secret_manager.rs +++ b/litellm-rust/crates/secrets-cyberark/tests/secret_manager.rs @@ -7,6 +7,8 @@ use std::{ }; use base64::{Engine, engine::general_purpose::STANDARD}; +use litellm_core_utils::settings::Lookup; +use litellm_http::{HttpClientPool, HttpSettings, Resolution, media::PublicDnsResolver}; use litellm_secrets_cyberark::{CyberArkSecretManager, DeleteOutcome, Error}; use litellm_secrets_types::{BaseSecretManager, CyberarkOperationContext, SecretValue}; use rstest::{fixture, rstest}; diff --git a/litellm-rust/crates/secrets-cyberark/tests/secret_manager/configuration.rs b/litellm-rust/crates/secrets-cyberark/tests/secret_manager/configuration.rs index fbd4317f446..1bea77e49ae 100644 --- a/litellm-rust/crates/secrets-cyberark/tests/secret_manager/configuration.rs +++ b/litellm-rust/crates/secrets-cyberark/tests/secret_manager/configuration.rs @@ -77,7 +77,7 @@ async fn authentication_encodes_login(#[case] username: &str, #[case] expected_p .mount(&server) .await; let manager = CyberArkSecretManager::with_client( - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), server.uri().parse().unwrap(), "acct".into(), username.into(), @@ -211,25 +211,25 @@ fn new_validates_credentials_before_license_and_configuration() { let empty: Arc = Arc::new(|_: &str| None); assert!(matches!( - CyberArkSecretManager::new(empty, true), + from_environment(empty, true), Err(Error::MissingCredentials) )); assert!(matches!( - CyberArkSecretManager::new( + from_environment( Arc::new(|name: &str| (name == "CYBERARK_API_KEY").then(|| "k3y".into())), false ), Err(Error::EnterpriseRequired) )); assert!(matches!( - CyberArkSecretManager::new( + from_environment( Arc::new(|name: &str| (name == "CYBERARK_CLIENT_CERT").then(|| "cert".into())), true ), Err(Error::MissingCredentials) )); assert!(matches!( - CyberArkSecretManager::new( + from_environment( Arc::new(|name: &str| match name { "CYBERARK_API_KEY" => Some("k3y".into()), "CYBERARK_REFRESH_INTERVAL" => Some("abc".into()), @@ -240,7 +240,7 @@ fn new_validates_credentials_before_license_and_configuration() { Err(Error::RefreshInterval) )); assert!(matches!( - CyberArkSecretManager::new( + from_environment( Arc::new(|name: &str| match name { "CYBERARK_API_KEY" => Some("k3y".into()), "CYBERARK_API_BASE" => Some("not a url".into()), @@ -254,7 +254,7 @@ fn new_validates_credentials_before_license_and_configuration() { #[rstest] fn certificate_only_credentials_are_validated_as_a_client_identity() { - let result = CyberArkSecretManager::new( + let result = from_environment( Arc::new(|name: &str| match name { "CYBERARK_CLIENT_CERT" => Some("/missing/cert".into()), "CYBERARK_CLIENT_KEY" => Some("/missing/key".into()), @@ -295,7 +295,7 @@ async fn configured_client_identity_preserves_auth_request_and_read_result( let endpoint = server.uri(); let certificate = client_identity_directory.path().join("client.crt"); let key = client_identity_directory.path().join("client.key"); - let manager = CyberArkSecretManager::new( + let manager = from_environment( Arc::new(move |name: &str| match name { "CYBERARK_API_BASE" => Some(endpoint.clone()), "CYBERARK_API_KEY" => Some(api_key.into()), @@ -337,7 +337,7 @@ fn invalid_client_identity_is_not_ignored( let certificate = client_identity_directory.path().join("client.crt"); let key = client_identity_directory.path().join("client.key"); - let result = CyberArkSecretManager::new( + let result = from_environment( Arc::new(move |name: &str| match name { "CYBERARK_API_KEY" => Some(api_key.into()), "CYBERARK_CLIENT_CERT" => Some(certificate.to_str().unwrap().into()), @@ -354,7 +354,7 @@ fn invalid_client_identity_is_not_ignored( #[case::certificate_only("")] #[case::certificate_and_api_key("k3y")] fn client_identity_does_not_bypass_the_enterprise_requirement(#[case] api_key: &'static str) { - let result = CyberArkSecretManager::new( + let result = from_environment( Arc::new(move |name: &str| match name { "CYBERARK_API_KEY" => Some(api_key.into()), "CYBERARK_CLIENT_CERT" => Some("/missing/cert".into()), @@ -381,7 +381,7 @@ async fn new_reads_environment_defaults_end_to_end() { .mount(&server) .await; let endpoint = server.uri(); - let manager = CyberArkSecretManager::new( + let manager = from_environment( Arc::new(move |name: &str| match name { "CYBERARK_API_BASE" => Some(endpoint.clone()), "CYBERARK_API_KEY" => Some("k3y".into()), @@ -404,7 +404,7 @@ async fn new_reads_environment_defaults_end_to_end() { #[rstest] fn new_reports_missing_client_certificate_files() { assert!(matches!( - CyberArkSecretManager::new( + from_environment( Arc::new(|name: &str| match name { "CYBERARK_API_KEY" => Some("k3y".into()), "CYBERARK_CLIENT_CERT" => Some("/missing/cert".into()), @@ -433,7 +433,7 @@ async fn trailing_slash_endpoint_preserves_base_path() { .await; let endpoint = format!("{}/prefix/", server.uri()).parse().unwrap(); let manager = CyberArkSecretManager::with_client( - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), endpoint, "acct".into(), "admin".into(), diff --git a/litellm-rust/crates/secrets-cyberark/tests/secret_manager/support.rs b/litellm-rust/crates/secrets-cyberark/tests/secret_manager/support.rs index f5ba7a63273..6fa15ecba9b 100644 --- a/litellm-rust/crates/secrets-cyberark/tests/secret_manager/support.rs +++ b/litellm-rust/crates/secrets-cyberark/tests/secret_manager/support.rs @@ -48,9 +48,21 @@ pub(super) fn client_identity_directory() -> tempfile::TempDir { directory } +pub(super) fn from_environment( + environment: Arc, + enterprise_enabled: bool, +) -> Result { + CyberArkSecretManager::new( + &HttpClientPool::new(Arc::new(PublicDnsResolver)), + &Resolution::from(&HttpSettings::default()).config, + environment, + enterprise_enabled, + ) +} + pub(super) fn manager(server: &MockServer, ttl: Duration) -> CyberArkSecretManager { CyberArkSecretManager::with_client( - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), server.uri().parse().unwrap(), "acct".into(), "admin".into(), diff --git a/litellm-rust/crates/secrets-cyberark/tests/secret_manager/writes.rs b/litellm-rust/crates/secrets-cyberark/tests/secret_manager/writes.rs index 331a26c6119..e9a027091fa 100644 --- a/litellm-rust/crates/secrets-cyberark/tests/secret_manager/writes.rs +++ b/litellm-rust/crates/secrets-cyberark/tests/secret_manager/writes.rs @@ -166,7 +166,7 @@ async fn writes_match_python_parity_fixture(parity_fixture: ParityFixture) { .mount(&server) .await; let manager = CyberArkSecretManager::with_client( - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), server.uri().parse().unwrap(), parity_fixture.account, parity_fixture.username, @@ -230,7 +230,7 @@ async fn live_conjur_round_trip() { .as_nanos() ); let manager = CyberArkSecretManager::with_client( - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), endpoint.clone(), account.clone(), username.clone(), @@ -245,7 +245,7 @@ async fn live_conjur_round_trip() { .await .unwrap(); let verifier = CyberArkSecretManager::with_client( - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), endpoint.clone(), account.clone(), username.clone(), diff --git a/litellm-rust/crates/secrets-google/Cargo.toml b/litellm-rust/crates/secrets-google/Cargo.toml index 805eb80740d..208b5ddd03f 100644 --- a/litellm-rust/crates/secrets-google/Cargo.toml +++ b/litellm-rust/crates/secrets-google/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-http.workspace = true moka.workspace = true tokio.workspace = true litellm-auth-gcp = { workspace = true, features = ["google-sdk"] } @@ -24,6 +25,7 @@ serde.workspace = true reqwest.workspace = true [dev-dependencies] +litellm-http = { workspace = true, features = ["test-support"] } google-cloud-auth.workspace = true rstest.workspace = true wiremock = "0.6.5" diff --git a/litellm-rust/crates/secrets-google/src/secret_manager.rs b/litellm-rust/crates/secrets-google/src/secret_manager.rs index b8787999e12..08ba466b799 100644 --- a/litellm-rust/crates/secrets-google/src/secret_manager.rs +++ b/litellm-rust/crates/secrets-google/src/secret_manager.rs @@ -23,7 +23,7 @@ const CACHE_CAPACITY: u64 = 200; #[derive(Clone)] pub struct GoogleSecretManager { - client: reqwest::Client, + client: litellm_http::Client, credentials: Arc, endpoint: reqwest::Url, project: String, @@ -46,7 +46,7 @@ struct Payload { impl GoogleSecretManager { pub fn with_client( - client: reqwest::Client, + client: litellm_http::Client, endpoint: reqwest::Url, project: String, environment: Arc, @@ -79,6 +79,7 @@ impl GoogleSecretManager { } pub fn new( + client: litellm_http::Client, environment: Arc, enterprise_enabled: bool, ) -> Result { @@ -104,7 +105,7 @@ impl GoogleSecretManager { .get(GOOGLE_SECRET_MANAGER_ALWAYS_READ_SECRET_MANAGER) .is_some_and(|v| v.eq_ignore_ascii_case("true")); Self::with_client( - reqwest::Client::new(), + client, reqwest::Url::parse("https://secretmanager.googleapis.com").expect("static URL"), project, environment, diff --git a/litellm-rust/crates/secrets-google/tests/secret_manager.rs b/litellm-rust/crates/secrets-google/tests/secret_manager.rs index 0d7efc4b1b3..e9bee7633f5 100644 --- a/litellm-rust/crates/secrets-google/tests/secret_manager.rs +++ b/litellm-rust/crates/secrets-google/tests/secret_manager.rs @@ -11,7 +11,7 @@ use wiremock::{ fn manager(server: &MockServer, always_read: bool, ttl: Duration) -> GoogleSecretManager { GoogleSecretManager::with_client( - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), server.uri().parse().unwrap(), "project".into(), Arc::new(|name: &str| (name == "VERTEX_AI_API_KEY").then(|| "token".into())), @@ -214,11 +214,19 @@ async fn always_read_and_expired_cache_fetch_again( #[rstest] fn google_manager_requires_host_license_and_project_configuration() { assert!(matches!( - GoogleSecretManager::new(Arc::new(|_: &str| None), false), + GoogleSecretManager::new( + litellm_http::Client::plain_for_test(), + Arc::new(|_: &str| None), + false + ), Err(Error::EnterpriseRequired) )); assert!(matches!( - GoogleSecretManager::new(Arc::new(|_: &str| None), true), + GoogleSecretManager::new( + litellm_http::Client::plain_for_test(), + Arc::new(|_: &str| None), + true + ), Err(Error::MissingEnvironment( "GOOGLE_SECRET_MANAGER_PROJECT_ID" )) @@ -236,7 +244,7 @@ fn google_manager_rejects_invalid_refresh_intervals(#[case] variable: &'static s }); assert!(matches!( - GoogleSecretManager::new(environment, true), + GoogleSecretManager::new(litellm_http::Client::plain_for_test(), environment, true), Err(Error::RefreshInterval) )); } diff --git a/litellm-rust/crates/secrets/Cargo.toml b/litellm-rust/crates/secrets/Cargo.toml index 17acc01682b..fe61d6cb5b4 100644 --- a/litellm-rust/crates/secrets/Cargo.toml +++ b/litellm-rust/crates/secrets/Cargo.toml @@ -23,6 +23,7 @@ litellm-secrets-hashicorp = { workspace = true, optional = true } litellm-secrets-azure = { workspace = true, optional = true } litellm-secrets-cyberark = { workspace = true, optional = true } litellm-core-utils.workspace = true +litellm-http.workspace = true base64.workspace = true serde.workspace = true strum.workspace = true @@ -33,6 +34,7 @@ moka.workspace = true tokio = { workspace = true, features = ["fs"] } [dev-dependencies] +litellm-http = { workspace = true, features = ["test-support"] } rstest.workspace = true wiremock = "0.6.5" tempfile = "3" diff --git a/litellm-rust/crates/secrets/src/error.rs b/litellm-rust/crates/secrets/src/error.rs index 07f2f205bec..7fc756c5c7d 100644 --- a/litellm-rust/crates/secrets/src/error.rs +++ b/litellm-rust/crates/secrets/src/error.rs @@ -10,6 +10,8 @@ pub enum Error { InvalidCiphertext, #[error("decrypted value is not UTF-8")] Utf8, + #[error(transparent)] + Client(#[from] litellm_http::Error), #[error("unsupported OIDC provider or missing build feature")] UnsupportedOidc, #[error("OIDC reference requires a provider and audience")] diff --git a/litellm-rust/crates/secrets/src/native.rs b/litellm-rust/crates/secrets/src/native.rs index 80f0e46245c..f1dc7ccb732 100644 --- a/litellm-rust/crates/secrets/src/native.rs +++ b/litellm-rust/crates/secrets/src/native.rs @@ -1,10 +1,13 @@ use std::sync::Arc; use litellm_core_utils::settings::Lookup; +use litellm_http::{HttpClientConfig, HttpClientPool}; use crate::{Error, KeyManagementSettings, KeyManagementSystem, SecretManager}; pub async fn load_native_manager( + pool: &HttpClientPool, + config: &HttpClientConfig, system: KeyManagementSystem, settings: KeyManagementSettings, environment: Arc, @@ -29,14 +32,19 @@ pub async fn load_native_manager( } #[cfg(feature = "azure")] (KeyManagementSystem::AzureKeyVault, _, environment, _) => Ok( - SecretManager::AzureKeyVault(crate::azure::AzureKeyVault::new(environment)?), + SecretManager::AzureKeyVault(crate::azure::AzureKeyVault::new( + pool.client(config, litellm_http::ClientVariant::Provider)?, + environment, + )?), ), #[cfg(feature = "google")] - (KeyManagementSystem::GoogleSecretManager, _, environment, enterprise_enabled) => { - Ok(SecretManager::GoogleSecretManager( - crate::google::GoogleSecretManager::new(environment, enterprise_enabled)?, - )) - } + (KeyManagementSystem::GoogleSecretManager, _, environment, enterprise_enabled) => Ok( + SecretManager::GoogleSecretManager(crate::google::GoogleSecretManager::new( + pool.client(config, litellm_http::ClientVariant::Provider)?, + environment, + enterprise_enabled, + )?), + ), #[cfg(feature = "google")] (KeyManagementSystem::GoogleKms, _, environment, _) => { crate::google::load_google_kms(Some(true), environment) @@ -51,11 +59,14 @@ pub async fn load_native_manager( )) } #[cfg(feature = "cyberark")] - (KeyManagementSystem::Cyberark, _, environment, enterprise_enabled) => { - Ok(SecretManager::Cyberark( - crate::cyberark::CyberArkSecretManager::new(environment, enterprise_enabled)?, - )) - } + (KeyManagementSystem::Cyberark, _, environment, enterprise_enabled) => Ok( + SecretManager::Cyberark(crate::cyberark::CyberArkSecretManager::new( + pool, + config, + environment, + enterprise_enabled, + )?), + ), _ => Err(Error::NativeBackendUnavailable), } } diff --git a/litellm-rust/crates/secrets/src/oidc.rs b/litellm-rust/crates/secrets/src/oidc.rs index f3c1e38ce7b..b6e8dbc123b 100644 --- a/litellm-rust/crates/secrets/src/oidc.rs +++ b/litellm-rust/crates/secrets/src/oidc.rs @@ -5,6 +5,7 @@ use std::{ use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; use litellm_core_utils::settings::Lookup; +use litellm_http::Client; use moka::future::Cache; use serde::Deserialize; @@ -82,7 +83,7 @@ impl NumericDate { } pub struct OidcResolver { - client: reqwest::Client, + client: Client, google_identity_endpoint: reqwest::Url, cache: Cache, clock: fn() -> SystemTime, @@ -90,25 +91,17 @@ pub struct OidcResolver { azure_token_provider: std::sync::Arc, } -impl Default for OidcResolver { - fn default() -> Self { - let client = reqwest::Client::builder() - .timeout(Duration::from_secs(600)) - .connect_timeout(Duration::from_secs(5)) - .build() - .expect("HTTP client configuration"); - Self::new( - client, - reqwest::Url::parse("http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity").expect("static URL"), - ) - } -} +const GOOGLE_IDENTITY_ENDPOINT: &str = + "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity"; + +const REQUEST_TIMEOUT: Duration = Duration::from_secs(600); impl OidcResolver { - pub fn new(client: reqwest::Client, google_identity_endpoint: reqwest::Url) -> Self { + pub fn new(client: Client) -> Self { Self { client, - google_identity_endpoint, + google_identity_endpoint: reqwest::Url::parse(GOOGLE_IDENTITY_ENDPOINT) + .expect("static URL"), cache: Cache::builder() .max_capacity(200) .time_to_live(GOOGLE_TOKEN_MAX_TTL) @@ -121,6 +114,13 @@ impl OidcResolver { } } + pub fn with_google_identity_endpoint(self, google_identity_endpoint: reqwest::Url) -> Self { + Self { + google_identity_endpoint, + ..self + } + } + #[cfg(feature = "azure")] pub fn with_azure_token_provider( self, @@ -180,6 +180,7 @@ impl OidcResolver { let response = self .client .get(url) + .timeout(REQUEST_TIMEOUT) .query(&[("audience", audience)]) .bearer_auth(authorization) .header("Accept", "application/json; api-version=2.0") @@ -214,6 +215,7 @@ impl OidcResolver { let response = self .client .get(self.google_identity_endpoint.clone()) + .timeout(REQUEST_TIMEOUT) .query(&[("audience", audience)]) .header("Metadata-Flavor", "Google") .send() diff --git a/litellm-rust/crates/secrets/src/resolver.rs b/litellm-rust/crates/secrets/src/resolver.rs index 69445e5b410..830e7c22cd5 100644 --- a/litellm-rust/crates/secrets/src/resolver.rs +++ b/litellm-rust/crates/secrets/src/resolver.rs @@ -1,10 +1,7 @@ use std::sync::Arc; use crate::compatibility::python_manager_string; -use litellm_core_utils::{ - serde_compat::parse_str_bool, - settings::{Lookup, ProcessEnvironment}, -}; +use litellm_core_utils::{serde_compat::parse_str_bool, settings::Lookup}; use crate::state::{LookupTarget, normalize_secret_name}; use crate::{Error, OidcResolver, Secret, SecretManagerState, SecretValue}; @@ -24,16 +21,6 @@ pub struct SecretResolver { python_compatible: bool, } -impl Default for SecretResolver { - fn default() -> Self { - Self::new( - Arc::new(SecretManagerState::default()), - Arc::new(ProcessEnvironment), - OidcResolver::default(), - ) - } -} - impl SecretResolver { pub fn new( state: Arc, diff --git a/litellm-rust/crates/secrets/src/source.rs b/litellm-rust/crates/secrets/src/source.rs index a1615bad055..3c86fd25e9a 100644 --- a/litellm-rust/crates/secrets/src/source.rs +++ b/litellm-rust/crates/secrets/src/source.rs @@ -37,15 +37,14 @@ impl SecretSource for SecretResolver { } } -#[derive(Default)] pub struct EnvironmentSecrets(SecretResolver); impl EnvironmentSecrets { - pub fn python_compatible() -> Self { + pub fn python_compatible(client: litellm_http::Client) -> Self { Self(SecretResolver::new_python_compatible( Arc::new(crate::SecretManagerState::default()), Arc::new(litellm_core_utils::settings::ProcessEnvironment), - crate::OidcResolver::default(), + crate::OidcResolver::new(client), )) } } diff --git a/litellm-rust/crates/secrets/tests/aws.rs b/litellm-rust/crates/secrets/tests/aws.rs index 174d0881339..910b1855b77 100644 --- a/litellm-rust/crates/secrets/tests/aws.rs +++ b/litellm-rust/crates/secrets/tests/aws.rs @@ -53,7 +53,7 @@ async fn read_results_follow_the_selected_failure_policy( }, )), Arc::new(move |_: &str| environment.map(str::to_owned)), - OidcResolver::default(), + OidcResolver::new(litellm_http::Client::plain_for_test()), ) .with_failure_policy(policy); let result = resolver @@ -97,7 +97,7 @@ async fn primary_secret_values_other_than_strings_resolve_to_none( let resolver = SecretResolver::new_python_compatible( Arc::new(state(&server, settings)), Arc::new(|_: &str| Some("fallback".into())), - OidcResolver::default(), + OidcResolver::new(litellm_http::Client::plain_for_test()), ); let text = value.as_str(); assert_eq!( @@ -157,7 +157,7 @@ async fn gating_prediction_matches_actual_lookup( let resolver = SecretResolver::new_python_compatible( Arc::new(state), Arc::new(|_: &str| Some("environment".into())), - OidcResolver::default(), + OidcResolver::new(litellm_http::Client::plain_for_test()), ); assert_eq!( resolver diff --git a/litellm-rust/crates/secrets/tests/azure.rs b/litellm-rust/crates/secrets/tests/azure.rs index b844b198cd4..60f165add54 100644 --- a/litellm-rust/crates/secrets/tests/azure.rs +++ b/litellm-rust/crates/secrets/tests/azure.rs @@ -22,7 +22,7 @@ async fn azure_handler_reads_missing_and_failed_secrets() { .await; let manager = SecretManager::AzureKeyVault( AzureKeyVault::with_client( - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), server.uri().parse().unwrap(), std::sync::Arc::new(|name: &str| (name == "AZURE_AD_TOKEN").then(|| "fake".to_owned())), ) @@ -81,7 +81,7 @@ async fn successful_azure_responses_do_not_fall_back_when_the_value_is_empty_or_ .mount(&server) .await; let manager = AzureKeyVault::with_client( - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), server.uri().parse().unwrap(), Arc::new(|name: &str| (name == "AZURE_AD_TOKEN").then(|| "token".into())), ) @@ -92,7 +92,7 @@ async fn successful_azure_responses_do_not_fall_back_when_the_value_is_empty_or_ Default::default(), )), Arc::new(|_: &str| Some("environment".into())), - OidcResolver::default(), + OidcResolver::new(litellm_http::Client::plain_for_test()), ); assert_eq!( resolver diff --git a/litellm-rust/crates/secrets/tests/common_read_contract.rs b/litellm-rust/crates/secrets/tests/common_read_contract.rs index 698e1ad8f63..2e8fbf46008 100644 --- a/litellm-rust/crates/secrets/tests/common_read_contract.rs +++ b/litellm-rust/crates/secrets/tests/common_read_contract.rs @@ -59,7 +59,7 @@ fn manager(provider: Provider, server: &MockServer) -> SecretManager { ), Provider::Azure => SecretManager::AzureKeyVault( AzureKeyVault::with_client( - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), server.uri().parse().unwrap(), environment, ) @@ -67,7 +67,7 @@ fn manager(provider: Provider, server: &MockServer) -> SecretManager { ), Provider::Google => SecretManager::GoogleSecretManager( GoogleSecretManager::with_client( - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), server.uri().parse().unwrap(), "project".into(), environment, @@ -84,7 +84,7 @@ fn manager(provider: Provider, server: &MockServer) -> SecretManager { .unwrap(), ), Provider::Cyberark => SecretManager::Cyberark(CyberArkSecretManager::with_client( - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), server.uri().parse().unwrap(), "acct".into(), "admin".into(), @@ -240,7 +240,7 @@ async fn python_read_failures_preserve_provider_fallback_rules( KeyManagementSettings::default(), )), Arc::new(move |_: &str| environment_value.map(str::to_owned)), - OidcResolver::default(), + OidcResolver::new(litellm_http::Client::plain_for_test()), ); let expected = if matches!(provider, Provider::Aws) { None diff --git a/litellm-rust/crates/secrets/tests/cyberark.rs b/litellm-rust/crates/secrets/tests/cyberark.rs index 706c35752d7..b94bd9dc0ad 100644 --- a/litellm-rust/crates/secrets/tests/cyberark.rs +++ b/litellm-rust/crates/secrets/tests/cyberark.rs @@ -24,7 +24,7 @@ async fn cyberark_handler_reads_values_and_surfaces_errors() { .mount(&server) .await; let manager = SecretManager::Cyberark(CyberArkSecretManager::with_client( - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), server.uri().parse().unwrap(), "acct".into(), "admin".into(), diff --git a/litellm-rust/crates/secrets/tests/google.rs b/litellm-rust/crates/secrets/tests/google.rs index 67954fedb78..0b45a90c10b 100644 --- a/litellm-rust/crates/secrets/tests/google.rs +++ b/litellm-rust/crates/secrets/tests/google.rs @@ -25,7 +25,7 @@ async fn google_resolver_distinguishes_absence_from_failure(#[case] status: u16) _ => None, }); let manager = GoogleSecretManager::with_client( - reqwest::Client::new(), + litellm_http::Client::plain_for_test(), server.uri().parse().unwrap(), "project".into(), environment.clone(), @@ -40,7 +40,7 @@ async fn google_resolver_distinguishes_absence_from_failure(#[case] status: u16) let resolver = SecretResolver::new_python_compatible( Arc::new(state), environment, - OidcResolver::default(), + OidcResolver::new(litellm_http::Client::plain_for_test()), ) .with_failure_policy(FailurePolicy::Propagate); let result = resolver.get_secret_str("KEY", None).await; diff --git a/litellm-rust/crates/secrets/tests/hashicorp.rs b/litellm-rust/crates/secrets/tests/hashicorp.rs index bc35b88018e..e10e903c25a 100644 --- a/litellm-rust/crates/secrets/tests/hashicorp.rs +++ b/litellm-rust/crates/secrets/tests/hashicorp.rs @@ -56,7 +56,7 @@ async fn hashicorp_handler_resolves_found_missing_and_failed_values() { }, )), Arc::new(|_: &str| None), - litellm_secrets::OidcResolver::default(), + litellm_secrets::OidcResolver::new(litellm_http::Client::plain_for_test()), ); assert_eq!( found_resolver @@ -131,7 +131,7 @@ async fn hashicorp_handler_resolves_found_missing_and_failed_values() { let failed_resolver = SecretResolver::new_python_compatible( Arc::new(failed_state), Arc::new(|_: &str| None), - litellm_secrets::OidcResolver::default(), + litellm_secrets::OidcResolver::new(litellm_http::Client::plain_for_test()), ) .with_failure_policy(FailurePolicy::Propagate); assert!(matches!( diff --git a/litellm-rust/crates/secrets/tests/oidc.rs b/litellm-rust/crates/secrets/tests/oidc.rs index afc49e8231d..6f72bd9d645 100644 --- a/litellm-rust/crates/secrets/tests/oidc.rs +++ b/litellm-rust/crates/secrets/tests/oidc.rs @@ -30,7 +30,7 @@ async fn environment_sources_resolve_expected_value( ("CIRCLE_OIDC_TOKEN_V2", "circle-v2"), ]); assert_eq!( - OidcResolver::default() + OidcResolver::new(litellm_http::Client::plain_for_test()) .resolve(reference, env.as_ref()) .await .unwrap() @@ -43,7 +43,7 @@ async fn environment_sources_resolve_expected_value( #[tokio::test] async fn environment_sources_bypass_boolean_conversion_and_defaults() { let env = environment(&[("TOKEN", "true")]); - let oidc = OidcResolver::default(); + let oidc = OidcResolver::new(litellm_http::Client::plain_for_test()); let resolver = SecretResolver::new(Arc::new(SecretManagerState::default()), env, oidc); assert_eq!( resolver @@ -94,7 +94,7 @@ async fn github_requests_are_authenticated_cached_and_revalidate_environment() { ), ("ACTIONS_ID_TOKEN_REQUEST_TOKEN", "request-token"), ]); - let oidc = OidcResolver::default(); + let oidc = OidcResolver::new(litellm_http::Client::plain_for_test()); for _ in 0..2 { assert_eq!( oidc.resolve("oidc/github/https://service/oidc/path", env.as_ref()) @@ -131,7 +131,7 @@ async fn file_allowlist_resolves_symlinks_while_environment_paths_remain_explici ("PATH_TOKEN", private.to_str().unwrap()), ("AZURE_FEDERATED_TOKEN_FILE", token.to_str().unwrap()), ]); - let oidc = OidcResolver::default(); + let oidc = OidcResolver::new(litellm_http::Client::plain_for_test()); assert_eq!( oidc.resolve(&format!("oidc/file/{}", token.display()), env.as_ref()) .await @@ -213,8 +213,9 @@ async fn google_expiry_caps_cache_and_preserves_audience( .expect(calls) .mount(&server) .await; - let oidc = - OidcResolver::new(reqwest::Client::new(), server.uri().parse().unwrap()).with_clock(now); + let oidc = OidcResolver::new(litellm_http::Client::plain_for_test()) + .with_google_identity_endpoint(server.uri().parse().unwrap()) + .with_clock(now); for _ in 0..2 { assert_eq!( oidc.resolve( @@ -234,7 +235,7 @@ async fn google_expiry_caps_cache_and_preserves_audience( #[tokio::test] async fn google_oidc_requires_its_build_feature() { assert!(matches!( - OidcResolver::default() + OidcResolver::new(litellm_http::Client::plain_for_test()) .resolve("oidc/google/audience", environment(&[]).as_ref()) .await, Err(Error::UnsupportedOidc) @@ -245,7 +246,7 @@ async fn google_oidc_requires_its_build_feature() { #[tokio::test] async fn azure_oidc_without_a_token_file_requires_its_build_feature() { assert!(matches!( - OidcResolver::default() + OidcResolver::new(litellm_http::Client::plain_for_test()) .resolve("oidc/azure/scope", environment(&[]).as_ref()) .await, Err(Error::UnsupportedOidc) @@ -261,7 +262,7 @@ async fn invalid_references_fail_before_environment_lookup( #[case] reference: &str, #[case] unsupported: bool, ) { - let error = OidcResolver::default() + let error = OidcResolver::new(litellm_http::Client::plain_for_test()) .resolve(reference, &|_: &str| { panic!("invalid reference reached environment lookup") }) @@ -283,7 +284,8 @@ async fn unreadable_expiry_keeps_python_cache_fallback(#[case] token: &str) { .expect(1) .mount(&server) .await; - let resolver = OidcResolver::new(reqwest::Client::new(), server.uri().parse().unwrap()); + let resolver = OidcResolver::new(litellm_http::Client::plain_for_test()) + .with_google_identity_endpoint(server.uri().parse().unwrap()); for _ in 0..2 { assert_eq!( resolver @@ -334,7 +336,8 @@ async fn azure_oidc_acquires_the_requested_scope_and_preserves_failures(#[case] }) } } - let oidc = OidcResolver::default().with_azure_token_provider(Arc::new(Provider(failed))); + let oidc = OidcResolver::new(litellm_http::Client::plain_for_test()) + .with_azure_token_provider(Arc::new(Provider(failed))); let resolver = SecretResolver::new_python_compatible( Arc::new(SecretManagerState::default()), environment(&[("AZURE_CLIENT_ID", "client-id")]), @@ -361,7 +364,7 @@ async fn azure_oidc_acquires_the_requested_scope_and_preserves_failures(#[case] #[tokio::test] async fn missing_oidc_environment_is_an_error(#[case] reference: &str) { assert!(matches!( - OidcResolver::default() + OidcResolver::new(litellm_http::Client::plain_for_test()) .resolve(reference, environment(&[]).as_ref()) .await, Err(Error::MissingEnvironment) @@ -380,7 +383,8 @@ async fn google_oidc_failures_are_not_cached_or_hidden_by_defaults() { let resolver = SecretResolver::new_python_compatible( Arc::new(SecretManagerState::default()), environment(&[]), - OidcResolver::new(reqwest::Client::new(), server.uri().parse().unwrap()), + OidcResolver::new(litellm_http::Client::plain_for_test()) + .with_google_identity_endpoint(server.uri().parse().unwrap()), ); for _ in 0..2 { assert!(matches!( @@ -420,7 +424,8 @@ async fn google_tokens_expire_at_the_python_cache_deadline( .expect(2) .mount(&server) .await; - let resolver = OidcResolver::new(reqwest::Client::new(), server.uri().parse().unwrap()) + let resolver = OidcResolver::new(litellm_http::Client::plain_for_test()) + .with_google_identity_endpoint(server.uri().parse().unwrap()) .with_clock(|| UNIX_EPOCH + Duration::from_secs(1000)); assert_eq!( resolver @@ -472,7 +477,8 @@ async fn google_cache_uses_payload_expiry_without_requiring_a_jwt_header() { .expect(2) .mount(&server) .await; - let resolver = OidcResolver::new(reqwest::Client::new(), server.uri().parse().unwrap()) + let resolver = OidcResolver::new(litellm_http::Client::plain_for_test()) + .with_google_identity_endpoint(server.uri().parse().unwrap()) .with_clock(|| UNIX_EPOCH + Duration::from_secs(1000)); for _ in 0..2 { assert_eq!( diff --git a/litellm-rust/crates/secrets/tests/resolution.rs b/litellm-rust/crates/secrets/tests/resolution.rs index bed762adc59..37557c36fb5 100644 --- a/litellm-rust/crates/secrets/tests/resolution.rs +++ b/litellm-rust/crates/secrets/tests/resolution.rs @@ -12,7 +12,7 @@ fn resolver(value: Option<&str>) -> SecretResolver { SecretResolver::new_python_compatible( Arc::new(SecretManagerState::default()), Arc::new(move |_: &str| value.clone()), - OidcResolver::default(), + OidcResolver::new(litellm_http::Client::plain_for_test()), ) } @@ -35,7 +35,7 @@ async fn native_reads_preserve_strings_and_report_conversion_errors(#[case] mana let resolver = SecretResolver::new( Arc::new(state), Arc::new(move |_: &str| Some(raw.to_owned())), - OidcResolver::default(), + OidcResolver::new(litellm_http::Client::plain_for_test()), ); assert_eq!( resolver @@ -75,7 +75,7 @@ async fn native_defaults_apply_to_absence_but_never_hide_provider_failures() { KeyManagementSettings::default(), )), Arc::new(|_: &str| None), - OidcResolver::default(), + OidcResolver::new(litellm_http::Client::plain_for_test()), ); let result = resolver .get_secret_str("key", Some(SecretValue::new("default"))) @@ -189,7 +189,7 @@ fn managed(reply: Result, ()>, environment: Option<&'static str>) KeyManagementSettings::default(), )), Arc::new(move |_: &str| environment.map(str::to_owned)), - OidcResolver::default(), + OidcResolver::new(litellm_http::Client::plain_for_test()), ) .with_failure_policy(FailurePolicy::EnvironmentFallback) } @@ -281,7 +281,7 @@ async fn prefix_is_removed_once_and_resolved_from_environment() { let resolver = SecretResolver::new_python_compatible( Arc::new(state), Arc::new(|name: &str| (name == "os.environ/KEY").then(|| "value".into())), - OidcResolver::default(), + OidcResolver::new(litellm_http::Client::plain_for_test()), ); assert_eq!( resolver @@ -340,7 +340,7 @@ async fn excluded_hosted_keys_keep_the_python_manager_conversion_path( }, )), Arc::new(move |_: &str| Some(raw.to_owned())), - OidcResolver::default(), + OidcResolver::new(litellm_http::Client::plain_for_test()), ); assert_eq!( resolver.get_secret("KEY", None).await.unwrap(), @@ -373,7 +373,7 @@ async fn azure_callback_absence_preserves_none_but_errors_fall_back( KeyManagementSettings::default(), )), Arc::new(|_: &str| Some("environment".into())), - OidcResolver::default(), + OidcResolver::new(litellm_http::Client::plain_for_test()), ); assert_eq!( resolver diff --git a/litellm-rust/crates/secrets/tests/source.rs b/litellm-rust/crates/secrets/tests/source.rs index b4782c6af86..17210940a67 100644 --- a/litellm-rust/crates/secrets/tests/source.rs +++ b/litellm-rust/crates/secrets/tests/source.rs @@ -15,7 +15,7 @@ mod tests { #[case] expected: Option<&str>, ) { unsafe { std::env::set_var(name, value) }; - let secret = EnvironmentSecrets::python_compatible() + let secret = EnvironmentSecrets::python_compatible(litellm_http::Client::plain_for_test()) .resolve(&[name]) .await .unwrap() @@ -42,7 +42,7 @@ async fn dynamic_names_use_the_same_resolver_and_snapshots_never_do_fresh_lookup reads.fetch_add(1, Ordering::SeqCst); (name != "missing").then(|| name.to_owned()) }), - OidcResolver::default(), + OidcResolver::new(litellm_http::Client::plain_for_test()), ); let snapshot = source.resolve(&["declared", "missing"]).await.unwrap(); let name = format!("runtime-{}", "key"); diff --git a/litellm-rust/crates/testkit/src/lib.rs b/litellm-rust/crates/testkit/src/lib.rs index 9ea6123a176..423adeec426 100644 --- a/litellm-rust/crates/testkit/src/lib.rs +++ b/litellm-rust/crates/testkit/src/lib.rs @@ -1,3 +1,9 @@ +#![allow( + clippy::disallowed_types, + clippy::disallowed_methods, + reason = "a dev-only installer tool that never talks to providers" +)] + mod agent; mod error; mod install;