diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 6632fd279fd..ff2232ed0e7 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3664,6 +3664,7 @@ version = "0.1.0" dependencies = [ "litellm-config", "litellm-core-utils", + "litellm-http-types", "rstest", "serde_yaml_ng", "tempfile", @@ -3851,6 +3852,7 @@ dependencies = [ "http 1.4.2", "litellm-auth-types", "litellm-config", + "litellm-gateway-auth", "litellm-secrets", "moka", "rmcp", @@ -3958,6 +3960,7 @@ dependencies = [ "http 1.4.2", "hyper-util", "litellm-core-utils", + "litellm-http-types", "rcgen", "reqwest 0.12.28", "reqwest 0.13.5", @@ -3977,6 +3980,15 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "litellm-http-types" +version = "0.1.0" +dependencies = [ + "litellm-core-utils", + "rstest", + "veil", +] + [[package]] name = "litellm-inference" version = "0.1.0" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index ce86003de28..d290ca4ce7d 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -48,6 +48,7 @@ litellm-secrets-hashicorp = { path = "crates/secrets-hashicorp" } litellm-secrets-azure = { path = "crates/secrets-azure" } litellm-secrets-cyberark = { path = "crates/secrets-cyberark" } litellm-http = { path = "crates/http" } +litellm-http-types = { path = "crates/http-types" } litellm-llms = { path = "crates/llms" } litellm-llms-types = { path = "crates/llms-types" } litellm-core-utils = { path = "crates/core-utils" } diff --git a/litellm-rust/crates/auth-types/AGENTS.md b/litellm-rust/crates/auth-types/AGENTS.md new file mode 100644 index 00000000000..60998e4d399 --- /dev/null +++ b/litellm-rust/crates/auth-types/AGENTS.md @@ -0,0 +1,35 @@ +## Scope + +- This crate is the vocabulary for authenticating LiteLLM to a provider: how a credential is named before it is known (`CredentialRef`, `CredentialPlan`, `ParamSpec`), how it is obtained (`CredentialResolver`, `TokenProvider`, `ResolvedCredential`), how it is carried (`SecretValue`, `Sourced`), how it lands on a request (`CredentialPlacement`, `ProviderAuthPolicy`) and how that fails (`Error`) +- Outbound only. Who is calling the gateway is `gateway-auth`. The one exception is `UiCredentials`, which is here because `gateway-auth` and `gateway-ui` both read it and neither may depend on the other +- No I/O and no clock. Resolving a reference, fetching a token or reading a file is a `CredentialResolver` or `TokenProvider` implementation in `auth-aws`, `auth-azure`, `auth-gcp` or a host. This crate only defines the traits and runs the plan over an injected handle +- Depends on `serde`, `serde_json`, `subtle`, `thiserror` and `veil` only. Nothing here may pull in an HTTP client, a cloud SDK or a provider crate + +## Layering + +- `auth-types` (contracts) is below `auth-aws`, `auth-azure` and `auth-gcp` (one cloud's acquisition each), which are below `auth` (the `AuthServices` facade that bundles them behind features). `llms`, `inference-*`, `secrets-*` and `cache-*` take the types from here and the services from `auth` +- Import every type from `litellm_auth_types`. `auth` still glob re-exports this crate and `secrets-types` re-exports `SecretValue`, both against the workspace rule in `../inference/AGENTS.md`. Do not add a third path, and move consumers off those two when touching them +- A provider's own credential shape (`AwsParams`, `VertexParams`) lives in `src/params/` only because routing, config and the provider all read it. A field or env name used by one provider stays in that provider's crate + +## Contracts + +- `SecretValue` is the only way a secret moves between crates. `Debug` is redacted through `veil`, equality is constant time, and `expose()` is the single read. Never add `Display`, `Serialize` or an `AsRef` to it +- `Sourced` records whether a value came from the request, the deployment or the environment. Precedence between those is decided by whoever builds the `Sourced`, never by this crate +- `CredentialPlan::resolve` has three outcomes and they are not interchangeable. `Resolved` carries a credential, `Unavailable` means the caller may fall back to the next plan, and `Err` is terminal. A resolver that cannot answer returns `Declined` or `Missing`, it never errors to mean "not mine" +- A `CredentialRef::Explicit` resolves without consulting the resolver. Everything else, including `Host`, is resolved at acquisition time so rotated secrets are read fresh +- `apply_credential` prepends the credential header and refuses an empty credential or a header already present. Whether an existing header is preserved or rejected is `ProviderAuthPolicy::existing_header_behavior`, chosen per provider, never a default here +- `ParamSpec` lists every place Python reads one connection param, in order: wire names, the `litellm.` module global, then environment names. `resolve` walks that order and treats blank as absent. The static tables in `src/params/` are the one copy of those names + +## Errors + +- `Error` is the single credential error every provider nests. `auth-aws` maps its SDK failures into it with `From`, `auth-azure` and `auth-gcp` build the variants directly. Either way a consumer never sees a cloud SDK error type +- `MissingApiKey`, `MissingApiBase` and `MissingParam` are what the end user reads when a setting is absent. The message names what to pass or set, and `ParamSpec::guidance` derives that from the spec so the two never drift +- `ErrorDetail::Failed` wraps a source error by `Arc`, so equality is pointer identity. Tests compare the `Message`, `Empty`, `InvalidType` and `DuplicateHeader` variants and only match on `Failed` + +## Known gaps + +- `ProviderAuthPolicy` and `CredentialPlan` have no production consumer yet, `auth/tests/facade.rs` is the only user. Wire them through `llms/src/base_llm/auth.rs` or delete them, do not grow them unused + +## Tests + +- Test what crosses the boundary: redaction in `Debug` output, which header a placement writes, which plan outcome a resolver answer produces. Never test that a type derives something diff --git a/litellm-rust/crates/auth-types/src/lib.rs b/litellm-rust/crates/auth-types/src/lib.rs index c875bda7979..3e807431610 100644 --- a/litellm-rust/crates/auth-types/src/lib.rs +++ b/litellm-rust/crates/auth-types/src/lib.rs @@ -7,6 +7,7 @@ mod params; mod policy; mod secret; mod token; +mod ui; use serde::{Deserialize, Serialize}; @@ -57,3 +58,4 @@ pub use params::{AwsParams, ParamSpec, VertexParams}; pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy}; pub use secret::SecretValue; pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle}; +pub use ui::UiCredentials; diff --git a/litellm-rust/crates/auth-types/src/ui.rs b/litellm-rust/crates/auth-types/src/ui.rs new file mode 100644 index 00000000000..9e8aaa6d1b7 --- /dev/null +++ b/litellm-rust/crates/auth-types/src/ui.rs @@ -0,0 +1,9 @@ +use serde::Deserialize; + +use crate::SecretValue; + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct UiCredentials { + pub username: String, + pub password: SecretValue, +} diff --git a/litellm-rust/crates/config-load/AGENTS.md b/litellm-rust/crates/config-load/AGENTS.md index bd4892d8d40..bfd68815a9a 100644 --- a/litellm-rust/crates/config-load/AGENTS.md +++ b/litellm-rust/crates/config-load/AGENTS.md @@ -9,3 +9,4 @@ ## Precedence - Each lowered value takes the env, the file and then the default in the order Python uses for that key today, until that order is normalized +- HTTP client settings are lowered into `litellm-http-types` layers (`http_settings`), one layer per source, so this crate never defines its own transport types diff --git a/litellm-rust/crates/config-load/Cargo.toml b/litellm-rust/crates/config-load/Cargo.toml index 5810cccb77b..49343a46353 100644 --- a/litellm-rust/crates/config-load/Cargo.toml +++ b/litellm-rust/crates/config-load/Cargo.toml @@ -8,6 +8,7 @@ repository.workspace = true [dependencies] litellm-config.workspace = true litellm-core-utils.workspace = true +litellm-http-types.workspace = true serde_yaml_ng = "0.10.0" thiserror.workspace = true diff --git a/litellm-rust/crates/config-load/src/http.rs b/litellm-rust/crates/config-load/src/http.rs new file mode 100644 index 00000000000..f672cea4ae8 --- /dev/null +++ b/litellm-rust/crates/config-load/src/http.rs @@ -0,0 +1,156 @@ +use litellm_config::{LiteLlmSettings, Spelled}; +use litellm_http_types::{HttpSettings, HttpSettingsLayer, SslVerify}; + +use crate::Environment; + +pub fn http_settings(settings: &LiteLlmSettings, environment: &Environment) -> HttpSettings { + HttpSettings::from_layers([ + HttpSettingsLayer::from_environment(environment), + written(settings), + ]) +} + +fn written(settings: &LiteLlmSettings) -> HttpSettingsLayer { + HttpSettingsLayer { + ssl_verify: settings.tls.ssl_verify.as_ref().map(ssl_verify), + ssl_certificate: settings + .client_certificate + .ssl_certificate + .clone() + .map(Into::into), + ssl_security_level: settings.tls.ssl_security_level.clone(), + ssl_ecdh_curve: settings.tls.ssl_ecdh_curve.clone(), + force_ipv4: settings.socket.force_ipv4, + http2: settings.http2.http2, + aiohttp_trust_env: settings.aiohttp.aiohttp_trust_env, + disable_aiohttp_trust_env: settings.aiohttp.disable_aiohttp_trust_env, + disable_aiohttp_transport: settings.aiohttp.disable_aiohttp_transport, + ..HttpSettingsLayer::default() + } +} + +fn ssl_verify(value: &Spelled) -> SslVerify { + match value { + Spelled::Value(true) => SslVerify::Enabled, + Spelled::Value(false) => SslVerify::Disabled, + Spelled::Text(text) => SslVerify::parse(text), + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use rstest::rstest; + + use litellm_config::Config; + use litellm_http_types::{HttpSettings, SslVerify}; + + use super::http_settings; + use crate::Environment; + + fn lowered(yaml: &str, variables: &[(&str, &str)]) -> HttpSettings { + let config = Config::from_yaml(yaml).unwrap(); + let environment = variables + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .collect(); + http_settings(&config.litellm_settings, &environment) + } + + #[test] + fn absent_everywhere_is_the_default() { + assert_eq!(lowered("{}", &[]), HttpSettings::default()); + } + + #[rstest] + #[case::default("{}", &[], None)] + #[case::written("litellm_settings: {ssl_verify: false}", &[], Some(SslVerify::Disabled))] + #[case::written_text("litellm_settings: {ssl_verify: 'False'}", &[], Some(SslVerify::Disabled))] + #[case::written_bundle("litellm_settings: {ssl_verify: /etc/roots.pem}", &[], Some(SslVerify::CaBundle(PathBuf::from("/etc/roots.pem"))))] + #[case::environment("{}", &[("SSL_VERIFY", "False")], Some(SslVerify::Disabled))] + #[case::environment_over_file("litellm_settings: {ssl_verify: false}", &[("SSL_VERIFY", "true")], Some(SslVerify::Enabled))] + fn ssl_verify_takes_environment_then_file( + #[case] yaml: &str, + #[case] variables: &[(&str, &str)], + #[case] expected: Option, + ) { + assert_eq!(lowered(yaml, variables).ssl_verify, expected); + } + + #[rstest] + #[case::file("litellm_settings: {ssl_certificate: /etc/client.pem}", &[], "/etc/client.pem")] + #[case::environment_over_file("litellm_settings: {ssl_certificate: /etc/client.pem}", &[("SSL_CERTIFICATE", "/env/client.pem")], "/env/client.pem")] + fn client_certificate_takes_environment_then_file( + #[case] yaml: &str, + #[case] variables: &[(&str, &str)], + #[case] expected: &str, + ) { + assert_eq!( + lowered(yaml, variables).ssl_certificate, + Some(PathBuf::from(expected)) + ); + } + + #[rstest] + #[case::file("litellm_settings: {http2: true}", &[], true)] + #[case::environment("{}", &[("LITELLM_HTTP2", "TRUE")], true)] + #[case::environment_false_does_not_override_file("litellm_settings: {http2: true}", &[("LITELLM_HTTP2", "false")], true)] + fn flags_are_only_turned_on_by_the_environment( + #[case] yaml: &str, + #[case] variables: &[(&str, &str)], + #[case] expected: bool, + ) { + assert_eq!(lowered(yaml, variables).http2, expected); + } + + #[test] + fn force_ipv4_comes_from_the_file_only() { + assert!(lowered("litellm_settings: {force_ipv4: true}", &[]).force_ipv4); + assert!(!lowered("{}", &[("FORCE_IPV4", "true")]).force_ipv4); + } + + #[rstest] + #[case::file("litellm_settings: {ssl_security_level: '1'}", &[], Some("1"))] + #[case::environment_over_file("litellm_settings: {ssl_security_level: '1'}", &[("SSL_SECURITY_LEVEL", "2")], Some("2"))] + #[case::empty_is_unset("{}", &[("SSL_SECURITY_LEVEL", "")], None)] + fn text_settings_skip_empty_values( + #[case] yaml: &str, + #[case] variables: &[(&str, &str)], + #[case] expected: Option<&str>, + ) { + assert_eq!( + lowered(yaml, variables).ssl_security_level.as_deref(), + expected + ); + } + + #[test] + fn aiohttp_flags_from_the_file_decide_proxy_trust() { + assert!( + !lowered("litellm_settings: {disable_aiohttp_trust_env: true}", &[]).trust_proxy_env + ); + assert!( + lowered( + "litellm_settings: {disable_aiohttp_trust_env: true, aiohttp_trust_env: true}", + &[] + ) + .trust_proxy_env + ); + } + + #[test] + fn later_overlay_wins_and_unrelated_variables_are_ignored() { + let environment = Environment::from_iter([("SSL_VERIFY".to_owned(), "true".to_owned())]) + .overlaid([ + ("SSL_VERIFY".to_owned(), "false".to_owned()), + ("HTTP2".to_owned(), "true".to_owned()), + ]); + let settings = http_settings( + &Config::from_yaml("{}").unwrap().litellm_settings, + &environment, + ); + assert_eq!(settings.ssl_verify, Some(SslVerify::Disabled)); + assert!(!settings.http2); + } +} diff --git a/litellm-rust/crates/config-load/src/lib.rs b/litellm-rust/crates/config-load/src/lib.rs index 62f1ed876b5..0cd4f3b58a4 100644 --- a/litellm-rust/crates/config-load/src/lib.rs +++ b/litellm-rust/crates/config-load/src/lib.rs @@ -1,16 +1,13 @@ mod error; +mod http; pub mod include; -mod transport; use std::{collections::BTreeMap, ffi::OsString, path::Path}; use litellm_config::Config; use litellm_core_utils::settings::Lookup; -pub use self::{ - error::Error, - transport::{SslVerify, Transport}, -}; +pub use self::{error::Error, http::http_settings}; pub fn load(path: impl AsRef) -> Result { let path = path.as_ref(); diff --git a/litellm-rust/crates/config-load/src/transport.rs b/litellm-rust/crates/config-load/src/transport.rs deleted file mode 100644 index ba1f95b8296..00000000000 --- a/litellm-rust/crates/config-load/src/transport.rs +++ /dev/null @@ -1,166 +0,0 @@ -use std::path::PathBuf; - -use litellm_core_utils::{serde_compat::parse_str_bool, settings::Lookup}; - -use litellm_config::{LiteLlmSettings, Spelled}; - -use crate::Environment; - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub enum SslVerify { - #[default] - Enabled, - Disabled, - CaBundle(PathBuf), -} - -impl SslVerify { - fn parse(value: &str) -> Self { - match parse_str_bool(value) { - Some(true) => Self::Enabled, - Some(false) => Self::Disabled, - None => Self::CaBundle(PathBuf::from(value)), - } - } - - fn written(value: &Spelled) -> Self { - match value { - Spelled::Value(true) => Self::Enabled, - Spelled::Value(false) => Self::Disabled, - Spelled::Text(text) => Self::parse(text), - } - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct Transport { - pub ssl_verify: SslVerify, - pub ssl_certificate: Option, - pub ssl_security_level: Option, - pub ssl_ecdh_curve: Option, - pub force_ipv4: bool, - pub http2: bool, - pub aiohttp_trust_env: bool, - pub disable_aiohttp_trust_env: bool, - pub disable_aiohttp_transport: bool, -} - -impl Transport { - pub fn lower(settings: &LiteLlmSettings, environment: &Environment) -> Self { - let text = |name: &str, written: &Option| { - environment - .get(name) - .or_else(|| written.clone()) - .filter(|value| !value.is_empty()) - }; - let flag = |name: &str, written: Option| { - environment.enabled(name).or(written).unwrap_or(false) - }; - Self { - ssl_verify: environment - .get("SSL_VERIFY") - .map(|value| SslVerify::parse(&value)) - .or_else(|| settings.tls.ssl_verify.as_ref().map(SslVerify::written)) - .unwrap_or_default(), - ssl_certificate: environment - .get("SSL_CERTIFICATE") - .or_else(|| settings.client_certificate.ssl_certificate.clone()) - .map(PathBuf::from), - ssl_security_level: text("SSL_SECURITY_LEVEL", &settings.tls.ssl_security_level), - ssl_ecdh_curve: text("SSL_ECDH_CURVE", &settings.tls.ssl_ecdh_curve), - force_ipv4: settings.socket.force_ipv4.unwrap_or(false), - http2: flag("LITELLM_HTTP2", settings.http2.http2), - aiohttp_trust_env: flag("AIOHTTP_TRUST_ENV", settings.aiohttp.aiohttp_trust_env), - disable_aiohttp_trust_env: flag( - "DISABLE_AIOHTTP_TRUST_ENV", - settings.aiohttp.disable_aiohttp_trust_env, - ), - disable_aiohttp_transport: flag( - "DISABLE_AIOHTTP_TRANSPORT", - settings.aiohttp.disable_aiohttp_transport, - ), - } - } -} - -#[cfg(test)] -mod tests { - use std::path::PathBuf; - - use rstest::rstest; - - use super::{SslVerify, Transport}; - use litellm_config::Config; - - use crate::Environment; - - fn lowered(yaml: &str, variables: &[(&str, &str)]) -> Transport { - let config = Config::from_yaml(yaml).unwrap(); - let environment = variables - .iter() - .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) - .collect(); - Transport::lower(&config.litellm_settings, &environment) - } - - #[rstest] - fn absent_everywhere_is_the_default() { - assert_eq!(lowered("{}", &[]), Transport::default()); - } - - #[rstest] - #[case::default("{}", &[], SslVerify::Enabled)] - #[case::written("litellm_settings: {ssl_verify: false}", &[], SslVerify::Disabled)] - #[case::written_bundle("litellm_settings: {ssl_verify: /etc/roots.pem}", &[], SslVerify::CaBundle(PathBuf::from("/etc/roots.pem")))] - #[case::environment("{}", &[("SSL_VERIFY", "False")], SslVerify::Disabled)] - #[case::environment_over_file("litellm_settings: {ssl_verify: false}", &[("SSL_VERIFY", "true")], SslVerify::Enabled)] - fn ssl_verify_takes_environment_then_file_then_default( - #[case] yaml: &str, - #[case] variables: &[(&str, &str)], - #[case] expected: SslVerify, - ) { - assert_eq!(lowered(yaml, variables).ssl_verify, expected); - } - - #[rstest] - #[case::file("litellm_settings: {http2: true}", &[], true)] - #[case::environment("{}", &[("LITELLM_HTTP2", "TRUE")], true)] - #[case::environment_false_does_not_override_file("litellm_settings: {http2: true}", &[("LITELLM_HTTP2", "false")], true)] - fn flags_are_only_turned_on_by_the_environment( - #[case] yaml: &str, - #[case] variables: &[(&str, &str)], - #[case] expected: bool, - ) { - assert_eq!(lowered(yaml, variables).http2, expected); - } - - #[rstest] - #[case::file("litellm_settings: {ssl_security_level: '1'}", &[], Some("1"))] - #[case::environment_over_file("litellm_settings: {ssl_security_level: '1'}", &[("SSL_SECURITY_LEVEL", "2")], Some("2"))] - #[case::empty_is_unset("{}", &[("SSL_SECURITY_LEVEL", "")], None)] - fn text_settings_skip_empty_values( - #[case] yaml: &str, - #[case] variables: &[(&str, &str)], - #[case] expected: Option<&str>, - ) { - assert_eq!( - lowered(yaml, variables).ssl_security_level.as_deref(), - expected - ); - } - - #[rstest] - fn later_overlay_wins_and_unrelated_variables_are_ignored() { - let environment = Environment::from_iter([("SSL_VERIFY".to_owned(), "true".to_owned())]) - .overlaid([ - ("SSL_VERIFY".to_owned(), "false".to_owned()), - ("HTTP2".to_owned(), "true".to_owned()), - ]); - let transport = Transport::lower( - &Config::from_yaml("{}").unwrap().litellm_settings, - &environment, - ); - assert_eq!(transport.ssl_verify, SslVerify::Disabled); - assert!(!transport.http2); - } -} diff --git a/litellm-rust/crates/gateway-auth/src/lib.rs b/litellm-rust/crates/gateway-auth/src/lib.rs index 1b400a94746..6d5621c8407 100644 --- a/litellm-rust/crates/gateway-auth/src/lib.rs +++ b/litellm-rust/crates/gateway-auth/src/lib.rs @@ -22,7 +22,7 @@ pub use identity::{ AuthenticatedCaller, Authentication, AuthenticationMethod, Principal, PrincipalKind, ResolvedIdentity, SharedCaller, VerifiedIdentity, }; -pub use ui::{UI_CSRF_KEY, UiAuthSession, UiBackend, UiCredentials, UiSession, UiUser}; +pub use ui::{UI_CSRF_KEY, UiAuthSession, UiBackend, UiSession, UiUser}; pub fn hash_token(token: &str) -> String { format!("{:x}", Sha256::digest(token.as_bytes())) diff --git a/litellm-rust/crates/gateway-auth/src/ui.rs b/litellm-rust/crates/gateway-auth/src/ui.rs index 9e6b77a17d0..e1ede25d786 100644 --- a/litellm-rust/crates/gateway-auth/src/ui.rs +++ b/litellm-rust/crates/gateway-auth/src/ui.rs @@ -3,8 +3,7 @@ use std::{convert::Infallible, sync::Arc}; use axum::extract::FromRequestParts; use axum::http::{header::AUTHORIZATION, request::Parts}; use axum_login::{AuthUser, AuthnBackend, UserId}; -use litellm_auth_types::SecretValue; -use serde::Deserialize; +use litellm_auth_types::{SecretValue, UiCredentials}; use sha2::{Digest, Sha256}; use subtle::ConstantTimeEq; use tower_sessions::Session; @@ -38,12 +37,6 @@ impl AuthUser for UiUser { } } -#[derive(Deserialize)] -pub struct UiCredentials { - pub username: String, - pub password: SecretValue, -} - #[derive(Clone)] pub struct UiBackend { user: UiUser, diff --git a/litellm-rust/crates/gateway-auth/tests/ui.rs b/litellm-rust/crates/gateway-auth/tests/ui.rs index 2e5f72d605d..cb8ef97d325 100644 --- a/litellm-rust/crates/gateway-auth/tests/ui.rs +++ b/litellm-rust/crates/gateway-auth/tests/ui.rs @@ -1,6 +1,6 @@ use axum_login::{AuthUser, AuthnBackend}; -use litellm_auth_types::SecretValue; -use litellm_gateway_auth::{UiBackend, UiCredentials}; +use litellm_auth_types::{SecretValue, UiCredentials}; +use litellm_gateway_auth::UiBackend; use rstest::{fixture, rstest}; #[fixture] diff --git a/litellm-rust/crates/gateway-mcp/Cargo.toml b/litellm-rust/crates/gateway-mcp/Cargo.toml index 07706dcb6a4..9b7db77d7ce 100644 --- a/litellm-rust/crates/gateway-mcp/Cargo.toml +++ b/litellm-rust/crates/gateway-mcp/Cargo.toml @@ -7,11 +7,13 @@ license.workspace = true repository.workspace = true [dependencies] -axum = { workspace = true, features = ["json", "query"] } -futures-util.workspace = true litellm-config.workspace = true +litellm-gateway-auth.workspace = true litellm-secrets.workspace = true litellm-auth-types.workspace = true + +axum = { workspace = true, features = ["json", "query"] } +futures-util.workspace = true base64.workspace = true http.workspace = true uuid.workspace = true diff --git a/litellm-rust/crates/gateway/src/auth.rs b/litellm-rust/crates/gateway-mcp/src/auth.rs similarity index 72% rename from litellm-rust/crates/gateway/src/auth.rs rename to litellm-rust/crates/gateway-mcp/src/auth.rs index 72892be783c..9a599372bd1 100644 --- a/litellm-rust/crates/gateway/src/auth.rs +++ b/litellm-rust/crates/gateway-mcp/src/auth.rs @@ -1,13 +1,13 @@ use std::sync::Arc; use axum::{extract::Request, middleware::Next, response::Response}; -use litellm_gateway_auth::{AccessRequest, AuthenticatedRequest, Error, McpAction}; -use litellm_gateway_mcp::{ - Authorization, GatewayFuture, OperationAuthorizer, ServerInfo, SessionOwner, - rmcp::model::ClientRequest, -}; +use rmcp::model::ClientRequest; -pub(crate) async fn bind_session_owner( +use litellm_gateway_auth::{AccessRequest, AuthenticatedRequest, McpAction}; + +use crate::{Authorization, GatewayFuture, OperationAuthorizer, ServerInfo, SessionOwner}; + +pub async fn bind_caller( identity: AuthenticatedRequest, mut request: Request, next: Next, @@ -47,7 +47,7 @@ impl OperationAuthorizer for McpPolicy { Some(ClientRequest::ReadResourceRequest(request)) => { McpAction::ReadResource(request.params.uri.clone()) } - Some(_) => return Err(litellm_gateway_mcp::Error::Forbidden), + Some(_) => return Err(crate::Error::Forbidden), }; let access = AccessRequest::Mcp { server: server.server_id.clone(), @@ -55,19 +55,9 @@ impl OperationAuthorizer for McpPolicy { }; self.0 .authorize(access.clone()) - .await - .map_err(mcp_error)? - .consume(self.0.caller(), &access) - .map_err(mcp_error) + .await? + .consume(self.0.caller(), &access)?; + Ok(()) }) } } - -fn mcp_error(error: Error) -> litellm_gateway_mcp::Error { - match error { - Error::InvalidToken | Error::Expired | Error::Forbidden => { - litellm_gateway_mcp::Error::Forbidden - } - _ => litellm_gateway_mcp::Error::Configuration("MCP authorization unavailable".into()), - } -} diff --git a/litellm-rust/crates/gateway-mcp/src/error.rs b/litellm-rust/crates/gateway-mcp/src/error.rs index 1aa72e34703..cee785a183b 100644 --- a/litellm-rust/crates/gateway-mcp/src/error.rs +++ b/litellm-rust/crates/gateway-mcp/src/error.rs @@ -3,6 +3,7 @@ use axum::{ http::StatusCode, response::{IntoResponse, Response}, }; +use litellm_gateway_auth::Error as AuthError; use rmcp::{ model::{ErrorCode, ErrorData}, service::ServiceError, @@ -23,6 +24,8 @@ pub enum Error { UnexpectedResult, #[error("MCP request cancelled")] Cancelled, + #[error(transparent)] + Auth(#[from] AuthError), } impl Error { @@ -30,7 +33,10 @@ impl Error { match self { Self::Upstream(ServiceError::McpError(error)) => error, Self::InvalidRequest(message) => ErrorData::invalid_params(message, None), - Self::Forbidden => ErrorData::new(ErrorCode(-32003), self.to_string(), None), + Self::Forbidden + | Self::Auth(AuthError::Forbidden | AuthError::InvalidToken | AuthError::Expired) => { + ErrorData::new(ErrorCode(-32003), Self::Forbidden.to_string(), None) + } _ => ErrorData::internal_error(self.to_string(), None), } } @@ -44,6 +50,7 @@ impl IntoResponse for Error { Self::Upstream(_) => StatusCode::BAD_GATEWAY, Self::Cancelled => StatusCode::REQUEST_TIMEOUT, Self::Configuration(_) | Self::UnexpectedResult => StatusCode::INTERNAL_SERVER_ERROR, + Self::Auth(error) => error.status(), }; (status, Json(json!({"detail": self.to_string()}))).into_response() } diff --git a/litellm-rust/crates/gateway-mcp/src/lib.rs b/litellm-rust/crates/gateway-mcp/src/lib.rs index dd72940e24b..639af9fc708 100644 --- a/litellm-rust/crates/gateway-mcp/src/lib.rs +++ b/litellm-rust/crates/gateway-mcp/src/lib.rs @@ -1,3 +1,4 @@ +mod auth; mod configured; mod error; mod legacy; @@ -12,6 +13,7 @@ use std::{collections::BTreeMap, future::Future, pin::Pin, sync::Arc}; use http::request::Parts; use rmcp::{RoleServer, model::*, service::RequestContext}; +pub use auth::bind_caller; pub use configured::ConfiguredGateway; pub use error::{ConnectError, Error}; pub use native::{Limits, NativeGateway, Registry, Server, ServerResolver}; diff --git a/litellm-rust/crates/gateway-ui/Cargo.toml b/litellm-rust/crates/gateway-ui/Cargo.toml index 94546ef9118..b69de1886d7 100644 --- a/litellm-rust/crates/gateway-ui/Cargo.toml +++ b/litellm-rust/crates/gateway-ui/Cargo.toml @@ -11,6 +11,7 @@ axum-login.workspace = true base64.workspace = true governor = { version = "0.10.4", default-features = false, features = ["std"] } jsonwebtoken = { workspace = true, features = ["rust_crypto"] } +litellm-auth-types.workspace = true litellm-gateway-auth.workspace = true rand.workspace = true serde.workspace = true @@ -24,7 +25,6 @@ tower-sessions.workspace = true [dev-dependencies] serde_json.workspace = true tower = { version = "0.5", features = ["util"] } -litellm-auth-types.workspace = true rstest.workspace = true tempfile.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/gateway-ui/src/session.rs b/litellm-rust/crates/gateway-ui/src/session.rs index b80f40bb376..a98080abe95 100644 --- a/litellm-rust/crates/gateway-ui/src/session.rs +++ b/litellm-rust/crates/gateway-ui/src/session.rs @@ -10,8 +10,9 @@ use tower_cookies::{Cookie, Cookies}; use tower_sessions::{Expiry, Session, cookie::SameSite}; use crate::Error; +use litellm_auth_types::UiCredentials; use litellm_gateway_auth::{ - AccessRequest, UI_CSRF_KEY, UiAction, UiAuthError, UiAuthSession, UiCredentials, UiSession, + AccessRequest, UI_CSRF_KEY, UiAction, UiAuthError, UiAuthSession, UiSession, }; pub struct State { diff --git a/litellm-rust/crates/gateway/src/lib.rs b/litellm-rust/crates/gateway/src/lib.rs index a495c95238b..001a5a0aa1a 100644 --- a/litellm-rust/crates/gateway/src/lib.rs +++ b/litellm-rust/crates/gateway/src/lib.rs @@ -1,4 +1,3 @@ -mod auth; mod error; mod secrets; pub use error::Error; @@ -15,13 +14,10 @@ use axum::{ use http_body_util::BodyExt; use litellm_config::Config; -use litellm_config_load::{Environment, Transport}; +use litellm_config_load::Environment; use litellm_gateway_auth::Auth; use litellm_gateway_inference::{Gateway, ModelRouter}; -use litellm_http::{ - ClientVariant, HttpClientPool, HttpSettings, HttpSettingsLayer, Resolution, SslVerify, - media::PublicDnsResolver, -}; +use litellm_http::{ClientVariant, HttpClientPool, Resolution, media::PublicDnsResolver}; use litellm_inference::resources::CoreResources; use litellm_secrets::source::EnvironmentSecrets; use litellm_tracing::ByteChunk; @@ -35,11 +31,7 @@ pub fn build_inference(config: &Config, process: &Environment) -> Result Result HttpSettingsLayer { - HttpSettingsLayer { - ssl_verify: Some(match &transport.ssl_verify { - litellm_config_load::SslVerify::Enabled => SslVerify::Enabled, - litellm_config_load::SslVerify::Disabled => SslVerify::Disabled, - litellm_config_load::SslVerify::CaBundle(path) => SslVerify::CaBundle(path.clone()), - }), - ssl_certificate: transport.ssl_certificate.clone(), - ssl_security_level: transport.ssl_security_level.clone(), - ssl_ecdh_curve: transport.ssl_ecdh_curve.clone(), - force_ipv4: Some(transport.force_ipv4), - http2: Some(transport.http2), - aiohttp_trust_env: Some(transport.aiohttp_trust_env), - disable_aiohttp_trust_env: Some(transport.disable_aiohttp_trust_env), - disable_aiohttp_transport: Some(transport.disable_aiohttp_transport), - ..Default::default() - } -} - pub async fn build_mcp( config: &Config, secrets: Arc, @@ -107,8 +80,12 @@ pub fn router( ) -> Router { let auth = Auth::from_config(config, inference.secrets.clone()); let inference = litellm_gateway_inference::router(inference) - .merge(mcp.unwrap_or_default()) - .route_layer(axum::middleware::from_fn(auth::bind_session_owner)) + .merge( + mcp.map(|mcp| { + mcp.route_layer(axum::middleware::from_fn(litellm_gateway_mcp::bind_caller)) + }) + .unwrap_or_default(), + ) .route_layer(axum::middleware::from_fn_with_state( auth, litellm_gateway_auth::authenticate, diff --git a/litellm-rust/crates/http-types/AGENTS.md b/litellm-rust/crates/http-types/AGENTS.md new file mode 100644 index 00000000000..ea60faa77b0 --- /dev/null +++ b/litellm-rust/crates/http-types/AGENTS.md @@ -0,0 +1,21 @@ +## Scope + +- This crate is the HTTP client settings as the user writes them, before anything decides what a client can do with them. `HttpSettingsLayer` is one source's values, where `None` means that source did not set the key. `HttpSettings::from_layers` merges layers highest precedence first and fills defaults. `HttpSettings` is the merged result +- It knows the key names the user writes (`SSL_VERIFY`, `AIOHTTP_TRUST_ENV`, `HTTP_PROXY`, ...) and how Python reads them (`SslVerify::parse` is bool-or-path, lowercase proxy names win, `HTTP_PROXY` is ignored under CGI). It knows nothing about reqwest, rustls or which cipher a security level selects. That is `../http`, through `Resolution::from(&HttpSettings)` +- No I/O. Every environment read goes through an injected `Lookup`, and `without_missing_files` takes the existence check as a closure +- Depends on `litellm-core-utils` (`Lookup`, `Layer`, `merge`) and `veil` only. It must stay buildable without reqwest, so a settings producer can emit a layer without linking a TLS stack + +## Producers and the consumer + +- A host produces one `HttpSettingsLayer` per source it trusts and lets `from_layers` decide. It never pre-merges two sources into one layer +- `config-load` produces the gateway's layers from config.yaml and the environment, `python-bridge` produces the SDK's from the Python settings snapshot. `../http` is the only consumer. Anything that wants a client goes through `HttpClientConfig`, not these types +- Defaults live in `HttpSettings::default` and are applied only in `from_layers`. A layer never carries a default +- Python flags that only exist to be combined (`aiohttp_trust_env`, `disable_aiohttp_trust_env`, `disable_aiohttp_transport`) stay as the user's flags on the layer. `from_layers` collapses them into `trust_proxy_env` once, so no consumer re-derives it + +## Redaction + +- Proxy URLs can embed credentials, so `EnvironmentProxies` redacts `all`, `http` and `https` and exposes them only through accessors. Keep `Debug` derived through `veil`, never hand-written + +## Tests + +- Assert the merged `HttpSettings` for a given stack of layers, and the parsed layer for a given injected environment, never the shape of the helpers diff --git a/litellm-rust/crates/http-types/Cargo.toml b/litellm-rust/crates/http-types/Cargo.toml new file mode 100644 index 00000000000..71d0d86aa1c --- /dev/null +++ b/litellm-rust/crates/http-types/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "litellm-http-types" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-core-utils.workspace = true +veil.workspace = true + +[dev-dependencies] +rstest.workspace = true diff --git a/litellm-rust/crates/http-types/src/lib.rs b/litellm-rust/crates/http-types/src/lib.rs new file mode 100644 index 00000000000..e62d76f049d --- /dev/null +++ b/litellm-rust/crates/http-types/src/lib.rs @@ -0,0 +1,7 @@ +#![forbid(unsafe_code)] + +mod proxy; +mod settings; + +pub use proxy::EnvironmentProxies; +pub use settings::{HttpSettings, HttpSettingsLayer, SslVerify, TcpKeepalive}; diff --git a/litellm-rust/crates/http-types/src/proxy.rs b/litellm-rust/crates/http-types/src/proxy.rs new file mode 100644 index 00000000000..51c2f658a1a --- /dev/null +++ b/litellm-rust/crates/http-types/src/proxy.rs @@ -0,0 +1,126 @@ +use litellm_core_utils::settings::Lookup; +use veil::Redact; + +#[derive(Clone, Redact, Default, PartialEq, Eq, Hash)] +pub struct EnvironmentProxies { + #[redact] + all: String, + #[redact] + http: String, + #[redact] + https: String, + no: String, +} + +impl EnvironmentProxies { + pub fn from_environment(env: &impl Lookup) -> Self { + Self::resolve(env, cfg!(windows)) + } + + fn resolve(env: &impl Lookup, names_ignore_case: bool) -> Self { + let lowercase_first = |upper: Option<&str>, lower: &str| { + env.get(lower) + .or_else(|| upper.and_then(|name| env.truthy(name))) + .unwrap_or_default() + }; + let is_cgi = env.get("REQUEST_METHOD").is_some(); + Self { + all: lowercase_first(Some("ALL_PROXY"), "all_proxy"), + http: if is_cgi && names_ignore_case { + String::new() + } else { + lowercase_first((!is_cgi).then_some("HTTP_PROXY"), "http_proxy") + }, + https: lowercase_first(Some("HTTPS_PROXY"), "https_proxy"), + no: lowercase_first(Some("NO_PROXY"), "no_proxy"), + } + } + + pub fn all(&self) -> &str { + &self.all + } + + pub fn http(&self) -> &str { + &self.http + } + + pub fn https(&self) -> &str { + &self.https + } + + pub fn no(&self) -> &str { + &self.no + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn env_of(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option { + move |name| { + values + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.to_string()) + } + } + + #[rstest] + #[case::lowercase_wins(&[("HTTPS_PROXY", "http://upper:3128"), ("https_proxy", "http://lower:3128")], &[("https_proxy", "http://lower:3128")])] + #[case::empty_uppercase_falls_through_to_lowercase(&[("HTTPS_PROXY", ""), ("https_proxy", "http://lower:3128")], &[("https_proxy", "http://lower:3128")])] + #[case::empty_uppercase_alone_is_unset(&[("HTTPS_PROXY", "")], &[])] + #[case::empty_lowercase_clears_the_uppercase_value(&[("HTTPS_PROXY", "http://upper:3128"), ("https_proxy", "")], &[])] + #[case::lowercase_no_proxy_wins(&[("NO_PROXY", "upper.test"), ("no_proxy", "lower.test")], &[("no_proxy", "lower.test")])] + #[case::cgi_forgets_the_client_settable_http_proxy(&[("REQUEST_METHOD", "GET"), ("HTTP_PROXY", "http://attacker:3128")], &[])] + #[case::cgi_keeps_lowercase_http_proxy(&[("REQUEST_METHOD", "GET"), ("HTTP_PROXY", "http://attacker:3128"), ("http_proxy", "http://lower:3128")], &[("http_proxy", "http://lower:3128")])] + #[case::cgi_keeps_every_other_variable(&[("REQUEST_METHOD", "GET"), ("HTTPS_PROXY", "http://proxy:3128"), ("ALL_PROXY", "http://all:3128"), ("NO_PROXY", "internal.test")], &[("HTTPS_PROXY", "http://proxy:3128"), ("ALL_PROXY", "http://all:3128"), ("NO_PROXY", "internal.test")])] + fn variables_resolve_like_urllib_getproxies_environment( + #[case] env: &'static [(&'static str, &'static str)], + #[case] equivalent: &'static [(&'static str, &'static str)], + ) { + assert_eq!( + EnvironmentProxies::from_environment(&env_of(env)), + EnvironmentProxies::from_environment(&env_of(equivalent)) + ); + } + + #[test] + fn cgi_drops_http_proxy_entirely_where_variable_names_ignore_case() { + let windows_env = |name: &str| match name.to_ascii_uppercase().as_str() { + "REQUEST_METHOD" => Some("GET".to_string()), + "HTTP_PROXY" => Some("http://attacker:3128".to_string()), + "HTTPS_PROXY" => Some("http://proxy:3128".to_string()), + _ => None, + }; + let ignoring_case = EnvironmentProxies::resolve(&windows_env, true); + assert_eq!(ignoring_case.http(), ""); + assert_eq!(ignoring_case.https(), "http://proxy:3128"); + assert_eq!( + EnvironmentProxies::resolve(&windows_env, false).http(), + "http://attacker:3128" + ); + } + + #[test] + fn debug_output_hides_proxy_credentials_but_shows_which_variables_are_set() { + let proxies = EnvironmentProxies::from_environment(&env_of(&[ + ("HTTPS_PROXY", "http://operator:hunter2@proxy.corp:3128"), + ("NO_PROXY", "internal.test"), + ])); + let debug = format!("{proxies:?}"); + assert!(!debug.contains("hunter2") && !debug.contains("operator")); + assert!(debug.contains("internal.test")); + assert_ne!(debug, format!("{:?}", EnvironmentProxies::default())); + } + + #[test] + fn an_empty_environment_proxies_nothing() { + assert_eq!( + EnvironmentProxies::from_environment(&env_of(&[])), + EnvironmentProxies::default() + ); + } +} diff --git a/litellm-rust/crates/http/src/settings.rs b/litellm-rust/crates/http-types/src/settings.rs similarity index 99% rename from litellm-rust/crates/http/src/settings.rs rename to litellm-rust/crates/http-types/src/settings.rs index 1f437df225c..d7092ac97d5 100644 --- a/litellm-rust/crates/http/src/settings.rs +++ b/litellm-rust/crates/http-types/src/settings.rs @@ -8,7 +8,7 @@ use litellm_core_utils::{ settings::{Layer, Lookup, merge}, }; -use crate::proxy::EnvironmentProxies; +use crate::EnvironmentProxies; #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub enum SslVerify { diff --git a/litellm-rust/crates/http/AGENTS.md b/litellm-rust/crates/http/AGENTS.md index db29b59f80e..612287c7792 100644 --- a/litellm-rust/crates/http/AGENTS.md +++ b/litellm-rust/crates/http/AGENTS.md @@ -1,7 +1,38 @@ -- https://github.com/BerriAI/litellm-docs/blob/main/docs/guides/security_settings.md +## Scope -- Own shared HTTP mechanics, including case-insensitive header lookup, defaults, replacement, and transport -- Do not choose provider credentials, OAuth policy, beta requirements, or model behavior -- Apply provider-supplied header decisions without importing provider implementations -- Preserve existing header precedence, duplicate handling, and unrelated forwarded headers -- Verify observable request behavior rather than the structure of helper functions +- This crate is the one place a `reqwest` client is built. `clippy.toml` disallows `reqwest::Client::new`, `Client::builder` and the TLS builder methods everywhere else, so every outbound connection honors the host's TLS, proxy, keepalive and timeout settings by construction +- Input is `HttpClientConfig`, output is a pooled `Client`. `Resolution::from(&HttpSettings)` is the only way to turn user settings into an `HttpClientConfig`, and it is where names the user wrote (`ssl_ecdh_curve`, `ssl_security_level`) meet what rustls supports. A value rustls cannot honor keeps the default and is reported in `Resolution::unsupported`, never silently dropped and never an error +- Settings as the user writes them (`HttpSettings`, `HttpSettingsLayer`, `SslVerify`, `TcpKeepalive`, `EnvironmentProxies`) live in `../http-types`, which links neither reqwest nor rustls. Crates that produce settings depend on that crate, not on this one +- Route-neutral and Rust-only. Nothing here names a route, a provider, an API format, a credential or a config.yaml key. Provider credentials, OAuth, beta headers and model behavior are decided above and arrive as plain headers +- `gateway` and `python-bridge` are both hosts of this crate. Each lowers its own settings source into an `HttpSettingsLayer` and takes clients from one process-wide `HttpClientPool` + +## What lives here + +- `pool.rs`: `HttpClientPool`, keyed by `(HttpClientConfig, ClientVariant)` with a TTL. `ClientVariant` is the full list of ways a client may differ from the configured one (redirect policy, pinned DNS, dropped client certificate and proxies). Add a variant only for a difference that must be fixed when the client is built, otherwise set it on the request +- `config.rs` and `tls.rs`: `HttpClientConfig` to `reqwest::ClientBuilder` and to `rustls::ClientConfig` +- `proxy.rs`: `EnvironmentProxies` to reqwest proxies, and the URL matcher the media fetcher uses to decide whether a URL leaves through the proxy +- `media.rs`: the SSRF-safe `MediaFetcher`. It is here because address pinning is a build-time property of the client (`dns_resolver2`) and because it needs the proxy matcher. `UrlPolicy` and `DownloadPolicy` are the host's inputs, the host never decides a URL is safe on its own +- `request.rs`: case-insensitive header helpers over `Vec<(String, String)>` and `HeaderPolicy` for forwarding. They keep existing precedence, duplicate handling and unrelated forwarded headers. `string_headers` and `HeaderError` validate the shape of a caller's `extra_headers` and are the only place a request field is named +- `outbound.rs`: `OutboundRequest` serializes the body once and a `RequestSigner` sees those exact bytes. A route builds it after its hooks ran and cannot change it afterwards +- `transport.rs`: `Error::{Http, Network, Connect}` is the one transport error every crate above nests. `Connect` means the request never reached the provider, so a caller may reroute. Messages never carry URLs +- `websocket.rs` and `mcp.rs`: the two connections the reqwest 0.12 builder cannot make. `mcp.rs` repeats the builder chain for the reqwest 0.13 that rmcp needs, delete it when the two versions converge + +## Boundary with consumers + +- Consumers take `litellm_http::Client`, which derefs to `reqwest::Client`, and use reqwest's request and response types directly. This crate owns how a client is built and configured, it is not a wrapper over reqwest's request API +- A consumer builds its request from the `Client` it was given. It never reads the environment, a config file or a settings snapshot to decide TLS, proxy or timeouts +- Tests take `Client::for_test`, `plain_for_test` or `no_redirect_for_test` behind the `test-support` feature. Production code never calls them + +## Known gaps + +- `websocket.rs` builds its own rustls config from native roots and ignores `HttpClientConfig` (`ssl_verify`, CA bundle, client certificate, proxies). Make it take an `HttpClientConfig` like the pool does before adding another WebSocket caller +- `HttpSettings::connect_timeout` has no layer field, so it is always the default + +## Tests + +- Verify observable request behavior (which headers were sent, which proxy was picked, whether a client was reused), never helper structure +- TLS tests generate certificates with `rcgen` and run a local listener. They never depend on a public host + +# references + +- https://github.com/BerriAI/litellm-docs/blob/main/docs/guides/security_settings.md owns the user-facing meaning of `ssl_verify`, `ssl_certificate`, `ssl_security_level` and `ssl_ecdh_curve` diff --git a/litellm-rust/crates/http/Cargo.toml b/litellm-rust/crates/http/Cargo.toml index 7d64467f2f7..d88aa742287 100644 --- a/litellm-rust/crates/http/Cargo.toml +++ b/litellm-rust/crates/http/Cargo.toml @@ -14,6 +14,7 @@ rmcp = { version = "=3.4.1", default-features = false, features = ["client", "tr reqwest-mcp = { package = "reqwest", version = "0.13.2", default-features = false, features = ["rustls-no-provider", "json", "stream", "http2"], optional = true } http.workspace = true litellm-core-utils.workspace = true +litellm-http-types.workspace = true hyper-util.workspace = true reqwest.workspace = true rustls.workspace = true diff --git a/litellm-rust/crates/http/src/config.rs b/litellm-rust/crates/http/src/config.rs index 2f36784bc70..62854cb4b29 100644 --- a/litellm-rust/crates/http/src/config.rs +++ b/litellm-rust/crates/http/src/config.rs @@ -4,10 +4,10 @@ use std::{ time::Duration, }; +use litellm_http_types::{EnvironmentProxies, HttpSettings, SslVerify, TcpKeepalive}; + use crate::{ error::Error, - proxy::EnvironmentProxies, - settings::{HttpSettings, SslVerify, TcpKeepalive}, tls::{CipherSelection, KeyExchangeGroup, Tls12CipherSuite, Unsupported}, }; @@ -122,9 +122,7 @@ impl TryFrom<&HttpClientConfig> for reqwest::ClientBuilder { Some(agent) => with_protocol.user_agent(agent), None => with_protocol, }; - Ok(config - .proxies - .reqwest_proxies() + Ok(crate::proxy::reqwest_proxies(&config.proxies) .into_iter() .fold(with_agent.no_proxy(), reqwest::ClientBuilder::proxy)) } diff --git a/litellm-rust/crates/http/src/lib.rs b/litellm-rust/crates/http/src/lib.rs index d53da607d39..2f4002ac74e 100644 --- a/litellm-rust/crates/http/src/lib.rs +++ b/litellm-rust/crates/http/src/lib.rs @@ -14,7 +14,6 @@ pub mod outbound; mod pool; mod proxy; pub mod request; -mod settings; mod tls; pub mod transport; pub mod websocket; @@ -22,7 +21,8 @@ pub mod websocket; pub use client::Client; pub use config::{ClientIdentity, HttpClientConfig, Resolution, Verify}; pub use error::{Error, TlsSource}; +pub use litellm_http_types::{ + EnvironmentProxies, HttpSettings, HttpSettingsLayer, SslVerify, TcpKeepalive, +}; pub use pool::{ClientVariant, HttpClientPool}; -pub use proxy::EnvironmentProxies; -pub use settings::{HttpSettings, HttpSettingsLayer, SslVerify, TcpKeepalive}; pub use tls::{KeyExchangeGroup, Tls12CipherSuite, Unsupported}; diff --git a/litellm-rust/crates/http/src/mcp.rs b/litellm-rust/crates/http/src/mcp.rs index 2b101fb9c98..faabbe2648c 100644 --- a/litellm-rust/crates/http/src/mcp.rs +++ b/litellm-rust/crates/http/src/mcp.rs @@ -66,9 +66,7 @@ fn build(config: &HttpClientConfig) -> Result { Some(value) => protocol.user_agent(value), None => protocol, }; - config - .proxies - .mcp_proxies() + crate::proxy::mcp_proxies(&config.proxies) .into_iter() .fold(agent.no_proxy(), reqwest_mcp::ClientBuilder::proxy) .build() diff --git a/litellm-rust/crates/http/src/media.rs b/litellm-rust/crates/http/src/media.rs index 22a881831cf..88f9173d21e 100644 --- a/litellm-rust/crates/http/src/media.rs +++ b/litellm-rust/crates/http/src/media.rs @@ -126,7 +126,7 @@ impl MediaFetcher { config: &HttpClientConfig, url_policy: UrlPolicy, ) -> Result { - let uses_proxy: ProxyMatch = Arc::new(config.proxies.matcher()); + let uses_proxy: ProxyMatch = Arc::new(crate::proxy::matcher(&config.proxies)); Self::with_resolution( pool, config, @@ -375,7 +375,9 @@ mod tests { }; use super::*; - use crate::{HttpSettings, Resolution}; + use litellm_http_types::HttpSettings; + + use crate::Resolution; async fn serve(response: &'static [u8]) -> (Url, tokio::task::JoinHandle<()>) { let listener = TcpListener::bind("127.0.0.1:0") diff --git a/litellm-rust/crates/http/src/pool.rs b/litellm-rust/crates/http/src/pool.rs index 78932f9e29f..163acfada05 100644 --- a/litellm-rust/crates/http/src/pool.rs +++ b/litellm-rust/crates/http/src/pool.rs @@ -6,7 +6,9 @@ use std::{ use reqwest::dns::Resolve; -use crate::{client::Client, config::HttpClientConfig, error::Error, proxy::EnvironmentProxies}; +use litellm_http_types::EnvironmentProxies; + +use crate::{client::Client, config::HttpClientConfig, error::Error}; #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum ClientVariant { @@ -128,7 +130,9 @@ mod tests { }; use super::*; - use crate::{ClientIdentity, HttpSettings, Resolution, Verify}; + use litellm_http_types::HttpSettings; + + use crate::{ClientIdentity, Resolution, Verify}; struct FixedResolver(SocketAddr); diff --git a/litellm-rust/crates/http/src/proxy.rs b/litellm-rust/crates/http/src/proxy.rs index e7e54ed6ae2..e1c17b0955e 100644 --- a/litellm-rust/crates/http/src/proxy.rs +++ b/litellm-rust/crates/http/src/proxy.rs @@ -1,81 +1,47 @@ use hyper_util::client::proxy::matcher::Matcher; -use litellm_core_utils::settings::Lookup; -use veil::Redact; +use litellm_http_types::EnvironmentProxies; -#[derive(Clone, Redact, Default, PartialEq, Eq, Hash)] -pub struct EnvironmentProxies { - #[redact] - all: String, - #[redact] - http: String, - #[redact] - https: String, - no: String, +pub(crate) fn matcher( + proxies: &EnvironmentProxies, +) -> impl Fn(&reqwest::Url) -> bool + Send + Sync + use<> { + let matcher = Matcher::builder() + .all(proxies.all()) + .http(proxies.http()) + .https(proxies.https()) + .no(proxies.no()) + .build(); + move |url| { + url.as_str() + .parse::() + .is_ok_and(|uri| matcher.intercept(&uri).is_some()) + } } -impl EnvironmentProxies { - pub fn from_environment(env: &impl Lookup) -> Self { - Self::resolve(env, cfg!(windows)) - } +pub(crate) fn reqwest_proxies(proxies: &EnvironmentProxies) -> Vec { + let no_proxy = reqwest::NoProxy::from_string(proxies.no()); + [ + reqwest::Proxy::http(proxies.http()), + reqwest::Proxy::https(proxies.https()), + reqwest::Proxy::all(proxies.all()), + ] + .into_iter() + .filter_map(Result::ok) + .map(|proxy| proxy.no_proxy(no_proxy.clone())) + .collect() +} - fn resolve(env: &impl Lookup, names_ignore_case: bool) -> Self { - let lowercase_first = |upper: Option<&str>, lower: &str| { - env.get(lower) - .or_else(|| upper.and_then(|name| env.truthy(name))) - .unwrap_or_default() - }; - let is_cgi = env.get("REQUEST_METHOD").is_some(); - Self { - all: lowercase_first(Some("ALL_PROXY"), "all_proxy"), - http: if is_cgi && names_ignore_case { - String::new() - } else { - lowercase_first((!is_cgi).then_some("HTTP_PROXY"), "http_proxy") - }, - https: lowercase_first(Some("HTTPS_PROXY"), "https_proxy"), - no: lowercase_first(Some("NO_PROXY"), "no_proxy"), - } - } - - pub(crate) fn matcher(&self) -> impl Fn(&reqwest::Url) -> bool + Send + Sync + use<> { - let matcher = Matcher::builder() - .all(self.all.clone()) - .http(self.http.clone()) - .https(self.https.clone()) - .no(self.no.clone()) - .build(); - move |url| { - url.as_str() - .parse::() - .is_ok_and(|uri| matcher.intercept(&uri).is_some()) - } - } - - pub(crate) fn reqwest_proxies(&self) -> Vec { - let no_proxy = reqwest::NoProxy::from_string(&self.no); - [ - reqwest::Proxy::http(self.http.as_str()), - reqwest::Proxy::https(self.https.as_str()), - reqwest::Proxy::all(self.all.as_str()), - ] - .into_iter() - .filter_map(Result::ok) - .map(|proxy| proxy.no_proxy(no_proxy.clone())) - .collect() - } - #[cfg(feature = "mcp")] - pub(crate) fn mcp_proxies(&self) -> Vec { - let no_proxy = reqwest_mcp::NoProxy::from_string(&self.no); - [ - reqwest_mcp::Proxy::http(self.http.as_str()), - reqwest_mcp::Proxy::https(self.https.as_str()), - reqwest_mcp::Proxy::all(self.all.as_str()), - ] - .into_iter() - .filter_map(Result::ok) - .map(|proxy| proxy.no_proxy(no_proxy.clone())) - .collect() - } +#[cfg(feature = "mcp")] +pub(crate) fn mcp_proxies(proxies: &EnvironmentProxies) -> Vec { + let no_proxy = reqwest_mcp::NoProxy::from_string(proxies.no()); + [ + reqwest_mcp::Proxy::http(proxies.http()), + reqwest_mcp::Proxy::https(proxies.https()), + reqwest_mcp::Proxy::all(proxies.all()), + ] + .into_iter() + .filter_map(Result::ok) + .map(|proxy| proxy.no_proxy(no_proxy.clone())) + .collect() } #[cfg(test)] @@ -103,75 +69,18 @@ mod tests { #[case::all_covers_https(&[("ALL_PROXY", "http://proxy:3128")], "https://api.test/", true)] #[case::lowercase(&[("https_proxy", "http://proxy:3128")], "https://api.test/", true)] #[case::no_proxy_bypass(&[("HTTPS_PROXY", "http://proxy:3128"), ("NO_PROXY", "api.test")], "https://api.test/", false)] + #[case::cgi_still_proxies_https(&[("REQUEST_METHOD", "GET"), ("HTTPS_PROXY", "http://proxy:3128")], "https://api.test/", true)] fn proxies_follow_the_injected_environment( #[case] env: &'static [(&'static str, &'static str)], #[case] target: &str, #[case] expected: bool, ) { let proxies = EnvironmentProxies::from_environment(&env_of(env)); - assert_eq!(proxies.matcher()(&url(target)), expected); - } - - #[rstest] - #[case::lowercase_wins(&[("HTTPS_PROXY", "http://upper:3128"), ("https_proxy", "http://lower:3128")], &[("https_proxy", "http://lower:3128")])] - #[case::empty_uppercase_falls_through_to_lowercase(&[("HTTPS_PROXY", ""), ("https_proxy", "http://lower:3128")], &[("https_proxy", "http://lower:3128")])] - #[case::empty_uppercase_alone_is_unset(&[("HTTPS_PROXY", "")], &[])] - #[case::empty_lowercase_clears_the_uppercase_value(&[("HTTPS_PROXY", "http://upper:3128"), ("https_proxy", "")], &[])] - #[case::lowercase_no_proxy_wins(&[("NO_PROXY", "upper.test"), ("no_proxy", "lower.test")], &[("no_proxy", "lower.test")])] - #[case::cgi_forgets_the_client_settable_http_proxy(&[("REQUEST_METHOD", "GET"), ("HTTP_PROXY", "http://attacker:3128")], &[])] - #[case::cgi_keeps_lowercase_http_proxy(&[("REQUEST_METHOD", "GET"), ("HTTP_PROXY", "http://attacker:3128"), ("http_proxy", "http://lower:3128")], &[("http_proxy", "http://lower:3128")])] - #[case::cgi_keeps_every_other_variable(&[("REQUEST_METHOD", "GET"), ("HTTPS_PROXY", "http://proxy:3128"), ("ALL_PROXY", "http://all:3128"), ("NO_PROXY", "internal.test")], &[("HTTPS_PROXY", "http://proxy:3128"), ("ALL_PROXY", "http://all:3128"), ("NO_PROXY", "internal.test")])] - fn variables_resolve_like_urllib_getproxies_environment( - #[case] env: &'static [(&'static str, &'static str)], - #[case] equivalent: &'static [(&'static str, &'static str)], - ) { - assert_eq!( - EnvironmentProxies::from_environment(&env_of(env)), - EnvironmentProxies::from_environment(&env_of(equivalent)) - ); - } - - #[test] - fn cgi_drops_http_proxy_entirely_where_variable_names_ignore_case() { - let windows_env = |name: &str| match name.to_ascii_uppercase().as_str() { - "REQUEST_METHOD" => Some("GET".to_string()), - "HTTP_PROXY" => Some("http://attacker:3128".to_string()), - "HTTPS_PROXY" => Some("http://proxy:3128".to_string()), - _ => None, - }; - let proxies = EnvironmentProxies::resolve(&windows_env, true); - assert!(!proxies.matcher()(&url("http://api.test/"))); - assert!(proxies.matcher()(&url("https://api.test/"))); - assert!(EnvironmentProxies::resolve(&windows_env, false).matcher()( - &url("http://api.test/") - )); - } - - #[test] - fn a_cgi_request_still_proxies_https_through_the_configured_proxy() { - let proxies = EnvironmentProxies::from_environment(&env_of(&[ - ("REQUEST_METHOD", "GET"), - ("HTTPS_PROXY", "http://proxy:3128"), - ])); - assert!(proxies.matcher()(&url("https://api.test/"))); - } - - #[test] - fn debug_output_hides_proxy_credentials_but_shows_which_variables_are_set() { - let proxies = EnvironmentProxies::from_environment(&env_of(&[ - ("HTTPS_PROXY", "http://operator:hunter2@proxy.corp:3128"), - ("NO_PROXY", "internal.test"), - ])); - let debug = format!("{proxies:?}"); - assert!(!debug.contains("hunter2") && !debug.contains("operator")); - assert!(debug.contains("internal.test")); - assert_ne!(debug, format!("{:?}", EnvironmentProxies::default())); + assert_eq!(matcher(&proxies)(&url(target)), expected); } #[test] fn an_empty_environment_proxies_nothing() { - let proxies = EnvironmentProxies::from_environment(&env_of(&[])); - assert_eq!(proxies, EnvironmentProxies::default()); - assert!(proxies.reqwest_proxies().is_empty()); + assert!(reqwest_proxies(&EnvironmentProxies::default()).is_empty()); } } diff --git a/litellm-rust/crates/http/src/tls.rs b/litellm-rust/crates/http/src/tls.rs index 0b1c44bf6fe..391babe643b 100644 --- a/litellm-rust/crates/http/src/tls.rs +++ b/litellm-rust/crates/http/src/tls.rs @@ -315,7 +315,9 @@ mod tests { use rustls::NamedGroup; use super::*; - use crate::{HttpSettings, Resolution}; + use litellm_http_types::HttpSettings; + + use crate::Resolution; fn config(settings: HttpSettings) -> HttpClientConfig { Resolution::from(&settings).config