From e361f7acaee7876e7bad58f7772e0474b9874033 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 23:00:07 +0000 Subject: [PATCH 01/15] feat(gateway): forward HTTP requests to the Python proxy --- litellm-rust/Cargo.lock | 1 + litellm-rust/crates/gateway/Cargo.toml | 1 + litellm-rust/crates/gateway/src/error.rs | 6 + litellm-rust/crates/gateway/src/forward.rs | 308 ++++++++++++++++++++ litellm-rust/crates/gateway/src/lib.rs | 1 + litellm-rust/crates/gateway/src/main.rs | 24 ++ litellm-rust/crates/gateway/src/settings.rs | 27 ++ 7 files changed, 368 insertions(+) create mode 100644 litellm-rust/crates/gateway/src/forward.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index f91deaaa401..f571a49b0c4 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3808,6 +3808,7 @@ dependencies = [ "envy", "futures-util", "http-body-util", + "hyper-util", "litellm-circuit-breaker", "litellm-config", "litellm-error", diff --git a/litellm-rust/crates/gateway/Cargo.toml b/litellm-rust/crates/gateway/Cargo.toml index 6eeaa43e013..e1b1e8d486c 100644 --- a/litellm-rust/crates/gateway/Cargo.toml +++ b/litellm-rust/crates/gateway/Cargo.toml @@ -10,6 +10,7 @@ litellm-secret-value.workspace = true axum.workspace = true envy = "0.4.2" http-body-util = "0.1" +hyper-util = { workspace = true, features = ["client-legacy", "http1", "tokio"] } litellm-inference.workspace = true litellm-gateway-inference.workspace = true litellm-gateway-mcp.workspace = true diff --git a/litellm-rust/crates/gateway/src/error.rs b/litellm-rust/crates/gateway/src/error.rs index e2f646c5448..6911ce17df9 100644 --- a/litellm-rust/crates/gateway/src/error.rs +++ b/litellm-rust/crates/gateway/src/error.rs @@ -1,5 +1,11 @@ #[derive(Debug, thiserror::Error)] pub enum Error { + #[error("invalid forwarding upstream URI")] + ForwardUri(#[source] axum::http::uri::InvalidUri), + #[error( + "LITELLM_FORWARD_UPSTREAM must be a loopback IPv4 http origin without a path, query or credentials" + )] + ForwardUpstream, #[error(transparent)] Http(#[from] litellm_http::Error), #[error("environment_variables values must be strings, numbers or booleans")] diff --git a/litellm-rust/crates/gateway/src/forward.rs b/litellm-rust/crates/gateway/src/forward.rs new file mode 100644 index 00000000000..c88536d8e34 --- /dev/null +++ b/litellm-rust/crates/gateway/src/forward.rs @@ -0,0 +1,308 @@ +use std::net::{Ipv4Addr, SocketAddr}; + +use axum::{ + Router, + body::Body, + extract::{ConnectInfo, DefaultBodyLimit, Request, State}, + http::{HeaderMap, HeaderValue, StatusCode, Uri, header}, + response::Response, +}; +use hyper_util::{ + client::legacy::{Client, connect::HttpConnector}, + rt::TokioExecutor, +}; + +use crate::Error; + +#[derive(Clone)] +pub struct Forward { + upstream: Uri, + client: Client, +} + +impl Forward { + pub fn new(upstream: &str) -> Result { + let upstream: Uri = upstream.parse().map_err(Error::ForwardUri)?; + if upstream.scheme_str() != Some("http") + || !upstream + .host() + .and_then(|host| host.parse::().ok()) + .is_some_and(|ip| ip.is_loopback()) + || upstream + .authority() + .is_some_and(|authority| authority.as_str().contains('@')) + || upstream.path() != "/" + || upstream.query().is_some() + { + return Err(Error::ForwardUpstream); + } + let client = Client::builder(TokioExecutor::new()) + .retry_canceled_requests(false) + .build_http(); + Ok(Self { upstream, client }) + } + + pub fn router(self) -> Router { + Router::new() + .fallback(forward) + .layer(DefaultBodyLimit::disable()) + .with_state(self) + } +} + +async fn forward( + State(state): State, + ConnectInfo(peer): ConnectInfo, + mut request: Request, +) -> Result { + let mut uri = state.upstream.into_parts(); + uri.path_and_query = request.uri().path_and_query().cloned(); + *request.uri_mut() = Uri::from_parts(uri).map_err(|_| StatusCode::BAD_GATEWAY)?; + *request.version_mut() = axum::http::Version::HTTP_11; + let headers = request.headers_mut(); + strip_hop_headers(headers); + let forwarded = headers + .get_all("x-forwarded-for") + .iter() + .map(|value| value.as_bytes()) + .collect::>(); + let mut forwarded = forwarded.join(b", ".as_slice()); + if !forwarded.is_empty() { + forwarded.extend_from_slice(b", "); + } + forwarded.extend_from_slice(peer.ip().to_string().as_bytes()); + headers.insert( + "x-forwarded-for", + HeaderValue::from_bytes(&forwarded).map_err(|_| StatusCode::BAD_REQUEST)?, + ); + if !headers.contains_key("x-forwarded-proto") { + headers.insert("x-forwarded-proto", HeaderValue::from_static("http")); + } + let mut response = state + .client + .request(request) + .await + .map_err(|_| StatusCode::BAD_GATEWAY)?; + strip_hop_headers(response.headers_mut()); + Ok(response.map(Body::new)) +} + +fn strip_hop_headers(headers: &mut HeaderMap) { + let connection_headers: Vec<_> = headers + .get_all(header::CONNECTION) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .filter_map(|name| header::HeaderName::from_bytes(name.trim().as_bytes()).ok()) + .collect(); + for name in connection_headers { + headers.remove(name); + } + for name in [ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "transfer-encoding", + "upgrade", + "proxy-connection", + ] { + headers.remove(name); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{body::Bytes, http::Version}; + use http_body_util::BodyExt; + use rstest::rstest; + use std::{convert::Infallible, time::Duration}; + use tokio::{net::TcpListener, sync::mpsc}; + use tower::ServiceExt; + + #[rstest] + #[case::https("https://127.0.0.1:4000")] + #[case::public("http://192.0.2.1:4000")] + #[case::hostname("http://localhost:4000")] + #[case::ipv6("http://[::1]:4000")] + #[case::path("http://127.0.0.1:4000/api")] + #[case::query("http://127.0.0.1:4000/?x=y")] + #[case::credentials("http://user@127.0.0.1:4000")] + fn rejects_invalid_upstream(#[case] upstream: &str) { + assert!(Forward::new(upstream).is_err()); + } + + #[rstest] + #[case::standard("http://127.0.0.1:4000")] + #[case::loopback_range("http://127.0.0.2:4000/")] + fn accepts_loopback_origin(#[case] upstream: &str) { + assert!(Forward::new(upstream).is_ok()); + } + + #[rstest] + #[tokio::test] + async fn preserves_request_and_response_without_hop_headers() { + let upstream = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}", upstream.local_addr().unwrap()); + let task = tokio::spawn(async move { + axum::serve( + upstream, + Router::new().fallback(|request: Request| async { + assert_eq!(request.method(), "PATCH"); + assert_eq!(request.uri(), "/v1/messages?x=a%2Fb&x=c"); + assert_eq!(request.headers()["host"], "public.example"); + assert_eq!(request.headers()["authorization"], "Bearer test"); + assert_eq!( + request.headers()["x-forwarded-for"], + "192.0.2.1, 192.0.2.2, 198.51.100.5" + ); + assert_eq!(request.headers()["x-forwarded-proto"], "https"); + assert!(!request.headers().contains_key("x-remove")); + assert!(!request.headers().contains_key("proxy-authorization")); + assert_eq!( + request.into_body().collect().await.unwrap().to_bytes(), + "payload" + ); + Response::builder() + .status(StatusCode::UNAUTHORIZED) + .header("connection", "x-upstream-hop") + .header("x-upstream-hop", "remove") + .header("x-response", "keep") + .body(Body::from("python error")) + .unwrap() + }), + ) + .await + .unwrap(); + }); + let request = Request::builder() + .method("PATCH") + .version(Version::HTTP_2) + .uri("/v1/messages?x=a%2Fb&x=c") + .header("host", "public.example") + .header("authorization", "Bearer test") + .header("x-forwarded-for", "192.0.2.1") + .header("x-forwarded-for", "192.0.2.2") + .header("x-forwarded-proto", "https") + .header("connection", "x-remove, keep-alive") + .header("x-remove", "remove") + .header("proxy-authorization", "remove") + .extension(ConnectInfo( + "198.51.100.5:1234".parse::().unwrap(), + )) + .body(Body::from("payload")) + .unwrap(); + let response = Forward::new(&origin) + .unwrap() + .router() + .oneshot(request) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert_eq!(response.headers()["x-response"], "keep"); + assert!(!response.headers().contains_key("connection")); + assert!(!response.headers().contains_key("x-upstream-hop")); + assert_eq!( + response.into_body().collect().await.unwrap().to_bytes(), + "python error" + ); + task.abort(); + } + + #[rstest] + #[case::health("/health/liveliness")] + #[case::readiness("/health/readiness")] + #[case::messages("/v1/messages")] + #[tokio::test] + async fn unreachable_upstream_is_502(#[case] path: &str) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + drop(listener); + let response = Forward::new(&origin) + .unwrap() + .router() + .oneshot( + Request::builder() + .uri(path) + .extension(ConnectInfo( + "198.51.100.5:1234".parse::().unwrap(), + )) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + } + + #[rstest] + #[tokio::test] + async fn streams_upload_and_response_before_eof_without_body_limit() { + let upstream = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}", upstream.local_addr().unwrap()); + let upstream_task = tokio::spawn(async move { + axum::serve( + upstream, + Router::new().fallback(|request: Request| async { + assert_eq!(request.headers()["x-forwarded-for"], "127.0.0.1"); + assert_eq!(request.headers()["x-forwarded-proto"], "http"); + Response::new(request.into_body()) + }), + ) + .await + .unwrap(); + }); + let public = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let public_uri = format!("http://{}/chat/completions", public.local_addr().unwrap()); + let forward_task = tokio::spawn(async move { + axum::serve( + public, + Forward::new(&origin) + .unwrap() + .router() + .into_make_service_with_connect_info::(), + ) + .await + .unwrap(); + }); + let (sender, receiver) = mpsc::channel::>(1); + let stream = futures_util::stream::unfold(receiver, |mut receiver| async { + receiver.recv().await.map(|chunk| (chunk, receiver)) + }); + sender.send(Ok(Bytes::from_static(b"first"))).await.unwrap(); + let client: Client = + Client::builder(TokioExecutor::new()).build_http(); + let mut response = tokio::time::timeout( + Duration::from_secs(5), + client.request( + Request::post(public_uri) + .body(Body::from_stream(stream)) + .unwrap(), + ), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let frame = tokio::time::timeout(Duration::from_secs(5), response.body_mut().frame()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(frame.into_data().unwrap(), "first"); + let large = Bytes::from(vec![b'x'; 3 * 1024 * 1024]); + sender.send(Ok(large.clone())).await.unwrap(); + drop(sender); + let rest = tokio::time::timeout(Duration::from_secs(5), response.into_body().collect()) + .await + .unwrap() + .unwrap() + .to_bytes(); + assert_eq!(rest, large); + upstream_task.abort(); + forward_task.abort(); + } +} diff --git a/litellm-rust/crates/gateway/src/lib.rs b/litellm-rust/crates/gateway/src/lib.rs index 12a9bf9110a..8fa322628a7 100644 --- a/litellm-rust/crates/gateway/src/lib.rs +++ b/litellm-rust/crates/gateway/src/lib.rs @@ -1,5 +1,6 @@ mod auth; mod error; +pub mod forward; mod redis; mod secrets; pub use error::Error; diff --git a/litellm-rust/crates/gateway/src/main.rs b/litellm-rust/crates/gateway/src/main.rs index 47c1fe83f50..168a4966ef7 100644 --- a/litellm-rust/crates/gateway/src/main.rs +++ b/litellm-rust/crates/gateway/src/main.rs @@ -41,6 +41,30 @@ impl Sink for StderrSink { #[tokio::main] async fn main() -> Result<(), Box> { + if let Some(upstream) = std::env::var_os("LITELLM_FORWARD_UPSTREAM") { + let settings: settings::ForwardSettings = settings::from_iter(std::env::vars_os())?; + Logger::new(StderrSink { + level: settings.log_level(), + }) + .install_global()?; + let forward = litellm_gateway::forward::Forward::new( + &upstream + .into_string() + .map_err(|_| "invalid upstream encoding")?, + )?; + let listener = + tokio::net::TcpListener::bind((settings.host.as_str(), settings.port)).await?; + tracing::info!(address = %listener.local_addr()?, "forwarding gateway listening"); + axum::serve( + listener, + forward + .router() + .into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(shutdown_signal()) + .await?; + return Ok(()); + } let settings: Settings = settings::from_iter(std::env::vars_os())?; let level = settings.log_level(); Logger::new(StderrSink { level }).install_global()?; diff --git a/litellm-rust/crates/gateway/src/settings.rs b/litellm-rust/crates/gateway/src/settings.rs index 445071504ed..332c774b600 100644 --- a/litellm-rust/crates/gateway/src/settings.rs +++ b/litellm-rust/crates/gateway/src/settings.rs @@ -4,6 +4,33 @@ use litellm_secret_value::SecretValue; use litellm_tracing::Level; use serde::{Deserialize, de::DeserializeOwned}; +#[derive(Deserialize)] +#[serde(default)] +pub(super) struct ForwardSettings { + pub host: String, + pub port: u16, + rust_log: Option, +} + +impl Default for ForwardSettings { + fn default() -> Self { + Self { + host: "0.0.0.0".into(), + port: 4000, + rust_log: None, + } + } +} + +impl ForwardSettings { + pub fn log_level(&self) -> Level { + self.rust_log + .as_deref() + .and_then(|value| value.parse().ok()) + .unwrap_or(Level::INFO) + } +} + #[derive(Deserialize)] #[serde(default)] pub(super) struct Settings { From 7a581ae3f169633c90ad74d2ef199bb9ec1cac5e Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 06:16:04 +0900 Subject: [PATCH 02/15] fix(gateway): log the upstream error when forwarding returns 502 Co-Authored-By: Claude Opus 5.5 --- litellm-rust/crates/gateway/src/forward.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/litellm-rust/crates/gateway/src/forward.rs b/litellm-rust/crates/gateway/src/forward.rs index c88536d8e34..42e87260b26 100644 --- a/litellm-rust/crates/gateway/src/forward.rs +++ b/litellm-rust/crates/gateway/src/forward.rs @@ -82,7 +82,10 @@ async fn forward( .client .request(request) .await - .map_err(|_| StatusCode::BAD_GATEWAY)?; + .map_err(|error| { + tracing::warn!(%error, "python proxy unreachable"); + StatusCode::BAD_GATEWAY + })?; strip_hop_headers(response.headers_mut()); Ok(response.map(Body::new)) } From b99ea3a0aa9c2cc3fc091a833fc0da9673829c26 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 08:49:33 +0900 Subject: [PATCH 03/15] fix(gateway): pass forwarding headers through and carry the peer in x-litellm-rust-peer Appending to X-Forwarded-For made Rust look like one more proxy, so Python code that compares or counts X-Forwarded-For entries behaved differently once Rust was enabled. Rust now leaves every forwarding header as the client sent it and hands Python the TCP peer in a private header, replacing any client-sent copy. Co-Authored-By: Claude Opus 5.5 --- litellm-rust/crates/gateway/src/forward.rs | 67 +++++++++++++--------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/litellm-rust/crates/gateway/src/forward.rs b/litellm-rust/crates/gateway/src/forward.rs index 42e87260b26..72127f1980b 100644 --- a/litellm-rust/crates/gateway/src/forward.rs +++ b/litellm-rust/crates/gateway/src/forward.rs @@ -14,6 +14,8 @@ use hyper_util::{ use crate::Error; +pub const PEER_HEADER: &str = "x-litellm-rust-peer"; + #[derive(Clone)] pub struct Forward { upstream: Uri, @@ -61,35 +63,23 @@ async fn forward( *request.version_mut() = axum::http::Version::HTTP_11; let headers = request.headers_mut(); strip_hop_headers(headers); - let forwarded = headers - .get_all("x-forwarded-for") - .iter() - .map(|value| value.as_bytes()) - .collect::>(); - let mut forwarded = forwarded.join(b", ".as_slice()); - if !forwarded.is_empty() { - forwarded.extend_from_slice(b", "); - } - forwarded.extend_from_slice(peer.ip().to_string().as_bytes()); + // Forwarding headers pass through untouched so Python sees the request as if the client + // connected to it directly. Inserting replaces any client-sent copy: the peer is the one + // value a client must not choose. headers.insert( - "x-forwarded-for", - HeaderValue::from_bytes(&forwarded).map_err(|_| StatusCode::BAD_REQUEST)?, + PEER_HEADER, + HeaderValue::from_str(&peer.to_string()).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?, ); - if !headers.contains_key("x-forwarded-proto") { - headers.insert("x-forwarded-proto", HeaderValue::from_static("http")); - } - let mut response = state - .client - .request(request) - .await - .map_err(|error| { - tracing::warn!(%error, "python proxy unreachable"); - StatusCode::BAD_GATEWAY - })?; + let mut response = state.client.request(request).await.map_err(|error| { + tracing::warn!(%error, "python proxy unreachable"); + StatusCode::BAD_GATEWAY + })?; strip_hop_headers(response.headers_mut()); Ok(response.map(Body::new)) } +// Per-connection headers (RFC 9110 section 7.6.1) describe the client's connection to Rust, not +// Rust's connection to Python, so a proxy must drop them instead of forwarding them. fn strip_hop_headers(headers: &mut HeaderMap) { let connection_headers: Vec<_> = headers .get_all(header::CONNECTION) @@ -159,10 +149,23 @@ mod tests { assert_eq!(request.headers()["host"], "public.example"); assert_eq!(request.headers()["authorization"], "Bearer test"); assert_eq!( - request.headers()["x-forwarded-for"], - "192.0.2.1, 192.0.2.2, 198.51.100.5" + request + .headers() + .get_all("x-forwarded-for") + .iter() + .collect::>(), + ["192.0.2.1", "192.0.2.2"] ); assert_eq!(request.headers()["x-forwarded-proto"], "https"); + assert_eq!(request.headers()["forwarded"], "for=192.0.2.1;proto=https"); + assert_eq!( + request + .headers() + .get_all(PEER_HEADER) + .iter() + .collect::>(), + ["198.51.100.5:1234"] + ); assert!(!request.headers().contains_key("x-remove")); assert!(!request.headers().contains_key("proxy-authorization")); assert_eq!( @@ -190,6 +193,9 @@ mod tests { .header("x-forwarded-for", "192.0.2.1") .header("x-forwarded-for", "192.0.2.2") .header("x-forwarded-proto", "https") + .header("forwarded", "for=192.0.2.1;proto=https") + .header(PEER_HEADER, "10.1.2.3:1") + .header(PEER_HEADER, "10.1.2.4:1") .header("connection", "x-remove, keep-alive") .header("x-remove", "remove") .header("proxy-authorization", "remove") @@ -250,8 +256,15 @@ mod tests { axum::serve( upstream, Router::new().fallback(|request: Request| async { - assert_eq!(request.headers()["x-forwarded-for"], "127.0.0.1"); - assert_eq!(request.headers()["x-forwarded-proto"], "http"); + let headers = request.headers(); + assert!(!headers.contains_key("x-forwarded-for")); + assert!(!headers.contains_key("x-forwarded-proto")); + assert!( + headers[PEER_HEADER] + .to_str() + .unwrap() + .starts_with("127.0.0.1:") + ); Response::new(request.into_body()) }), ) From 1c27bd33f0fa20f1768b9a91e000e05816f21def Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 10:08:59 +0900 Subject: [PATCH 04/15] feat(gateway): relay WebSocket upgrades to the Python proxy Hop-by-hop stripping dropped Upgrade and Connection, so realtime and live WebSocket routes got 404 from Python instead of 101. Ported from #48087 without its x-forwarded-for append or handshake timeouts, since Python directly has neither. Open sockets get a five second drain on shutdown. Co-Authored-By: Claude Opus 5.5 --- litellm-rust/Cargo.lock | 3 + litellm-rust/crates/gateway/Cargo.toml | 6 +- .../src/{forward.rs => forward/mod.rs} | 55 +++- .../crates/gateway/src/forward/websocket.rs | 297 ++++++++++++++++++ litellm-rust/crates/gateway/src/main.rs | 9 +- 5 files changed, 356 insertions(+), 14 deletions(-) rename litellm-rust/crates/gateway/src/{forward.rs => forward/mod.rs} (87%) create mode 100644 litellm-rust/crates/gateway/src/forward/websocket.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index f571a49b0c4..ca39435c8dc 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3808,6 +3808,7 @@ dependencies = [ "envy", "futures-util", "http-body-util", + "hyper 1.10.1", "hyper-util", "litellm-circuit-breaker", "litellm-config", @@ -3830,6 +3831,7 @@ dependencies = [ "tempfile", "thiserror 2.0.19", "tokio", + "tokio-tungstenite", "tokio-util", "tower", "tower-sessions-moka-store", @@ -7571,6 +7573,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", "pin-project-lite", "tokio", ] diff --git a/litellm-rust/crates/gateway/Cargo.toml b/litellm-rust/crates/gateway/Cargo.toml index e1b1e8d486c..24a786f741d 100644 --- a/litellm-rust/crates/gateway/Cargo.toml +++ b/litellm-rust/crates/gateway/Cargo.toml @@ -10,11 +10,12 @@ litellm-secret-value.workspace = true axum.workspace = true envy = "0.4.2" http-body-util = "0.1" +hyper = { version = "1", features = ["http1"] } hyper-util = { workspace = true, features = ["client-legacy", "http1", "tokio"] } litellm-inference.workspace = true litellm-gateway-inference.workspace = true litellm-gateway-mcp.workspace = true -tokio-util = "0.7" +tokio-util = { version = "0.7", features = ["rt"] } thiserror.workspace = true futures-util.workspace = true litellm-gateway-auth.workspace = true @@ -30,7 +31,7 @@ litellm-tracing.workspace = true serde_json.workspace = true serde.workspace = true tracing.workspace = true -tokio = { workspace = true, features = ["signal"] } +tokio = { workspace = true, features = ["signal", "io-util"] } uuid.workspace = true tower-sessions-moka-store = "0.15.0" @@ -40,4 +41,5 @@ base64.workspace = true rstest.workspace = true tempfile.workspace = true tokio = { workspace = true, features = ["sync"] } +tokio-tungstenite.workspace = true tower = { version = "0.5", features = ["util"] } diff --git a/litellm-rust/crates/gateway/src/forward.rs b/litellm-rust/crates/gateway/src/forward/mod.rs similarity index 87% rename from litellm-rust/crates/gateway/src/forward.rs rename to litellm-rust/crates/gateway/src/forward/mod.rs index 72127f1980b..98e971b7a29 100644 --- a/litellm-rust/crates/gateway/src/forward.rs +++ b/litellm-rust/crates/gateway/src/forward/mod.rs @@ -4,15 +4,19 @@ use axum::{ Router, body::Body, extract::{ConnectInfo, DefaultBodyLimit, Request, State}, - http::{HeaderMap, HeaderValue, StatusCode, Uri, header}, + http::{HeaderMap, HeaderValue, StatusCode, Uri, Version, header}, response::Response, }; use hyper_util::{ client::legacy::{Client, connect::HttpConnector}, rt::TokioExecutor, }; +use tokio::net::TcpListener; use crate::Error; +use websocket::Upgrades; + +mod websocket; pub const PEER_HEADER: &str = "x-litellm-rust-peer"; @@ -20,6 +24,7 @@ pub const PEER_HEADER: &str = "x-litellm-rust-peer"; pub struct Forward { upstream: Uri, client: Client, + upgrades: Upgrades, } impl Forward { @@ -41,7 +46,11 @@ impl Forward { let client = Client::builder(TokioExecutor::new()) .retry_canceled_requests(false) .build_http(); - Ok(Self { upstream, client }) + Ok(Self { + upstream, + client, + upgrades: Upgrades::default(), + }) } pub fn router(self) -> Router { @@ -50,6 +59,32 @@ impl Forward { .layer(DefaultBodyLimit::disable()) .with_state(self) } + + /// Serves until `shutdown` resolves, then gives open WebSockets a bounded drain before closing them. + pub async fn serve( + self, + listener: TcpListener, + shutdown: impl Future + Send + 'static, + ) -> std::io::Result<()> { + let upgrades = self.upgrades.clone(); + let _close_on_drop = upgrades.close_on_drop(); + let draining = upgrades.clone(); + let result = axum::serve( + listener, + self.router() + .into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(async move { + shutdown.await; + draining.begin_shutdown(); + }) + .await; + if result.is_err() { + upgrades.close(); + } + upgrades.wait().await; + result + } } async fn forward( @@ -57,12 +92,16 @@ async fn forward( ConnectInfo(peer): ConnectInfo, mut request: Request, ) -> Result { + let downstream = state.upgrades.take_handshake(&mut request)?; let mut uri = state.upstream.into_parts(); uri.path_and_query = request.uri().path_and_query().cloned(); *request.uri_mut() = Uri::from_parts(uri).map_err(|_| StatusCode::BAD_GATEWAY)?; - *request.version_mut() = axum::http::Version::HTTP_11; + *request.version_mut() = Version::HTTP_11; let headers = request.headers_mut(); strip_hop_headers(headers); + if downstream.is_some() { + websocket::restore_handshake_headers(headers); + } // Forwarding headers pass through untouched so Python sees the request as if the client // connected to it directly. Inserting replaces any client-sent copy: the peer is the one // value a client must not choose. @@ -70,10 +109,18 @@ async fn forward( PEER_HEADER, HeaderValue::from_str(&peer.to_string()).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?, ); - let mut response = state.client.request(request).await.map_err(|error| { + let response = state.client.request(request); + let mut response = match downstream { + Some(_) => state.upgrades.until_closed(response).await?, + None => response.await, + } + .map_err(|error| { tracing::warn!(%error, "python proxy unreachable"); StatusCode::BAD_GATEWAY })?; + if response.status() == StatusCode::SWITCHING_PROTOCOLS { + return state.upgrades.switch(downstream, response).await; + } strip_hop_headers(response.headers_mut()); Ok(response.map(Body::new)) } diff --git a/litellm-rust/crates/gateway/src/forward/websocket.rs b/litellm-rust/crates/gateway/src/forward/websocket.rs new file mode 100644 index 00000000000..c6d0ad7fe3c --- /dev/null +++ b/litellm-rust/crates/gateway/src/forward/websocket.rs @@ -0,0 +1,297 @@ +use std::time::Duration; + +use axum::{ + body::Body, + extract::Request, + http::{HeaderMap, HeaderValue, Method, StatusCode, Version, header}, + response::Response, +}; +use hyper::{ + body::Incoming, + upgrade::{OnUpgrade, Upgraded}, +}; +use hyper_util::rt::TokioIo; +use tokio_util::{ + sync::{CancellationToken, DropGuard}, + task::TaskTracker, +}; + +const DRAIN_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(Clone, Default)] +pub(super) struct Upgrades { + tasks: TaskTracker, + draining: CancellationToken, + closed: CancellationToken, +} + +impl Upgrades { + pub fn begin_shutdown(&self) { + self.draining.cancel(); + let closed = self.closed.clone(); + tokio::spawn(async move { + tokio::time::sleep(DRAIN_TIMEOUT).await; + closed.cancel(); + }); + } + + pub fn take_handshake(&self, request: &mut Request) -> Result, StatusCode> { + if request.method() != Method::GET + || request.version() != Version::HTTP_11 + || !is_handshake(request.headers()) + { + return Ok(None); + } + if self.draining.is_cancelled() { + return Err(StatusCode::SERVICE_UNAVAILABLE); + } + Ok(Some(hyper::upgrade::on(request))) + } + + pub async fn until_closed(&self, future: impl Future) -> Result { + tokio::select! { + _ = self.closed.cancelled() => Err(StatusCode::SERVICE_UNAVAILABLE), + output = future => Ok(output), + } + } + + pub async fn switch( + &self, + downstream: Option, + mut response: Response, + ) -> Result { + let Some(downstream) = downstream else { + return Err(StatusCode::BAD_GATEWAY); + }; + if !is_handshake(response.headers()) { + return Err(StatusCode::BAD_GATEWAY); + } + let upstream = self + .until_closed(hyper::upgrade::on(&mut response)) + .await? + .map_err(|error| { + tracing::warn!(%error, "python proxy websocket upgrade failed"); + StatusCode::BAD_GATEWAY + })?; + super::strip_hop_headers(response.headers_mut()); + restore_handshake_headers(response.headers_mut()); + self.relay(downstream, upstream); + Ok(response.map(|_| Body::empty())) + } + + pub fn close(&self) { + self.closed.cancel(); + } + + pub fn close_on_drop(&self) -> DropGuard { + self.closed.clone().drop_guard() + } + + pub async fn wait(&self) { + self.tasks.close(); + self.tasks.wait().await; + } + + fn relay(&self, downstream: OnUpgrade, upstream: Upgraded) { + let closed = self.closed.clone(); + self.tasks.spawn(async move { + tokio::select! { + _ = closed.cancelled() => {} + _ = async { + let Ok(downstream) = downstream.await else { + tracing::debug!("websocket upgrade did not complete"); + return; + }; + let mut downstream = TokioIo::new(downstream); + let mut upstream = TokioIo::new(upstream); + if let Err(error) = tokio::io::copy_bidirectional(&mut downstream, &mut upstream).await { + tracing::debug!(%error, "websocket relay disconnected"); + } + } => {} + } + }); + } +} + +fn is_handshake(headers: &HeaderMap) -> bool { + headers + .get(header::UPGRADE) + .is_some_and(|value| value.as_bytes().eq_ignore_ascii_case(b"websocket")) + && headers + .get_all(header::CONNECTION) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .any(|token| token.trim().eq_ignore_ascii_case("upgrade")) +} + +// Hop-by-hop stripping drops the handshake's Connection and Upgrade, so each hop gets them back. +pub(super) fn restore_handshake_headers(headers: &mut HeaderMap) { + headers.insert(header::CONNECTION, HeaderValue::from_static("upgrade")); + headers.insert(header::UPGRADE, HeaderValue::from_static("websocket")); +} + +#[cfg(test)] +mod tests { + use super::super::{Forward, PEER_HEADER}; + use axum::{Router, http::HeaderValue, http::StatusCode, response::IntoResponse}; + use futures_util::{SinkExt, StreamExt}; + use std::net::SocketAddr; + use std::time::{Duration, Instant}; + use tokio::net::TcpListener; + use tokio::{net::TcpStream, sync::oneshot}; + use tokio_tungstenite::{ + accept_hdr_async, client_async, + tungstenite::{ + Message, + client::IntoClientRequest, + handshake::server::{ErrorResponse, Request as Handshake, Response as Accepted}, + }, + }; + + async fn gateway( + upstream: impl FnOnce(TcpListener) -> tokio::task::JoinHandle<()>, + ) -> (SocketAddr, oneshot::Sender<()>, tokio::task::JoinHandle<()>) { + let python = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}", python.local_addr().unwrap()); + upstream(python); + let public = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = public.local_addr().unwrap(); + let (stop, stopped) = oneshot::channel::<()>(); + let server = tokio::spawn(async move { + Forward::new(&origin) + .unwrap() + .serve(public, async { + stopped.await.ok(); + }) + .await + .unwrap(); + }); + (address, stop, server) + } + + async fn connect( + address: SocketAddr, + ) -> Result< + tokio_tungstenite::WebSocketStream, + Box, + > { + let mut request = format!("ws://{address}/v1/realtime?model=m") + .into_client_request() + .unwrap(); + request + .headers_mut() + .insert("authorization", HeaderValue::from_static("Bearer test")); + request.headers_mut().insert( + "sec-websocket-protocol", + HeaderValue::from_static("realtime"), + ); + let stream = TcpStream::connect(address).await.unwrap(); + let (socket, response) = client_async(request, stream).await.map_err(Box::new)?; + assert_eq!(response.headers()["sec-websocket-protocol"], "realtime"); + Ok(socket) + } + + #[expect( + clippy::result_large_err, + reason = "tungstenite's handshake callback returns ErrorResponse" + )] + fn echo(listener: TcpListener) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.unwrap(); + tokio::spawn(async move { + let callback = |request: &Handshake, mut response: Accepted| { + assert_eq!(request.uri(), "/v1/realtime?model=m"); + assert_eq!(request.headers()["authorization"], "Bearer test"); + assert!(request.headers().contains_key(PEER_HEADER)); + response.headers_mut().insert( + "sec-websocket-protocol", + HeaderValue::from_static("realtime"), + ); + Ok::<_, ErrorResponse>(response) + }; + let mut socket = accept_hdr_async(stream, callback).await.unwrap(); + socket.send(Message::text("greeting")).await.unwrap(); + while let Some(Ok(message)) = socket.next().await { + if message.is_close() { + break; + } + socket.send(message).await.unwrap(); + } + }); + } + }) + } + + #[tokio::test] + async fn relays_messages_both_ways() { + let (address, stop, server) = gateway(echo).await; + let mut socket = connect(address).await.unwrap(); + assert_eq!( + socket.next().await.unwrap().unwrap(), + Message::text("greeting") + ); + socket.send(Message::text("hello")).await.unwrap(); + assert_eq!( + socket.next().await.unwrap().unwrap(), + Message::text("hello") + ); + socket.send(Message::binary(vec![0, 1, 2])).await.unwrap(); + assert_eq!( + socket.next().await.unwrap().unwrap(), + Message::binary(vec![0, 1, 2]) + ); + socket.close(None).await.unwrap(); + while socket.next().await.is_some() {} + drop(socket); + stop.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(4), server) + .await + .unwrap() + .unwrap(); + } + + #[tokio::test] + async fn passes_upstream_rejection_through() { + let (address, _stop, _server) = gateway(|listener| { + tokio::spawn(async move { + axum::serve( + listener, + Router::new().fallback(|| async { + (StatusCode::FORBIDDEN, "origin not allowed").into_response() + }), + ) + .await + .unwrap(); + }) + }) + .await; + let Err(error) = connect(address).await else { + panic!("expected the upstream rejection"); + }; + let tokio_tungstenite::tungstenite::Error::Http(response) = *error else { + panic!("expected the upstream rejection"); + }; + assert_eq!(response.status(), StatusCode::FORBIDDEN); + assert_eq!(response.body().as_deref(), Some(&b"origin not allowed"[..])); + } + + #[tokio::test] + async fn shutdown_closes_open_sockets_after_drain() { + let (address, stop, server) = gateway(echo).await; + let mut socket = connect(address).await.unwrap(); + assert_eq!( + socket.next().await.unwrap().unwrap(), + Message::text("greeting") + ); + let started = Instant::now(); + stop.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(10), server) + .await + .unwrap() + .unwrap(); + assert!(started.elapsed() >= Duration::from_secs(4)); + assert!(!matches!(socket.next().await, Some(Ok(Message::Text(_))))); + } +} diff --git a/litellm-rust/crates/gateway/src/main.rs b/litellm-rust/crates/gateway/src/main.rs index 168a4966ef7..078f5c0c014 100644 --- a/litellm-rust/crates/gateway/src/main.rs +++ b/litellm-rust/crates/gateway/src/main.rs @@ -55,14 +55,7 @@ async fn main() -> Result<(), Box> { let listener = tokio::net::TcpListener::bind((settings.host.as_str(), settings.port)).await?; tracing::info!(address = %listener.local_addr()?, "forwarding gateway listening"); - axum::serve( - listener, - forward - .router() - .into_make_service_with_connect_info::(), - ) - .with_graceful_shutdown(shutdown_signal()) - .await?; + forward.serve(listener, shutdown_signal()).await?; return Ok(()); } let settings: Settings = settings::from_iter(std::env::vars_os())?; From ba1f6164b7014f28cad5a1ea35b9aa3cf3545965 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 11:23:47 +0900 Subject: [PATCH 05/15] refactor(gateway): move forwarding into a gateway-forward crate with protocol tests The forwarder is a reverse proxy, not a mounted endpoint, so it gets its own crate with a socket-level test harness and a property test that any generated request and response cross it unchanged The tests found two bugs, fixed here. Pooled connections to Python outlived uvicorn's keep-alive, so a request could land on a connection Python was closing and get a 502. They now retire at half of LITELLM_FORWARD_KEEPALIVE_TIMEOUT (default 5s, uvicorn's default). A GET with a chunked body reached Python empty, because hyper's client drops an unknown-length GET body unless transfer-encoding is set Co-Authored-By: Claude Opus 5.5 --- .../.agents/skills/rust-gateway/SKILL.md | 2 +- litellm-rust/Cargo.lock | 24 +- litellm-rust/Cargo.toml | 1 + litellm-rust/crates/gateway-forward/AGENTS.md | 15 + .../crates/gateway-forward/Cargo.toml | 26 ++ .../crates/gateway-forward/src/error.rs | 9 + .../crates/gateway-forward/src/forward.rs | 137 +++++++ .../crates/gateway-forward/src/hop.rs | 35 ++ .../crates/gateway-forward/src/lib.rs | 7 + .../crates/gateway-forward/src/websocket.rs | 126 ++++++ .../gateway-forward/tests/connection.rs | 145 +++++++ .../crates/gateway-forward/tests/http.rs | 337 ++++++++++++++++ .../crates/gateway-forward/tests/roundtrip.rs | 203 ++++++++++ .../gateway-forward/tests/support/mod.rs | 299 ++++++++++++++ .../crates/gateway-forward/tests/upstream.rs | 23 ++ .../crates/gateway-forward/tests/websocket.rs | 186 +++++++++ litellm-rust/crates/gateway/Cargo.toml | 8 +- litellm-rust/crates/gateway/src/error.rs | 6 - .../crates/gateway/src/forward/mod.rs | 371 ------------------ .../crates/gateway/src/forward/websocket.rs | 297 -------------- litellm-rust/crates/gateway/src/lib.rs | 1 - litellm-rust/crates/gateway/src/main.rs | 3 +- litellm-rust/crates/gateway/src/settings.rs | 8 +- 23 files changed, 1583 insertions(+), 686 deletions(-) create mode 100644 litellm-rust/crates/gateway-forward/AGENTS.md create mode 100644 litellm-rust/crates/gateway-forward/Cargo.toml create mode 100644 litellm-rust/crates/gateway-forward/src/error.rs create mode 100644 litellm-rust/crates/gateway-forward/src/forward.rs create mode 100644 litellm-rust/crates/gateway-forward/src/hop.rs create mode 100644 litellm-rust/crates/gateway-forward/src/lib.rs create mode 100644 litellm-rust/crates/gateway-forward/src/websocket.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/connection.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/http.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/roundtrip.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/support/mod.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/upstream.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/websocket.rs delete mode 100644 litellm-rust/crates/gateway/src/forward/mod.rs delete mode 100644 litellm-rust/crates/gateway/src/forward/websocket.rs diff --git a/litellm-rust/.agents/skills/rust-gateway/SKILL.md b/litellm-rust/.agents/skills/rust-gateway/SKILL.md index 4fee0b54a68..923c9bf5ab6 100644 --- a/litellm-rust/.agents/skills/rust-gateway/SKILL.md +++ b/litellm-rust/.agents/skills/rust-gateway/SKILL.md @@ -7,7 +7,7 @@ metadata: # Rust gateway -`crates/gateway` mounts endpoint routers and serves the listener. `crates/gateway-inference`, `gateway-mcp` and `gateway-ui` own their endpoints, `gateway-auth` is the inbound authentication middleware and `gateway-management` holds management logic that has no router yet. A crate's `AGENTS.md`, where it has one, says what belongs in it. Read the reference for the part you are touching +`crates/gateway` mounts endpoint routers and serves the listener. `crates/gateway-inference`, `gateway-mcp` and `gateway-ui` own their endpoints, `gateway-auth` is the inbound authentication middleware and `gateway-management` holds management logic that has no router yet. `gateway-forward` is the reverse proxy that relays every request to the Python proxy and has no routes of its own. A crate's `AGENTS.md`, where it has one, says what belongs in it. Read the reference for the part you are touching - [surfaces](references/surfaces.md) - Each API surface mirrors the envelope the Python proxy returns there, so the shapes differ on purpose diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index ca39435c8dc..e9dcd822713 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3808,12 +3808,11 @@ dependencies = [ "envy", "futures-util", "http-body-util", - "hyper 1.10.1", - "hyper-util", "litellm-circuit-breaker", "litellm-config", "litellm-error", "litellm-gateway-auth", + "litellm-gateway-forward", "litellm-gateway-inference", "litellm-gateway-mcp", "litellm-gateway-ui", @@ -3831,7 +3830,6 @@ dependencies = [ "tempfile", "thiserror 2.0.19", "tokio", - "tokio-tungstenite", "tokio-util", "tower", "tower-sessions-moka-store", @@ -3861,6 +3859,26 @@ dependencies = [ "veil", ] +[[package]] +name = "litellm-gateway-forward" +version = "0.1.0" +dependencies = [ + "axum", + "bytes", + "futures-util", + "http-body-util", + "httparse", + "hyper 1.10.1", + "hyper-util", + "proptest", + "rstest", + "thiserror 2.0.19", + "tokio", + "tokio-tungstenite", + "tokio-util", + "tracing", +] + [[package]] name = "litellm-gateway-inference" version = "0.1.0" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 75cac2a5d63..ba2710dcb9d 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -25,6 +25,7 @@ litellm-inference-ocr = { path = "crates/inference-ocr" } litellm-inference-testing = { path = "crates/inference-testing" } litellm-gateway-mcp = { path = "crates/gateway-mcp" } litellm-gateway = { path = "crates/gateway" } +litellm-gateway-forward = { path = "crates/gateway-forward" } litellm-gateway-inference = { path = "crates/gateway-inference" } litellm-gateway-auth = { path = "crates/gateway-auth" } litellm-gateway-management = { path = "crates/gateway-management" } diff --git a/litellm-rust/crates/gateway-forward/AGENTS.md b/litellm-rust/crates/gateway-forward/AGENTS.md new file mode 100644 index 00000000000..5ef76f808f4 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/AGENTS.md @@ -0,0 +1,15 @@ +# Forwarding gateway + +`Forward` is a reverse proxy in front of the Python proxy on loopback. Python must see each request as if the client had connected to it directly, because every Python route (inference, MCP, pass-through, UI, WebSocket) is served behind it without Rust knowing which one it is + +- Never branch on a route or path. A behavior one endpoint needs is a protocol behavior every endpoint gets +- Forward the method, request target, headers, and body byte for byte. Only drop hop-by-hop headers (RFC 9110 section 7.6.1) and set `x-litellm-rust-peer`, which replaces any client copy +- Stream both bodies. Never buffer, cap, or time out a body, since uploads and provider streams have no upper bound +- A client disconnect closes the upstream connection at every stage, because that is how Python learns to stop a provider stream +- An upstream that dies mid-response fails the client body. Never end it cleanly, or SSE clients treat a truncated stream as complete +- Pooled upstream connections are retired well before Python's keep-alive timeout, so a request never lands on a connection Python is closing +- WebSocket is the one protocol switch Python serves. Upgrades are relayed as raw bytes after the 101 and drained on shutdown + +## Tests + +Test through real sockets with `tests/support`, never `oneshot`. `Gateway` runs the forwarder on a listener like production, `recording` is a well-behaved upstream that captures what Python would receive, and `raw_upstream` and `Raw` drive bytes by hand for framing, timing and disconnects a real server would not produce. `tests/roundtrip.rs` checks the forwarding invariant for generated requests and responses. Add a protocol edge case there or as a focused test next to its neighbors diff --git a/litellm-rust/crates/gateway-forward/Cargo.toml b/litellm-rust/crates/gateway-forward/Cargo.toml new file mode 100644 index 00000000000..e1c8fe2c5de --- /dev/null +++ b/litellm-rust/crates/gateway-forward/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "litellm-gateway-forward" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +axum.workspace = true +hyper = { version = "1", features = ["http1"] } +hyper-util = { workspace = true, features = ["client-legacy", "http1", "tokio"] } +thiserror.workspace = true +tokio = { workspace = true, features = ["io-util"] } +tokio-util = { version = "0.7", features = ["rt"] } +tracing.workspace = true + +[dev-dependencies] +bytes.workspace = true +futures-util.workspace = true +http-body-util = "0.1" +httparse = "1" +proptest.workspace = true +rstest.workspace = true +tokio = { workspace = true, features = ["io-util", "sync"] } +tokio-tungstenite.workspace = true diff --git a/litellm-rust/crates/gateway-forward/src/error.rs b/litellm-rust/crates/gateway-forward/src/error.rs new file mode 100644 index 00000000000..abbe415b39a --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/error.rs @@ -0,0 +1,9 @@ +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("invalid forwarding upstream URI")] + UpstreamUri(#[source] axum::http::uri::InvalidUri), + #[error( + "LITELLM_FORWARD_UPSTREAM must be a loopback IPv4 http origin without a path, query or credentials" + )] + Upstream, +} diff --git a/litellm-rust/crates/gateway-forward/src/forward.rs b/litellm-rust/crates/gateway-forward/src/forward.rs new file mode 100644 index 00000000000..ceedfeb231b --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/forward.rs @@ -0,0 +1,137 @@ +use std::{ + net::{Ipv4Addr, SocketAddr}, + time::Duration, +}; + +use axum::{ + Router, + body::{Body, HttpBody}, + extract::{ConnectInfo, DefaultBodyLimit, Request, State}, + http::{HeaderValue, StatusCode, Uri, Version, header}, + response::Response, +}; +use hyper_util::{ + client::legacy::{Client, connect::HttpConnector}, + rt::{TokioExecutor, TokioTimer}, +}; +use tokio::net::TcpListener; + +use crate::{Error, hop, websocket::Upgrades}; + +pub const PEER_HEADER: &str = "x-litellm-rust-peer"; + +#[derive(Clone)] +pub struct Forward { + upstream: Uri, + client: Client, + upgrades: Upgrades, +} + +impl Forward { + /// `upstream_keepalive` is how long the upstream keeps an idle connection open. Pooled + /// connections are dropped well before that, so a request never lands on one it is closing. + pub fn new(upstream: &str, upstream_keepalive: Duration) -> Result { + let upstream: Uri = upstream.parse().map_err(Error::UpstreamUri)?; + if upstream.scheme_str() != Some("http") + || !upstream + .host() + .and_then(|host| host.parse::().ok()) + .is_some_and(|ip| ip.is_loopback()) + || upstream + .authority() + .is_some_and(|authority| authority.as_str().contains('@')) + || upstream.path() != "/" + || upstream.query().is_some() + { + return Err(Error::Upstream); + } + let client = Client::builder(TokioExecutor::new()) + .pool_timer(TokioTimer::new()) + .pool_idle_timeout(upstream_keepalive / 2) + .build_http(); + Ok(Self { + upstream, + client, + upgrades: Upgrades::default(), + }) + } + + pub fn router(self) -> Router { + Router::new() + .fallback(forward) + .layer(DefaultBodyLimit::disable()) + .with_state(self) + } + + /// Serves until `shutdown` resolves, then gives open WebSockets a bounded drain before closing them. + pub async fn serve( + self, + listener: TcpListener, + shutdown: impl Future + Send + 'static, + ) -> std::io::Result<()> { + let upgrades = self.upgrades.clone(); + let _close_on_drop = upgrades.close_on_drop(); + let draining = upgrades.clone(); + let result = axum::serve( + listener, + self.router() + .into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(async move { + shutdown.await; + draining.begin_shutdown(); + }) + .await; + if result.is_err() { + upgrades.close(); + } + upgrades.wait().await; + result + } +} + +async fn forward( + State(state): State, + ConnectInfo(peer): ConnectInfo, + mut request: Request, +) -> Result { + let downstream = state.upgrades.take_handshake(&mut request)?; + let mut uri = state.upstream.into_parts(); + uri.path_and_query = request.uri().path_and_query().cloned(); + *request.uri_mut() = Uri::from_parts(uri).map_err(|_| StatusCode::BAD_GATEWAY)?; + *request.version_mut() = Version::HTTP_11; + let chunked = request.body().size_hint().exact().is_none(); + let headers = request.headers_mut(); + hop::strip(headers); + if chunked { + // hyper sends no body for a GET with an unknown length unless told it is chunked. + headers.insert( + header::TRANSFER_ENCODING, + HeaderValue::from_static("chunked"), + ); + } + if downstream.is_some() { + hop::restore_websocket_handshake(headers); + } + // Forwarding headers pass through untouched so Python sees the request as if the client + // connected to it directly. Inserting replaces any client-sent copy: the peer is the one + // value a client must not choose. + headers.insert( + PEER_HEADER, + HeaderValue::from_str(&peer.to_string()).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?, + ); + let response = state.client.request(request); + let mut response = match downstream { + Some(_) => state.upgrades.until_closed(response).await?, + None => response.await, + } + .map_err(|error| { + tracing::warn!(%error, "python proxy unreachable"); + StatusCode::BAD_GATEWAY + })?; + if response.status() == StatusCode::SWITCHING_PROTOCOLS { + return state.upgrades.switch(downstream, response).await; + } + hop::strip(response.headers_mut()); + Ok(response.map(Body::new)) +} diff --git a/litellm-rust/crates/gateway-forward/src/hop.rs b/litellm-rust/crates/gateway-forward/src/hop.rs new file mode 100644 index 00000000000..e572940bab4 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/hop.rs @@ -0,0 +1,35 @@ +use axum::http::{HeaderMap, HeaderValue, header}; + +// Per-connection headers (RFC 9110 section 7.6.1) describe the client's connection to Rust, not +// Rust's connection to Python, so a proxy must drop them instead of forwarding them. +pub(crate) fn strip(headers: &mut HeaderMap) { + let connection_headers: Vec<_> = headers + .get_all(header::CONNECTION) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .filter_map(|name| header::HeaderName::from_bytes(name.trim().as_bytes()).ok()) + .collect(); + for name in connection_headers { + headers.remove(name); + } + for name in [ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "transfer-encoding", + "upgrade", + "proxy-connection", + ] { + headers.remove(name); + } +} + +// Stripping drops the handshake's Connection and Upgrade, so each hop gets them back. +pub(crate) fn restore_websocket_handshake(headers: &mut HeaderMap) { + headers.insert(header::CONNECTION, HeaderValue::from_static("upgrade")); + headers.insert(header::UPGRADE, HeaderValue::from_static("websocket")); +} diff --git a/litellm-rust/crates/gateway-forward/src/lib.rs b/litellm-rust/crates/gateway-forward/src/lib.rs new file mode 100644 index 00000000000..606c42200e5 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/lib.rs @@ -0,0 +1,7 @@ +mod error; +mod forward; +mod hop; +mod websocket; + +pub use error::Error; +pub use forward::{Forward, PEER_HEADER}; diff --git a/litellm-rust/crates/gateway-forward/src/websocket.rs b/litellm-rust/crates/gateway-forward/src/websocket.rs new file mode 100644 index 00000000000..172ebc2da2f --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/websocket.rs @@ -0,0 +1,126 @@ +use std::time::Duration; + +use axum::{ + body::Body, + extract::Request, + http::{HeaderMap, Method, StatusCode, Version, header}, + response::Response, +}; +use hyper::{ + body::Incoming, + upgrade::{OnUpgrade, Upgraded}, +}; +use hyper_util::rt::TokioIo; +use tokio_util::{ + sync::{CancellationToken, DropGuard}, + task::TaskTracker, +}; + +const DRAIN_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(Clone, Default)] +pub(crate) struct Upgrades { + tasks: TaskTracker, + draining: CancellationToken, + closed: CancellationToken, +} + +impl Upgrades { + pub fn begin_shutdown(&self) { + self.draining.cancel(); + let closed = self.closed.clone(); + tokio::spawn(async move { + tokio::time::sleep(DRAIN_TIMEOUT).await; + closed.cancel(); + }); + } + + pub fn take_handshake(&self, request: &mut Request) -> Result, StatusCode> { + if request.method() != Method::GET + || request.version() != Version::HTTP_11 + || !is_handshake(request.headers()) + { + return Ok(None); + } + if self.draining.is_cancelled() { + return Err(StatusCode::SERVICE_UNAVAILABLE); + } + Ok(Some(hyper::upgrade::on(request))) + } + + pub async fn until_closed(&self, future: impl Future) -> Result { + tokio::select! { + _ = self.closed.cancelled() => Err(StatusCode::SERVICE_UNAVAILABLE), + output = future => Ok(output), + } + } + + pub async fn switch( + &self, + downstream: Option, + mut response: Response, + ) -> Result { + let Some(downstream) = downstream else { + return Err(StatusCode::BAD_GATEWAY); + }; + if !is_handshake(response.headers()) { + return Err(StatusCode::BAD_GATEWAY); + } + let upstream = self + .until_closed(hyper::upgrade::on(&mut response)) + .await? + .map_err(|error| { + tracing::warn!(%error, "python proxy websocket upgrade failed"); + StatusCode::BAD_GATEWAY + })?; + crate::hop::strip(response.headers_mut()); + crate::hop::restore_websocket_handshake(response.headers_mut()); + self.relay(downstream, upstream); + Ok(response.map(|_| Body::empty())) + } + + pub fn close(&self) { + self.closed.cancel(); + } + + pub fn close_on_drop(&self) -> DropGuard { + self.closed.clone().drop_guard() + } + + pub async fn wait(&self) { + self.tasks.close(); + self.tasks.wait().await; + } + + fn relay(&self, downstream: OnUpgrade, upstream: Upgraded) { + let closed = self.closed.clone(); + self.tasks.spawn(async move { + tokio::select! { + _ = closed.cancelled() => {} + _ = async { + let Ok(downstream) = downstream.await else { + tracing::debug!("websocket upgrade did not complete"); + return; + }; + let mut downstream = TokioIo::new(downstream); + let mut upstream = TokioIo::new(upstream); + if let Err(error) = tokio::io::copy_bidirectional(&mut downstream, &mut upstream).await { + tracing::debug!(%error, "websocket relay disconnected"); + } + } => {} + } + }); + } +} + +fn is_handshake(headers: &HeaderMap) -> bool { + headers + .get(header::UPGRADE) + .is_some_and(|value| value.as_bytes().eq_ignore_ascii_case(b"websocket")) + && headers + .get_all(header::CONNECTION) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .any(|token| token.trim().eq_ignore_ascii_case("upgrade")) +} diff --git a/litellm-rust/crates/gateway-forward/tests/connection.rs b/litellm-rust/crates/gateway-forward/tests/connection.rs new file mode 100644 index 00000000000..dae3b08368b --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/connection.rs @@ -0,0 +1,145 @@ +mod support; + +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use axum::{body::Body, extract::Request, http::StatusCode}; +use http_body_util::BodyExt; +use rstest::rstest; +use support::{Gateway, Raw, client, raw_upstream, unreachable_upstream, within}; +use tokio::sync::{Notify, mpsc}; + +/// Python stops a provider stream when it sees the client go away, so a client disconnect must +/// close Rust's connection to Python at every stage of the exchange. +#[rstest] +#[case::waiting_for_headers( + "POST /v1/chat/completions HTTP/1.1\r\nhost: x\r\ncontent-length: 0\r\n\r\n", + false +)] +#[case::mid_stream( + "POST /v1/chat/completions HTTP/1.1\r\nhost: x\r\ncontent-length: 0\r\n\r\n", + true +)] +#[case::mid_upload( + "POST /v1/files HTTP/1.1\r\nhost: x\r\ncontent-length: 1000000\r\n\r\npartial", + false +)] +#[tokio::test] +async fn client_disconnect_closes_the_upstream_connection( + #[case] request: &'static str, + #[case] headers_sent: bool, +) { + let (event, mut events) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python: Raw| { + let event = event.clone(); + async move { + python.request_head().await.unwrap(); + if headers_sent { + python + .send("HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n6\r\ndata: \r\n") + .await; + } + event.send("received").unwrap(); + python.closed().await; + event.send("closed").unwrap(); + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(request).await; + assert_eq!(within(events.recv()).await, Some("received")); + if headers_sent { + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + } + drop(raw); + assert_eq!(within(events.recv()).await, Some("closed")); +} + +/// uvicorn closes a connection that sat idle past its keep-alive timeout. A request Rust sends on +/// that connection as uvicorn closes it is lost, so Rust must retire idle connections first. +#[rstest] +#[case::reused_while_fresh(Duration::from_millis(0))] +#[case::past_the_pool_timeout(Duration::from_millis(250))] +#[case::past_python_keepalive(Duration::from_millis(450))] +#[tokio::test] +async fn never_reuses_a_connection_python_is_closing_for_idleness(#[case] idle: Duration) { + let keepalive = Duration::from_millis(400); + let upstream = raw_upstream(move |mut python: Raw| async move { + let mut idle_since = None::; + while python.request_head().await.is_some() { + if idle_since.is_some_and(|since| since.elapsed() >= keepalive) { + return; + } + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\nok") + .await; + idle_since = Some(Instant::now()); + } + }) + .await; + let gateway = Gateway::with_keepalive(upstream, keepalive).await; + let client = client(); + let first = within(client.request(Request::get(gateway.url("/")).body(Body::empty()).unwrap())) + .await + .unwrap(); + first.into_body().collect().await.unwrap(); + tokio::time::sleep(idle).await; + let second = + within(client.request(Request::get(gateway.url("/")).body(Body::empty()).unwrap())) + .await + .unwrap(); + assert_eq!(second.status(), StatusCode::OK); +} + +#[rstest] +#[case::health("/health/liveliness")] +#[case::messages("/v1/messages")] +#[tokio::test] +async fn unreachable_python_is_a_502(#[case] path: &str) { + let gateway = Gateway::start(unreachable_upstream()).await; + let response = + within(client().request(Request::get(gateway.url(path)).body(Body::empty()).unwrap())) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); +} + +#[rstest] +#[tokio::test] +async fn shutdown_lets_an_in_flight_stream_finish() { + let release = Arc::new(Notify::new()); + let released = release.clone(); + let upstream = raw_upstream(move |mut python: Raw| { + let released = released.clone(); + async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n5\r\nfirst\r\n") + .await; + released.notified().await; + python.send("4\r\nlast\r\n0\r\n\r\n").await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let response = + within(client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap())) + .await + .unwrap(); + let server = gateway.shutdown(); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(!server.is_finished()); + release.notify_one(); + let body = within(response.into_body().collect()) + .await + .unwrap() + .to_bytes(); + assert_eq!(body, "firstlast"); + within(server).await.unwrap().unwrap(); +} diff --git a/litellm-rust/crates/gateway-forward/tests/http.rs b/litellm-rust/crates/gateway-forward/tests/http.rs new file mode 100644 index 00000000000..3635b795cf0 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/http.rs @@ -0,0 +1,337 @@ +mod support; + +use std::convert::Infallible; + +use axum::{ + Router, + body::{Body, Bytes}, + extract::Request, + http::{HeaderValue, StatusCode}, + response::Response, +}; +use http_body_util::BodyExt; +use litellm_gateway_forward::PEER_HEADER; +use rstest::rstest; +use support::{Gateway, client, raw_upstream, recording, serve, within}; +use tokio::sync::mpsc; + +fn ok() -> Response { + Response::new(Body::from("ok")) +} + +#[rstest] +#[tokio::test] +async fn forwards_the_request_as_the_client_sent_it_minus_hop_headers() { + let (upstream, mut received) = recording(|| { + Response::builder() + .status(StatusCode::UNAUTHORIZED) + .header("connection", "x-upstream-hop") + .header("x-upstream-hop", "remove") + .header("x-response", "keep") + .body(Body::from("python error")) + .unwrap() + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send( + "PATCH /v1/messages?x=a%2Fb&x=c HTTP/1.1\r\n\ + host: public.example\r\n\ + authorization: Bearer test\r\n\ + x-forwarded-for: 192.0.2.1\r\n\ + x-forwarded-for: 192.0.2.2\r\n\ + x-forwarded-proto: https\r\n\ + forwarded: for=192.0.2.1;proto=https\r\n\ + x-litellm-rust-peer: 10.1.2.3:1\r\n\ + connection: x-remove, keep-alive\r\n\ + x-remove: remove\r\n\ + proxy-authorization: remove\r\n\ + te: trailers\r\n\ + content-length: 7\r\n\r\npayload", + ) + .await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::UNAUTHORIZED); + assert_eq!(head.header("x-response"), Some(&b"keep"[..])); + assert_eq!(head.header("x-upstream-hop"), None); + let body = raw.body(head.content_length().unwrap()).await; + assert_eq!(body, b"python error"); + + let request = received.recv().await.unwrap(); + assert_eq!(request.method, "PATCH"); + assert_eq!(request.uri, "/v1/messages?x=a%2Fb&x=c"); + assert_eq!(request.headers["host"], "public.example"); + assert_eq!(request.headers["authorization"], "Bearer test"); + let forwarded_for: Vec<_> = request.headers.get_all("x-forwarded-for").iter().collect(); + assert_eq!(forwarded_for, ["192.0.2.1", "192.0.2.2"]); + assert_eq!(request.headers["x-forwarded-proto"], "https"); + assert_eq!(request.headers["forwarded"], "for=192.0.2.1;proto=https"); + let peers: Vec<_> = request.headers.get_all(PEER_HEADER).iter().collect(); + assert_eq!(peers, [raw.local_address().to_string().as_str()]); + assert!(!request.headers.contains_key("x-remove")); + assert!(!request.headers.contains_key("proxy-authorization")); + assert!(!request.headers.contains_key("te")); + assert!(!request.headers.contains_key("connection")); + assert_eq!(request.body, "payload"); +} + +#[rstest] +#[case::get("GET")] +#[case::post("POST")] +#[case::put("PUT")] +#[case::delete("DELETE")] +#[case::options("OPTIONS")] +#[case::extension("PURGE")] +#[tokio::test] +async fn forwards_every_method(#[case] method: &str) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let response = client() + .request( + Request::builder() + .method(method) + .uri(gateway.url("/mcp")) + .body(Body::from("body")) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let request = received.recv().await.unwrap(); + assert_eq!(request.method, method); + assert_eq!(request.body, "body"); +} + +#[rstest] +#[case::double_slash("//v1//models")] +#[case::encoded_slash("/v1/files/a%2Fb/content")] +#[case::encoded_query("/v1/models?q=%E2%9C%93&empty=&flag")] +#[case::utf8_bytes_escaped("/mcp/%F0%9F%A6%80")] +#[tokio::test] +async fn forwards_the_request_target_byte_for_byte(#[case] target: &str) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!("GET {target} HTTP/1.1\r\nhost: x\r\n\r\n")) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(received.recv().await.unwrap().uri, target); +} + +#[rstest] +#[tokio::test] +async fn absolute_form_targets_reach_python_as_origin_form() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send( + "GET http://elsewhere.example/v1/models?x=1 HTTP/1.1\r\nhost: elsewhere.example\r\n\r\n", + ) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + let request = received.recv().await.unwrap(); + assert_eq!(request.uri, "/v1/models?x=1"); +} + +#[rstest] +#[tokio::test] +async fn keeps_repeated_response_headers_and_opaque_header_bytes() { + let (upstream, mut received) = recording(|| { + Response::builder() + .header("set-cookie", "a=1; Path=/") + .header("set-cookie", "b=2; HttpOnly") + .header("x-opaque", HeaderValue::from_bytes(b"caf\xe9").unwrap()) + .body(Body::empty()) + .unwrap() + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(b"GET / HTTP/1.1\r\nhost: x\r\nx-opaque: caf\xe9\r\n\r\n") + .await; + let head = within(raw.response_head()).await.unwrap(); + let cookies: Vec<_> = head + .headers + .iter() + .filter(|(name, _)| name == "set-cookie") + .map(|(_, value)| value.as_slice()) + .collect(); + assert_eq!(cookies, [&b"a=1; Path=/"[..], b"b=2; HttpOnly"]); + assert_eq!(head.header("x-opaque"), Some(&b"caf\xe9"[..])); + assert_eq!( + received.recv().await.unwrap().headers["x-opaque"].as_bytes(), + b"caf\xe9" + ); +} + +#[rstest] +#[tokio::test] +async fn streams_upload_and_response_incrementally_without_a_body_limit() { + let upstream = serve( + Router::new().fallback(|request: Request| async { Response::new(request.into_body()) }), + ) + .await; + let gateway = Gateway::start(upstream).await; + let (sender, receiver) = mpsc::channel::>(1); + let stream = futures_util::stream::unfold(receiver, |mut receiver| async { + receiver.recv().await.map(|chunk| (chunk, receiver)) + }); + sender.send(Ok(Bytes::from_static(b"first"))).await.unwrap(); + let mut response = within( + client().request( + Request::post(gateway.url("/v1/files")) + .body(Body::from_stream(stream)) + .unwrap(), + ), + ) + .await + .unwrap(); + let frame = within(response.body_mut().frame()).await.unwrap().unwrap(); + assert_eq!(frame.into_data().unwrap(), "first"); + let large = Bytes::from(vec![b'x'; 8 * 1024 * 1024]); + sender.send(Ok(large.clone())).await.unwrap(); + drop(sender); + let rest = within(response.into_body().collect()) + .await + .unwrap() + .to_bytes(); + assert_eq!(rest, large); +} + +#[rstest] +#[tokio::test] +async fn answers_expect_continue_so_large_uploads_start() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("POST /v1/audio/transcriptions HTTP/1.1\r\nhost: x\r\nexpect: 100-continue\r\ncontent-length: 4\r\n\r\n") + .await; + let interim = within(raw.response_head()).await.unwrap(); + assert_eq!(interim.status, StatusCode::CONTINUE); + raw.send("data").await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(received.recv().await.unwrap().body, "data"); +} + +#[rstest] +#[tokio::test] +async fn answers_pipelined_requests_in_order() { + let upstream = serve( + Router::new().fallback(|request: Request| async move { request.uri().path().to_owned() }), + ) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET /first HTTP/1.1\r\nhost: x\r\n\r\nHEAD /second HTTP/1.1\r\nhost: x\r\n\r\nGET /third HTTP/1.1\r\nhost: x\r\n\r\n") + .await; + let first = within(raw.response_head()).await.unwrap(); + assert_eq!(raw.body(first.content_length().unwrap()).await, b"/first"); + let second = within(raw.response_head()).await.unwrap(); + assert_eq!(second.content_length(), Some(7)); + let third = within(raw.response_head()).await.unwrap(); + assert_eq!(raw.body(third.content_length().unwrap()).await, b"/third"); +} + +#[rstest] +#[case::no_content(StatusCode::NO_CONTENT)] +#[case::not_modified(StatusCode::NOT_MODIFIED)] +#[tokio::test] +async fn bodiless_statuses_keep_the_connection_reusable(#[case] status: StatusCode) { + let upstream = serve(Router::new().fallback(move |request: Request| async move { + match request.uri().path() { + "/next" => (StatusCode::OK, "next"), + _ => (status, ""), + } + })) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET /empty HTTP/1.1\r\nhost: x\r\n\r\nGET /next HTTP/1.1\r\nhost: x\r\n\r\n") + .await; + assert_eq!(within(raw.response_head()).await.unwrap().status, status); + let next = within(raw.response_head()).await.unwrap(); + assert_eq!(raw.body(next.content_length().unwrap()).await, b"next"); +} + +#[rstest] +#[tokio::test] +async fn relays_a_body_delimited_by_the_upstream_closing() { + let upstream = raw_upstream(|mut python| async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\nconnection: close\r\n\r\nuntil eof") + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let response = client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap()); + let body = within(response).await.unwrap().into_body().collect(); + assert_eq!(within(body).await.unwrap().to_bytes(), "until eof"); +} + +#[rstest] +#[tokio::test] +async fn an_upstream_dying_mid_stream_fails_the_client_body_instead_of_ending_it() { + let upstream = raw_upstream(|mut python| async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\n\r\n6\r\ndata: \r\n") + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let response = + within(client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap())) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert!(within(response.into_body().collect()).await.is_err()); +} + +#[rstest] +#[tokio::test] +async fn non_websocket_upgrades_reach_python_as_plain_requests() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET / HTTP/1.1\r\nhost: x\r\nconnection: upgrade, http2-settings\r\nupgrade: h2c\r\nhttp2-settings: AAMAAABkAAQAoAAAAAIAAAAA\r\n\r\n") + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + let request = received.recv().await.unwrap(); + assert!(!request.headers.contains_key("upgrade")); + assert!(!request.headers.contains_key("connection")); + assert!(!request.headers.contains_key("http2-settings")); +} + +#[rstest] +#[case::get("GET")] +#[case::delete("DELETE")] +#[case::options("OPTIONS")] +#[case::post("POST")] +#[tokio::test] +async fn forwards_a_chunked_body_on_any_method(#[case] method: &str) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "{method} /v1/search HTTP/1.1\r\nhost: x\r\ntransfer-encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\n\r\n" + )) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(received.recv().await.unwrap().body, "body"); +} diff --git a/litellm-rust/crates/gateway-forward/tests/roundtrip.rs b/litellm-rust/crates/gateway-forward/tests/roundtrip.rs new file mode 100644 index 00000000000..6d5cb8ae86c --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/roundtrip.rs @@ -0,0 +1,203 @@ +mod support; + +use std::{ + convert::Infallible, + sync::{Arc, Mutex}, +}; + +use axum::{ + body::{Body, Bytes}, + http::{HeaderMap, HeaderName, HeaderValue, Method, Request, StatusCode}, + response::Response, +}; +use http_body_util::BodyExt; +use litellm_gateway_forward::PEER_HEADER; +use proptest::{prelude::*, test_runner::TestRunner}; +use support::{Gateway, client, recording}; + +#[derive(Clone, Debug)] +struct Message { + headers: Vec<(String, Vec>)>, + body: Vec, + chunked: bool, +} + +#[derive(Clone, Debug)] +struct Exchange { + method: Method, + target: String, + request: Message, + status: StatusCode, + response: Message, +} + +fn header_value() -> impl Strategy> { + prop::collection::vec(prop_oneof![0x21u8..=0x7e, 0x80u8..=0xff], 1..24) +} + +fn message() -> impl Strategy { + ( + prop::collection::btree_map( + "x-[a-z0-9-]{1,12}", + prop::collection::vec(header_value(), 1..4), + 0..12, + ), + prop::collection::vec(any::(), 0..32 * 1024), + any::(), + ) + .prop_map(|(headers, body, chunked)| Message { + headers: headers.into_iter().collect(), + body, + chunked, + }) +} + +fn target() -> impl Strategy { + let segment = prop_oneof!["[A-Za-z0-9._~!$&'()*+,;=:@-]{0,8}", "%[0-9A-F]{2}"]; + ( + prop::collection::vec(segment, 0..5), + prop::option::of("[A-Za-z0-9=&._~-]{0,20}"), + ) + .prop_map(|(segments, query)| { + let path = format!("/{}", segments.join("/")); + match query { + Some(query) => format!("{path}?{query}"), + None => path, + } + }) +} + +fn exchange() -> impl Strategy { + let method = prop_oneof![ + Just(Method::GET), + Just(Method::POST), + Just(Method::PUT), + Just(Method::PATCH), + Just(Method::DELETE), + Just(Method::OPTIONS), + Just(Method::from_bytes(b"PURGE").unwrap()), + ]; + let status = (200u16..600) + .prop_filter("bodiless status", |code| ![204, 205, 304].contains(code)) + .prop_map(|code| StatusCode::from_u16(code).unwrap()); + (method, target(), message(), status, message()).prop_map( + |(method, target, request, status, response)| Exchange { + method, + target, + request, + status, + response, + }, + ) +} + +fn body(message: &Message) -> Body { + if !message.chunked { + return Body::from(message.body.clone()); + } + let chunks: Vec> = message + .body + .chunks(1000) + .map(|chunk| Ok(Bytes::copy_from_slice(chunk))) + .collect(); + Body::from_stream(futures_util::stream::iter(chunks)) +} + +fn apply(headers: &mut HeaderMap, message: &Message) { + if message.chunked { + headers.insert("transfer-encoding", HeaderValue::from_static("chunked")); + } + for (name, values) in &message.headers { + for value in values { + headers.append( + HeaderName::from_bytes(name.as_bytes()).unwrap(), + HeaderValue::from_bytes(value).unwrap(), + ); + } + } +} + +fn assert_carried(headers: &HeaderMap, message: &Message, framing_and_hop: &[&str]) { + for (name, values) in &message.headers { + let carried: Vec<&[u8]> = headers + .get_all(name) + .iter() + .map(HeaderValue::as_bytes) + .collect(); + let sent: Vec<&[u8]> = values.iter().map(Vec::as_slice).collect(); + assert_eq!(carried, sent, "header {name}"); + } + for name in headers.keys() { + assert!( + framing_and_hop.contains(&name.as_str()) + || message + .headers + .iter() + .any(|(sent, _)| sent == name.as_str()), + "unexpected header {name}" + ); + } +} + +/// Any request a client can send and any response Python can give cross the forwarder unchanged, +/// apart from the framing and peer headers the forwarder owns. +#[test] +fn every_exchange_crosses_the_forwarder_unchanged() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + let current = Arc::new(Mutex::new(None::)); + let responding = current.clone(); + let (gateway, received) = runtime.block_on(async { + let (upstream, received) = recording(move || { + let exchange = responding.lock().unwrap().clone().unwrap(); + let mut response = Response::new(body(&exchange.response)); + *response.status_mut() = exchange.status; + apply(response.headers_mut(), &exchange.response); + response + }) + .await; + ( + Gateway::start(upstream).await, + tokio::sync::Mutex::new(received), + ) + }); + let client = client(); + TestRunner::new(ProptestConfig::with_cases(256)) + .run(&exchange(), |exchange| { + *current.lock().unwrap() = Some(exchange.clone()); + runtime.block_on(async { + while received.lock().await.try_recv().is_ok() {} + let mut request = Request::builder() + .method(exchange.method.clone()) + .uri(gateway.url(&exchange.target)) + .body(body(&exchange.request)) + .unwrap(); + apply(request.headers_mut(), &exchange.request); + let response = support::within(client.request(request)).await.unwrap(); + + let python = received.lock().await.recv().await.unwrap(); + assert_eq!(python.method, exchange.method); + assert_eq!(python.uri, exchange.target.as_str()); + assert_eq!(python.body, exchange.request.body); + assert_carried( + &python.headers, + &exchange.request, + &["host", "content-length", "transfer-encoding", PEER_HEADER], + ); + assert_eq!(python.headers.get_all(PEER_HEADER).iter().count(), 1); + + assert_eq!(response.status(), exchange.status); + let headers = response.headers().clone(); + assert_carried( + &headers, + &exchange.response, + &["date", "content-length", "transfer-encoding"], + ); + let body = support::within(response.into_body().collect()) + .await + .unwrap(); + assert_eq!(body.to_bytes(), exchange.response.body); + }); + Ok(()) + }) + .unwrap(); +} diff --git a/litellm-rust/crates/gateway-forward/tests/support/mod.rs b/litellm-rust/crates/gateway-forward/tests/support/mod.rs new file mode 100644 index 00000000000..b68a42cb66b --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/support/mod.rs @@ -0,0 +1,299 @@ +#![allow( + dead_code, + reason = "each test binary uses a different subset of the harness" +)] + +use std::{future::Future, net::SocketAddr, time::Duration}; + +use axum::{ + Router, + body::{Body, Bytes}, + extract::Request, + http::{HeaderMap, Method, StatusCode, Uri}, +}; +use http_body_util::BodyExt; +use hyper_util::{ + client::legacy::{Client, connect::HttpConnector}, + rt::TokioExecutor, +}; +use litellm_gateway_forward::Forward; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, + sync::{mpsc, oneshot}, + task::JoinHandle, +}; + +pub const WAIT: Duration = Duration::from_secs(5); +pub const UVICORN_KEEPALIVE: Duration = Duration::from_secs(5); + +pub async fn within(future: impl Future) -> T { + tokio::time::timeout(WAIT, future).await.expect("timed out") +} + +/// A running forwarder in front of `upstream`, listening on a real socket like production. +pub struct Gateway { + pub address: SocketAddr, + stop: oneshot::Sender<()>, + server: JoinHandle>, +} + +impl Gateway { + pub async fn start(upstream: SocketAddr) -> Self { + Self::with_keepalive(upstream, UVICORN_KEEPALIVE).await + } + + pub async fn with_keepalive(upstream: SocketAddr, keepalive: Duration) -> Self { + let forward = Forward::new(&format!("http://{upstream}"), keepalive).unwrap(); + let public = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = public.local_addr().unwrap(); + let (stop, stopped) = oneshot::channel::<()>(); + let server = tokio::spawn(forward.serve(public, async { + stopped.await.ok(); + })); + Self { + address, + stop, + server, + } + } + + pub fn url(&self, path_and_query: &str) -> Uri { + format!("http://{}{path_and_query}", self.address) + .parse() + .unwrap() + } + + pub async fn connect(&self) -> Raw { + Raw::new(TcpStream::connect(self.address).await.unwrap()) + } + + /// Starts graceful shutdown and returns the server task, which ends once in-flight work drains. + pub fn shutdown(self) -> JoinHandle> { + self.stop.send(()).unwrap(); + self.server + } +} + +pub fn client() -> Client { + Client::builder(TokioExecutor::new()).build_http() +} + +/// What the upstream received, after the forwarder rewrote it. +pub struct Received { + pub method: Method, + pub uri: Uri, + pub headers: HeaderMap, + pub body: Bytes, +} + +/// An axum upstream that records every request it fully reads and answers with `respond`. +pub async fn recording( + respond: impl Fn() -> axum::response::Response + Clone + Send + Sync + 'static, +) -> (SocketAddr, mpsc::UnboundedReceiver) { + let (sender, receiver) = mpsc::unbounded_channel(); + let address = serve(Router::new().fallback(move |request: Request| { + let sender = sender.clone(); + let respond = respond.clone(); + async move { + let (parts, body) = request.into_parts(); + let body = body.collect().await.unwrap().to_bytes(); + sender + .send(Received { + method: parts.method, + uri: parts.uri, + headers: parts.headers, + body, + }) + .unwrap(); + respond() + } + })) + .await; + (address, receiver) +} + +pub async fn serve(router: Router) -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + address +} + +/// An upstream that hands each accepted connection to `handle` as raw bytes, for behavior a +/// well-behaved HTTP server would never produce. +pub async fn raw_upstream(handle: F) -> SocketAddr +where + F: Fn(Raw) -> Fut + Send + Sync + 'static, + Fut: Future + Send + 'static, +{ + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.unwrap(); + tokio::spawn(handle(Raw::new(stream))); + } + }); + address +} + +pub fn unreachable_upstream() -> SocketAddr { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.local_addr().unwrap() +} + +pub struct Head { + pub status: StatusCode, + pub headers: Vec<(String, Vec)>, + pub path: String, +} + +impl Head { + pub fn header(&self, name: &str) -> Option<&[u8]> { + self.headers + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_slice()) + } + + pub fn content_length(&self) -> Option { + self.header("content-length") + .map(|value| std::str::from_utf8(value).unwrap().parse().unwrap()) + } +} + +/// One TCP connection driven byte by byte, so tests control framing, timing and disconnects. +/// Bytes read past a message head stay buffered for the next read. +pub struct Raw { + stream: TcpStream, + buffer: Vec, +} + +impl Raw { + pub fn new(stream: TcpStream) -> Self { + Self { + stream, + buffer: Vec::new(), + } + } + + pub fn into_stream(self) -> TcpStream { + assert!(self.buffer.is_empty(), "buffered bytes would be lost"); + self.stream + } + + pub fn local_address(&self) -> SocketAddr { + self.stream.local_addr().unwrap() + } + + pub async fn send(&mut self, bytes: impl AsRef<[u8]>) { + self.stream.write_all(bytes.as_ref()).await.unwrap(); + } + + /// Reads one request head, or `None` when the peer closed the connection first. + pub async fn request_head(&mut self) -> Option { + self.head(|bytes| { + let mut headers = [httparse::EMPTY_HEADER; 256]; + let mut request = httparse::Request::new(&mut headers); + let httparse::Status::Complete(length) = request.parse(bytes).unwrap() else { + return None; + }; + Some(( + length, + Head { + status: StatusCode::OK, + headers: owned(request.headers), + path: request.path.unwrap().to_owned(), + }, + )) + }) + .await + } + + pub async fn response_head(&mut self) -> Option { + self.head(|bytes| { + let mut headers = [httparse::EMPTY_HEADER; 256]; + let mut response = httparse::Response::new(&mut headers); + let httparse::Status::Complete(length) = response.parse(bytes).unwrap() else { + return None; + }; + Some(( + length, + Head { + status: StatusCode::from_u16(response.code.unwrap()).unwrap(), + headers: owned(response.headers), + path: String::new(), + }, + )) + }) + .await + } + + async fn head(&mut self, parse: impl Fn(&[u8]) -> Option<(usize, Head)>) -> Option { + loop { + if let Some((length, head)) = parse(&self.buffer) { + self.buffer.drain(..length); + return Some(head); + } + if self.fill().await == 0 { + return None; + } + } + } + + /// Reads exactly `length` body bytes, using what is already buffered first. + pub async fn body(&mut self, length: usize) -> Vec { + while self.buffer.len() < length { + assert_ne!(self.fill().await, 0, "closed mid-body"); + } + self.buffer.drain(..length).collect() + } + + /// Reads a whole chunked body and returns its decoded bytes. + pub async fn chunked_body(&mut self) -> Vec { + let mut decoded = Vec::new(); + loop { + let size_line = self.line().await; + let size = + usize::from_str_radix(size_line.split(';').next().unwrap().trim(), 16).unwrap(); + if size == 0 { + while !self.line().await.is_empty() {} + return decoded; + } + decoded.extend(self.body(size).await); + assert_eq!(self.body(2).await, b"\r\n"); + } + } + + async fn line(&mut self) -> String { + loop { + if let Some(end) = self.buffer.windows(2).position(|window| window == b"\r\n") { + let line = String::from_utf8(self.buffer[..end].to_vec()).unwrap(); + self.buffer.drain(..end + 2); + return line; + } + assert_ne!(self.fill().await, 0, "closed mid-line"); + } + } + + async fn fill(&mut self) -> usize { + let mut chunk = [0; 8192]; + let read = self.stream.read(&mut chunk).await.unwrap_or(0); + self.buffer.extend_from_slice(&chunk[..read]); + read + } + + /// Resolves once the peer closes the connection, discarding anything it still sends. + pub async fn closed(&mut self) { + let mut chunk = [0; 8192]; + while self.stream.read(&mut chunk).await.unwrap_or(0) != 0 {} + } +} + +fn owned(headers: &[httparse::Header<'_>]) -> Vec<(String, Vec)> { + headers + .iter() + .map(|header| (header.name.to_owned(), header.value.to_vec())) + .collect() +} diff --git a/litellm-rust/crates/gateway-forward/tests/upstream.rs b/litellm-rust/crates/gateway-forward/tests/upstream.rs new file mode 100644 index 00000000000..941208d5223 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/upstream.rs @@ -0,0 +1,23 @@ +use std::time::Duration; + +use litellm_gateway_forward::Forward; +use rstest::rstest; + +#[rstest] +#[case::https("https://127.0.0.1:4000")] +#[case::public("http://192.0.2.1:4000")] +#[case::hostname("http://localhost:4000")] +#[case::ipv6("http://[::1]:4000")] +#[case::path("http://127.0.0.1:4000/api")] +#[case::query("http://127.0.0.1:4000/?x=y")] +#[case::credentials("http://user@127.0.0.1:4000")] +fn rejects_anything_but_a_loopback_http_origin(#[case] upstream: &str) { + assert!(Forward::new(upstream, Duration::from_secs(5)).is_err()); +} + +#[rstest] +#[case::standard("http://127.0.0.1:4000")] +#[case::loopback_range("http://127.0.0.2:4000/")] +fn accepts_a_loopback_http_origin(#[case] upstream: &str) { + assert!(Forward::new(upstream, Duration::from_secs(5)).is_ok()); +} diff --git a/litellm-rust/crates/gateway-forward/tests/websocket.rs b/litellm-rust/crates/gateway-forward/tests/websocket.rs new file mode 100644 index 00000000000..02fbd1f9368 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/websocket.rs @@ -0,0 +1,186 @@ +mod support; + +use std::{ + net::SocketAddr, + time::{Duration, Instant}, +}; + +use axum::{ + Router, + http::{HeaderValue, StatusCode}, + response::IntoResponse, +}; +use futures_util::{SinkExt, StreamExt}; +use litellm_gateway_forward::PEER_HEADER; +use rstest::rstest; +use support::{Gateway, raw_upstream, serve, within}; +use tokio::net::TcpStream; +use tokio_tungstenite::{ + WebSocketStream, accept_hdr_async, client_async, + tungstenite::{ + Error, Message, + client::IntoClientRequest, + handshake::server::{ErrorResponse, Request as Handshake, Response as Accepted}, + protocol::{CloseFrame, frame::coding::CloseCode}, + }, +}; + +async fn connect( + address: SocketAddr, + connection: &'static str, +) -> Result, Box> { + let mut request = format!("ws://{address}/v1/realtime?model=m") + .into_client_request() + .unwrap(); + let headers = request.headers_mut(); + headers.insert("authorization", HeaderValue::from_static("Bearer test")); + headers.insert( + "sec-websocket-protocol", + HeaderValue::from_static("realtime"), + ); + headers.insert("connection", HeaderValue::from_static(connection)); + let stream = TcpStream::connect(address).await.unwrap(); + let (socket, response) = client_async(request, stream).await.map_err(Box::new)?; + assert_eq!(response.headers()["sec-websocket-protocol"], "realtime"); + Ok(socket) +} + +/// Greets, echoes every message, and closes with 4000 when told to "bye". +#[expect( + clippy::result_large_err, + reason = "tungstenite's handshake callback returns ErrorResponse" +)] +async fn echo() -> SocketAddr { + raw_upstream(|python| async move { + let callback = |request: &Handshake, mut response: Accepted| { + assert_eq!(request.uri(), "/v1/realtime?model=m"); + assert_eq!(request.headers()["authorization"], "Bearer test"); + assert!(request.headers().contains_key(PEER_HEADER)); + response.headers_mut().insert( + "sec-websocket-protocol", + HeaderValue::from_static("realtime"), + ); + Ok::<_, ErrorResponse>(response) + }; + let mut socket = accept_hdr_async(python.into_stream(), callback) + .await + .unwrap(); + socket.send(Message::text("greeting")).await.unwrap(); + while let Some(Ok(message)) = socket.next().await { + if message == Message::text("bye") { + socket + .close(Some(CloseFrame { + code: CloseCode::Library(4000), + reason: "done".into(), + })) + .await + .unwrap(); + continue; + } + if message.is_close() { + break; + } + socket.send(message).await.unwrap(); + } + }) + .await +} + +#[rstest] +#[tokio::test] +async fn relays_messages_both_ways( + #[values("Upgrade", "keep-alive, Upgrade", "upgrade")] connection: &'static str, + #[values( + Message::text("hello"), + Message::binary(vec![0, 1, 2]), + Message::binary(vec![7; 4 * 1024 * 1024]) + )] + message: Message, +) { + let gateway = Gateway::start(echo().await).await; + let mut socket = within(connect(gateway.address, connection)).await.unwrap(); + assert_eq!( + socket.next().await.unwrap().unwrap(), + Message::text("greeting") + ); + socket.send(message.clone()).await.unwrap(); + assert_eq!(within(socket.next()).await.unwrap().unwrap(), message); + socket.close(None).await.unwrap(); + while socket.next().await.is_some() {} + drop(socket); + within(gateway.shutdown()).await.unwrap().unwrap(); +} + +#[rstest] +#[tokio::test] +async fn relays_a_close_python_initiates() { + let gateway = Gateway::start(echo().await).await; + let mut socket = within(connect(gateway.address, "Upgrade")).await.unwrap(); + socket.next().await.unwrap().unwrap(); + socket.send(Message::text("bye")).await.unwrap(); + let Message::Close(Some(frame)) = within(socket.next()).await.unwrap().unwrap() else { + panic!("expected python's close frame"); + }; + assert_eq!(frame.code, CloseCode::Library(4000)); + assert_eq!(frame.reason, "done"); +} + +#[rstest] +#[tokio::test] +async fn passes_an_upstream_rejection_through() { + let upstream = serve( + Router::new() + .fallback(|| async { (StatusCode::FORBIDDEN, "origin not allowed").into_response() }), + ) + .await; + let gateway = Gateway::start(upstream).await; + let error = within(connect(gateway.address, "Upgrade")) + .await + .unwrap_err(); + let Error::Http(response) = *error else { + panic!("expected the upstream rejection, got {error}"); + }; + assert_eq!(response.status(), StatusCode::FORBIDDEN); + assert_eq!(response.body().as_deref(), Some(&b"origin not allowed"[..])); +} + +#[rstest] +#[tokio::test] +async fn shutdown_closes_open_sockets_after_drain() { + let gateway = Gateway::start(echo().await).await; + let mut socket = within(connect(gateway.address, "Upgrade")).await.unwrap(); + socket.next().await.unwrap().unwrap(); + let started = Instant::now(); + tokio::time::timeout(Duration::from_secs(10), gateway.shutdown()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(started.elapsed() >= Duration::from_secs(4)); + assert!(!matches!(socket.next().await, Some(Ok(Message::Text(_))))); +} + +#[rstest] +#[tokio::test] +async fn refuses_new_sockets_while_draining() { + let gateway = Gateway::start(echo().await).await; + let mut open = within(connect(gateway.address, "Upgrade")).await.unwrap(); + open.next().await.unwrap().unwrap(); + let address = gateway.address; + let server = gateway.shutdown(); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!( + TcpStream::connect(address).await.is_err() || connect(address, "Upgrade").await.is_err() + ); + open.send(Message::text("still open")).await.unwrap(); + assert_eq!( + within(open.next()).await.unwrap().unwrap(), + Message::text("still open") + ); + drop(open); + tokio::time::timeout(Duration::from_secs(10), server) + .await + .unwrap() + .unwrap() + .unwrap(); +} diff --git a/litellm-rust/crates/gateway/Cargo.toml b/litellm-rust/crates/gateway/Cargo.toml index 24a786f741d..d0cedbb3b36 100644 --- a/litellm-rust/crates/gateway/Cargo.toml +++ b/litellm-rust/crates/gateway/Cargo.toml @@ -10,12 +10,11 @@ litellm-secret-value.workspace = true axum.workspace = true envy = "0.4.2" http-body-util = "0.1" -hyper = { version = "1", features = ["http1"] } -hyper-util = { workspace = true, features = ["client-legacy", "http1", "tokio"] } litellm-inference.workspace = true litellm-gateway-inference.workspace = true +litellm-gateway-forward.workspace = true litellm-gateway-mcp.workspace = true -tokio-util = { version = "0.7", features = ["rt"] } +tokio-util = "0.7" thiserror.workspace = true futures-util.workspace = true litellm-gateway-auth.workspace = true @@ -31,7 +30,7 @@ litellm-tracing.workspace = true serde_json.workspace = true serde.workspace = true tracing.workspace = true -tokio = { workspace = true, features = ["signal", "io-util"] } +tokio = { workspace = true, features = ["signal"] } uuid.workspace = true tower-sessions-moka-store = "0.15.0" @@ -41,5 +40,4 @@ base64.workspace = true rstest.workspace = true tempfile.workspace = true tokio = { workspace = true, features = ["sync"] } -tokio-tungstenite.workspace = true tower = { version = "0.5", features = ["util"] } diff --git a/litellm-rust/crates/gateway/src/error.rs b/litellm-rust/crates/gateway/src/error.rs index 6911ce17df9..e2f646c5448 100644 --- a/litellm-rust/crates/gateway/src/error.rs +++ b/litellm-rust/crates/gateway/src/error.rs @@ -1,11 +1,5 @@ #[derive(Debug, thiserror::Error)] pub enum Error { - #[error("invalid forwarding upstream URI")] - ForwardUri(#[source] axum::http::uri::InvalidUri), - #[error( - "LITELLM_FORWARD_UPSTREAM must be a loopback IPv4 http origin without a path, query or credentials" - )] - ForwardUpstream, #[error(transparent)] Http(#[from] litellm_http::Error), #[error("environment_variables values must be strings, numbers or booleans")] diff --git a/litellm-rust/crates/gateway/src/forward/mod.rs b/litellm-rust/crates/gateway/src/forward/mod.rs deleted file mode 100644 index 98e971b7a29..00000000000 --- a/litellm-rust/crates/gateway/src/forward/mod.rs +++ /dev/null @@ -1,371 +0,0 @@ -use std::net::{Ipv4Addr, SocketAddr}; - -use axum::{ - Router, - body::Body, - extract::{ConnectInfo, DefaultBodyLimit, Request, State}, - http::{HeaderMap, HeaderValue, StatusCode, Uri, Version, header}, - response::Response, -}; -use hyper_util::{ - client::legacy::{Client, connect::HttpConnector}, - rt::TokioExecutor, -}; -use tokio::net::TcpListener; - -use crate::Error; -use websocket::Upgrades; - -mod websocket; - -pub const PEER_HEADER: &str = "x-litellm-rust-peer"; - -#[derive(Clone)] -pub struct Forward { - upstream: Uri, - client: Client, - upgrades: Upgrades, -} - -impl Forward { - pub fn new(upstream: &str) -> Result { - let upstream: Uri = upstream.parse().map_err(Error::ForwardUri)?; - if upstream.scheme_str() != Some("http") - || !upstream - .host() - .and_then(|host| host.parse::().ok()) - .is_some_and(|ip| ip.is_loopback()) - || upstream - .authority() - .is_some_and(|authority| authority.as_str().contains('@')) - || upstream.path() != "/" - || upstream.query().is_some() - { - return Err(Error::ForwardUpstream); - } - let client = Client::builder(TokioExecutor::new()) - .retry_canceled_requests(false) - .build_http(); - Ok(Self { - upstream, - client, - upgrades: Upgrades::default(), - }) - } - - pub fn router(self) -> Router { - Router::new() - .fallback(forward) - .layer(DefaultBodyLimit::disable()) - .with_state(self) - } - - /// Serves until `shutdown` resolves, then gives open WebSockets a bounded drain before closing them. - pub async fn serve( - self, - listener: TcpListener, - shutdown: impl Future + Send + 'static, - ) -> std::io::Result<()> { - let upgrades = self.upgrades.clone(); - let _close_on_drop = upgrades.close_on_drop(); - let draining = upgrades.clone(); - let result = axum::serve( - listener, - self.router() - .into_make_service_with_connect_info::(), - ) - .with_graceful_shutdown(async move { - shutdown.await; - draining.begin_shutdown(); - }) - .await; - if result.is_err() { - upgrades.close(); - } - upgrades.wait().await; - result - } -} - -async fn forward( - State(state): State, - ConnectInfo(peer): ConnectInfo, - mut request: Request, -) -> Result { - let downstream = state.upgrades.take_handshake(&mut request)?; - let mut uri = state.upstream.into_parts(); - uri.path_and_query = request.uri().path_and_query().cloned(); - *request.uri_mut() = Uri::from_parts(uri).map_err(|_| StatusCode::BAD_GATEWAY)?; - *request.version_mut() = Version::HTTP_11; - let headers = request.headers_mut(); - strip_hop_headers(headers); - if downstream.is_some() { - websocket::restore_handshake_headers(headers); - } - // Forwarding headers pass through untouched so Python sees the request as if the client - // connected to it directly. Inserting replaces any client-sent copy: the peer is the one - // value a client must not choose. - headers.insert( - PEER_HEADER, - HeaderValue::from_str(&peer.to_string()).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?, - ); - let response = state.client.request(request); - let mut response = match downstream { - Some(_) => state.upgrades.until_closed(response).await?, - None => response.await, - } - .map_err(|error| { - tracing::warn!(%error, "python proxy unreachable"); - StatusCode::BAD_GATEWAY - })?; - if response.status() == StatusCode::SWITCHING_PROTOCOLS { - return state.upgrades.switch(downstream, response).await; - } - strip_hop_headers(response.headers_mut()); - Ok(response.map(Body::new)) -} - -// Per-connection headers (RFC 9110 section 7.6.1) describe the client's connection to Rust, not -// Rust's connection to Python, so a proxy must drop them instead of forwarding them. -fn strip_hop_headers(headers: &mut HeaderMap) { - let connection_headers: Vec<_> = headers - .get_all(header::CONNECTION) - .iter() - .filter_map(|value| value.to_str().ok()) - .flat_map(|value| value.split(',')) - .filter_map(|name| header::HeaderName::from_bytes(name.trim().as_bytes()).ok()) - .collect(); - for name in connection_headers { - headers.remove(name); - } - for name in [ - "connection", - "keep-alive", - "proxy-authenticate", - "proxy-authorization", - "te", - "trailer", - "transfer-encoding", - "upgrade", - "proxy-connection", - ] { - headers.remove(name); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use axum::{body::Bytes, http::Version}; - use http_body_util::BodyExt; - use rstest::rstest; - use std::{convert::Infallible, time::Duration}; - use tokio::{net::TcpListener, sync::mpsc}; - use tower::ServiceExt; - - #[rstest] - #[case::https("https://127.0.0.1:4000")] - #[case::public("http://192.0.2.1:4000")] - #[case::hostname("http://localhost:4000")] - #[case::ipv6("http://[::1]:4000")] - #[case::path("http://127.0.0.1:4000/api")] - #[case::query("http://127.0.0.1:4000/?x=y")] - #[case::credentials("http://user@127.0.0.1:4000")] - fn rejects_invalid_upstream(#[case] upstream: &str) { - assert!(Forward::new(upstream).is_err()); - } - - #[rstest] - #[case::standard("http://127.0.0.1:4000")] - #[case::loopback_range("http://127.0.0.2:4000/")] - fn accepts_loopback_origin(#[case] upstream: &str) { - assert!(Forward::new(upstream).is_ok()); - } - - #[rstest] - #[tokio::test] - async fn preserves_request_and_response_without_hop_headers() { - let upstream = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let origin = format!("http://{}", upstream.local_addr().unwrap()); - let task = tokio::spawn(async move { - axum::serve( - upstream, - Router::new().fallback(|request: Request| async { - assert_eq!(request.method(), "PATCH"); - assert_eq!(request.uri(), "/v1/messages?x=a%2Fb&x=c"); - assert_eq!(request.headers()["host"], "public.example"); - assert_eq!(request.headers()["authorization"], "Bearer test"); - assert_eq!( - request - .headers() - .get_all("x-forwarded-for") - .iter() - .collect::>(), - ["192.0.2.1", "192.0.2.2"] - ); - assert_eq!(request.headers()["x-forwarded-proto"], "https"); - assert_eq!(request.headers()["forwarded"], "for=192.0.2.1;proto=https"); - assert_eq!( - request - .headers() - .get_all(PEER_HEADER) - .iter() - .collect::>(), - ["198.51.100.5:1234"] - ); - assert!(!request.headers().contains_key("x-remove")); - assert!(!request.headers().contains_key("proxy-authorization")); - assert_eq!( - request.into_body().collect().await.unwrap().to_bytes(), - "payload" - ); - Response::builder() - .status(StatusCode::UNAUTHORIZED) - .header("connection", "x-upstream-hop") - .header("x-upstream-hop", "remove") - .header("x-response", "keep") - .body(Body::from("python error")) - .unwrap() - }), - ) - .await - .unwrap(); - }); - let request = Request::builder() - .method("PATCH") - .version(Version::HTTP_2) - .uri("/v1/messages?x=a%2Fb&x=c") - .header("host", "public.example") - .header("authorization", "Bearer test") - .header("x-forwarded-for", "192.0.2.1") - .header("x-forwarded-for", "192.0.2.2") - .header("x-forwarded-proto", "https") - .header("forwarded", "for=192.0.2.1;proto=https") - .header(PEER_HEADER, "10.1.2.3:1") - .header(PEER_HEADER, "10.1.2.4:1") - .header("connection", "x-remove, keep-alive") - .header("x-remove", "remove") - .header("proxy-authorization", "remove") - .extension(ConnectInfo( - "198.51.100.5:1234".parse::().unwrap(), - )) - .body(Body::from("payload")) - .unwrap(); - let response = Forward::new(&origin) - .unwrap() - .router() - .oneshot(request) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - assert_eq!(response.headers()["x-response"], "keep"); - assert!(!response.headers().contains_key("connection")); - assert!(!response.headers().contains_key("x-upstream-hop")); - assert_eq!( - response.into_body().collect().await.unwrap().to_bytes(), - "python error" - ); - task.abort(); - } - - #[rstest] - #[case::health("/health/liveliness")] - #[case::readiness("/health/readiness")] - #[case::messages("/v1/messages")] - #[tokio::test] - async fn unreachable_upstream_is_502(#[case] path: &str) { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let origin = format!("http://{}", listener.local_addr().unwrap()); - drop(listener); - let response = Forward::new(&origin) - .unwrap() - .router() - .oneshot( - Request::builder() - .uri(path) - .extension(ConnectInfo( - "198.51.100.5:1234".parse::().unwrap(), - )) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::BAD_GATEWAY); - } - - #[rstest] - #[tokio::test] - async fn streams_upload_and_response_before_eof_without_body_limit() { - let upstream = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let origin = format!("http://{}", upstream.local_addr().unwrap()); - let upstream_task = tokio::spawn(async move { - axum::serve( - upstream, - Router::new().fallback(|request: Request| async { - let headers = request.headers(); - assert!(!headers.contains_key("x-forwarded-for")); - assert!(!headers.contains_key("x-forwarded-proto")); - assert!( - headers[PEER_HEADER] - .to_str() - .unwrap() - .starts_with("127.0.0.1:") - ); - Response::new(request.into_body()) - }), - ) - .await - .unwrap(); - }); - let public = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let public_uri = format!("http://{}/chat/completions", public.local_addr().unwrap()); - let forward_task = tokio::spawn(async move { - axum::serve( - public, - Forward::new(&origin) - .unwrap() - .router() - .into_make_service_with_connect_info::(), - ) - .await - .unwrap(); - }); - let (sender, receiver) = mpsc::channel::>(1); - let stream = futures_util::stream::unfold(receiver, |mut receiver| async { - receiver.recv().await.map(|chunk| (chunk, receiver)) - }); - sender.send(Ok(Bytes::from_static(b"first"))).await.unwrap(); - let client: Client = - Client::builder(TokioExecutor::new()).build_http(); - let mut response = tokio::time::timeout( - Duration::from_secs(5), - client.request( - Request::post(public_uri) - .body(Body::from_stream(stream)) - .unwrap(), - ), - ) - .await - .unwrap() - .unwrap(); - assert_eq!(response.status(), StatusCode::OK); - let frame = tokio::time::timeout(Duration::from_secs(5), response.body_mut().frame()) - .await - .unwrap() - .unwrap() - .unwrap(); - assert_eq!(frame.into_data().unwrap(), "first"); - let large = Bytes::from(vec![b'x'; 3 * 1024 * 1024]); - sender.send(Ok(large.clone())).await.unwrap(); - drop(sender); - let rest = tokio::time::timeout(Duration::from_secs(5), response.into_body().collect()) - .await - .unwrap() - .unwrap() - .to_bytes(); - assert_eq!(rest, large); - upstream_task.abort(); - forward_task.abort(); - } -} diff --git a/litellm-rust/crates/gateway/src/forward/websocket.rs b/litellm-rust/crates/gateway/src/forward/websocket.rs deleted file mode 100644 index c6d0ad7fe3c..00000000000 --- a/litellm-rust/crates/gateway/src/forward/websocket.rs +++ /dev/null @@ -1,297 +0,0 @@ -use std::time::Duration; - -use axum::{ - body::Body, - extract::Request, - http::{HeaderMap, HeaderValue, Method, StatusCode, Version, header}, - response::Response, -}; -use hyper::{ - body::Incoming, - upgrade::{OnUpgrade, Upgraded}, -}; -use hyper_util::rt::TokioIo; -use tokio_util::{ - sync::{CancellationToken, DropGuard}, - task::TaskTracker, -}; - -const DRAIN_TIMEOUT: Duration = Duration::from_secs(5); - -#[derive(Clone, Default)] -pub(super) struct Upgrades { - tasks: TaskTracker, - draining: CancellationToken, - closed: CancellationToken, -} - -impl Upgrades { - pub fn begin_shutdown(&self) { - self.draining.cancel(); - let closed = self.closed.clone(); - tokio::spawn(async move { - tokio::time::sleep(DRAIN_TIMEOUT).await; - closed.cancel(); - }); - } - - pub fn take_handshake(&self, request: &mut Request) -> Result, StatusCode> { - if request.method() != Method::GET - || request.version() != Version::HTTP_11 - || !is_handshake(request.headers()) - { - return Ok(None); - } - if self.draining.is_cancelled() { - return Err(StatusCode::SERVICE_UNAVAILABLE); - } - Ok(Some(hyper::upgrade::on(request))) - } - - pub async fn until_closed(&self, future: impl Future) -> Result { - tokio::select! { - _ = self.closed.cancelled() => Err(StatusCode::SERVICE_UNAVAILABLE), - output = future => Ok(output), - } - } - - pub async fn switch( - &self, - downstream: Option, - mut response: Response, - ) -> Result { - let Some(downstream) = downstream else { - return Err(StatusCode::BAD_GATEWAY); - }; - if !is_handshake(response.headers()) { - return Err(StatusCode::BAD_GATEWAY); - } - let upstream = self - .until_closed(hyper::upgrade::on(&mut response)) - .await? - .map_err(|error| { - tracing::warn!(%error, "python proxy websocket upgrade failed"); - StatusCode::BAD_GATEWAY - })?; - super::strip_hop_headers(response.headers_mut()); - restore_handshake_headers(response.headers_mut()); - self.relay(downstream, upstream); - Ok(response.map(|_| Body::empty())) - } - - pub fn close(&self) { - self.closed.cancel(); - } - - pub fn close_on_drop(&self) -> DropGuard { - self.closed.clone().drop_guard() - } - - pub async fn wait(&self) { - self.tasks.close(); - self.tasks.wait().await; - } - - fn relay(&self, downstream: OnUpgrade, upstream: Upgraded) { - let closed = self.closed.clone(); - self.tasks.spawn(async move { - tokio::select! { - _ = closed.cancelled() => {} - _ = async { - let Ok(downstream) = downstream.await else { - tracing::debug!("websocket upgrade did not complete"); - return; - }; - let mut downstream = TokioIo::new(downstream); - let mut upstream = TokioIo::new(upstream); - if let Err(error) = tokio::io::copy_bidirectional(&mut downstream, &mut upstream).await { - tracing::debug!(%error, "websocket relay disconnected"); - } - } => {} - } - }); - } -} - -fn is_handshake(headers: &HeaderMap) -> bool { - headers - .get(header::UPGRADE) - .is_some_and(|value| value.as_bytes().eq_ignore_ascii_case(b"websocket")) - && headers - .get_all(header::CONNECTION) - .iter() - .filter_map(|value| value.to_str().ok()) - .flat_map(|value| value.split(',')) - .any(|token| token.trim().eq_ignore_ascii_case("upgrade")) -} - -// Hop-by-hop stripping drops the handshake's Connection and Upgrade, so each hop gets them back. -pub(super) fn restore_handshake_headers(headers: &mut HeaderMap) { - headers.insert(header::CONNECTION, HeaderValue::from_static("upgrade")); - headers.insert(header::UPGRADE, HeaderValue::from_static("websocket")); -} - -#[cfg(test)] -mod tests { - use super::super::{Forward, PEER_HEADER}; - use axum::{Router, http::HeaderValue, http::StatusCode, response::IntoResponse}; - use futures_util::{SinkExt, StreamExt}; - use std::net::SocketAddr; - use std::time::{Duration, Instant}; - use tokio::net::TcpListener; - use tokio::{net::TcpStream, sync::oneshot}; - use tokio_tungstenite::{ - accept_hdr_async, client_async, - tungstenite::{ - Message, - client::IntoClientRequest, - handshake::server::{ErrorResponse, Request as Handshake, Response as Accepted}, - }, - }; - - async fn gateway( - upstream: impl FnOnce(TcpListener) -> tokio::task::JoinHandle<()>, - ) -> (SocketAddr, oneshot::Sender<()>, tokio::task::JoinHandle<()>) { - let python = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let origin = format!("http://{}", python.local_addr().unwrap()); - upstream(python); - let public = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = public.local_addr().unwrap(); - let (stop, stopped) = oneshot::channel::<()>(); - let server = tokio::spawn(async move { - Forward::new(&origin) - .unwrap() - .serve(public, async { - stopped.await.ok(); - }) - .await - .unwrap(); - }); - (address, stop, server) - } - - async fn connect( - address: SocketAddr, - ) -> Result< - tokio_tungstenite::WebSocketStream, - Box, - > { - let mut request = format!("ws://{address}/v1/realtime?model=m") - .into_client_request() - .unwrap(); - request - .headers_mut() - .insert("authorization", HeaderValue::from_static("Bearer test")); - request.headers_mut().insert( - "sec-websocket-protocol", - HeaderValue::from_static("realtime"), - ); - let stream = TcpStream::connect(address).await.unwrap(); - let (socket, response) = client_async(request, stream).await.map_err(Box::new)?; - assert_eq!(response.headers()["sec-websocket-protocol"], "realtime"); - Ok(socket) - } - - #[expect( - clippy::result_large_err, - reason = "tungstenite's handshake callback returns ErrorResponse" - )] - fn echo(listener: TcpListener) -> tokio::task::JoinHandle<()> { - tokio::spawn(async move { - loop { - let (stream, _) = listener.accept().await.unwrap(); - tokio::spawn(async move { - let callback = |request: &Handshake, mut response: Accepted| { - assert_eq!(request.uri(), "/v1/realtime?model=m"); - assert_eq!(request.headers()["authorization"], "Bearer test"); - assert!(request.headers().contains_key(PEER_HEADER)); - response.headers_mut().insert( - "sec-websocket-protocol", - HeaderValue::from_static("realtime"), - ); - Ok::<_, ErrorResponse>(response) - }; - let mut socket = accept_hdr_async(stream, callback).await.unwrap(); - socket.send(Message::text("greeting")).await.unwrap(); - while let Some(Ok(message)) = socket.next().await { - if message.is_close() { - break; - } - socket.send(message).await.unwrap(); - } - }); - } - }) - } - - #[tokio::test] - async fn relays_messages_both_ways() { - let (address, stop, server) = gateway(echo).await; - let mut socket = connect(address).await.unwrap(); - assert_eq!( - socket.next().await.unwrap().unwrap(), - Message::text("greeting") - ); - socket.send(Message::text("hello")).await.unwrap(); - assert_eq!( - socket.next().await.unwrap().unwrap(), - Message::text("hello") - ); - socket.send(Message::binary(vec![0, 1, 2])).await.unwrap(); - assert_eq!( - socket.next().await.unwrap().unwrap(), - Message::binary(vec![0, 1, 2]) - ); - socket.close(None).await.unwrap(); - while socket.next().await.is_some() {} - drop(socket); - stop.send(()).unwrap(); - tokio::time::timeout(Duration::from_secs(4), server) - .await - .unwrap() - .unwrap(); - } - - #[tokio::test] - async fn passes_upstream_rejection_through() { - let (address, _stop, _server) = gateway(|listener| { - tokio::spawn(async move { - axum::serve( - listener, - Router::new().fallback(|| async { - (StatusCode::FORBIDDEN, "origin not allowed").into_response() - }), - ) - .await - .unwrap(); - }) - }) - .await; - let Err(error) = connect(address).await else { - panic!("expected the upstream rejection"); - }; - let tokio_tungstenite::tungstenite::Error::Http(response) = *error else { - panic!("expected the upstream rejection"); - }; - assert_eq!(response.status(), StatusCode::FORBIDDEN); - assert_eq!(response.body().as_deref(), Some(&b"origin not allowed"[..])); - } - - #[tokio::test] - async fn shutdown_closes_open_sockets_after_drain() { - let (address, stop, server) = gateway(echo).await; - let mut socket = connect(address).await.unwrap(); - assert_eq!( - socket.next().await.unwrap().unwrap(), - Message::text("greeting") - ); - let started = Instant::now(); - stop.send(()).unwrap(); - tokio::time::timeout(Duration::from_secs(10), server) - .await - .unwrap() - .unwrap(); - assert!(started.elapsed() >= Duration::from_secs(4)); - assert!(!matches!(socket.next().await, Some(Ok(Message::Text(_))))); - } -} diff --git a/litellm-rust/crates/gateway/src/lib.rs b/litellm-rust/crates/gateway/src/lib.rs index 8fa322628a7..12a9bf9110a 100644 --- a/litellm-rust/crates/gateway/src/lib.rs +++ b/litellm-rust/crates/gateway/src/lib.rs @@ -1,6 +1,5 @@ mod auth; mod error; -pub mod forward; mod redis; mod secrets; pub use error::Error; diff --git a/litellm-rust/crates/gateway/src/main.rs b/litellm-rust/crates/gateway/src/main.rs index 078f5c0c014..8c8785a10ba 100644 --- a/litellm-rust/crates/gateway/src/main.rs +++ b/litellm-rust/crates/gateway/src/main.rs @@ -47,10 +47,11 @@ async fn main() -> Result<(), Box> { level: settings.log_level(), }) .install_global()?; - let forward = litellm_gateway::forward::Forward::new( + let forward = litellm_gateway_forward::Forward::new( &upstream .into_string() .map_err(|_| "invalid upstream encoding")?, + settings.upstream_keepalive(), )?; let listener = tokio::net::TcpListener::bind((settings.host.as_str(), settings.port)).await?; diff --git a/litellm-rust/crates/gateway/src/settings.rs b/litellm-rust/crates/gateway/src/settings.rs index 332c774b600..3fad2f0cbc3 100644 --- a/litellm-rust/crates/gateway/src/settings.rs +++ b/litellm-rust/crates/gateway/src/settings.rs @@ -1,4 +1,4 @@ -use std::ffi::OsString; +use std::{ffi::OsString, time::Duration}; use litellm_secret_value::SecretValue; use litellm_tracing::Level; @@ -10,6 +10,7 @@ pub(super) struct ForwardSettings { pub host: String, pub port: u16, rust_log: Option, + litellm_forward_keepalive_timeout: u64, } impl Default for ForwardSettings { @@ -18,11 +19,16 @@ impl Default for ForwardSettings { host: "0.0.0.0".into(), port: 4000, rust_log: None, + litellm_forward_keepalive_timeout: 5, } } } impl ForwardSettings { + pub fn upstream_keepalive(&self) -> Duration { + Duration::from_secs(self.litellm_forward_keepalive_timeout) + } + pub fn log_level(&self) -> Level { self.rust_log .as_deref() From bf3bab34adea18e3088dc4a547507643ad7035f2 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 11:58:00 +0900 Subject: [PATCH 06/15] fix(gateway): read Python's KEEPALIVE_TIMEOUT for the forwarding pool gateway.launch and proxy_cli both hand KEEPALIVE_TIMEOUT to uvicorn, and the Rust child inherits that environment, so reading the same variable keeps the two timeouts in step with no supervisor change Co-Authored-By: Claude Opus 5.5 --- litellm-rust/crates/gateway/src/settings.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/litellm-rust/crates/gateway/src/settings.rs b/litellm-rust/crates/gateway/src/settings.rs index 3fad2f0cbc3..1f63e75ff51 100644 --- a/litellm-rust/crates/gateway/src/settings.rs +++ b/litellm-rust/crates/gateway/src/settings.rs @@ -10,7 +10,7 @@ pub(super) struct ForwardSettings { pub host: String, pub port: u16, rust_log: Option, - litellm_forward_keepalive_timeout: u64, + keepalive_timeout: u64, } impl Default for ForwardSettings { @@ -19,14 +19,14 @@ impl Default for ForwardSettings { host: "0.0.0.0".into(), port: 4000, rust_log: None, - litellm_forward_keepalive_timeout: 5, + keepalive_timeout: 5, } } } impl ForwardSettings { pub fn upstream_keepalive(&self) -> Duration { - Duration::from_secs(self.litellm_forward_keepalive_timeout) + Duration::from_secs(self.keepalive_timeout) } pub fn log_level(&self) -> Level { @@ -107,6 +107,18 @@ mod tests { ) } + #[rstest] + #[case::uvicorn_default(&[], Duration::from_secs(5))] + #[case::shared_with_python(&[("KEEPALIVE_TIMEOUT", "2")], Duration::from_secs(2))] + fn forwarding_reads_the_keepalive_python_uses( + #[case] variables: &[(&str, &str)], + #[case] expected: Duration, + ) { + let settings: ForwardSettings = parse(variables).unwrap(); + + assert_eq!(settings.upstream_keepalive(), expected); + } + #[rstest] fn defaults_allow_startup_without_ui_credentials() { let settings: Settings = parse(&[]).unwrap(); From d27528b10d5a97425409567129139087b7bd75a7 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 12:23:14 +0900 Subject: [PATCH 07/15] fix(gateway): keep the public Host for HTTP/2 clients HTTP/2 requests carry the host only in :authority, which the upstream URI rewrite replaced, so Python saw 127.0.0.1: and built OAuth and redirect URLs from it. HTTP/2 serving is now declared by the crate instead of arriving through workspace feature unification Co-Authored-By: Claude Opus 5.5 --- .../crates/gateway-forward/Cargo.toml | 3 +- .../crates/gateway-forward/src/forward.rs | 8 ++++++ .../crates/gateway-forward/tests/http.rs | 28 +++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/litellm-rust/crates/gateway-forward/Cargo.toml b/litellm-rust/crates/gateway-forward/Cargo.toml index e1c8fe2c5de..8e2eeea4702 100644 --- a/litellm-rust/crates/gateway-forward/Cargo.toml +++ b/litellm-rust/crates/gateway-forward/Cargo.toml @@ -9,7 +9,7 @@ repository.workspace = true [dependencies] axum.workspace = true hyper = { version = "1", features = ["http1"] } -hyper-util = { workspace = true, features = ["client-legacy", "http1", "tokio"] } +hyper-util = { workspace = true, features = ["client-legacy", "http1", "http2", "tokio"] } thiserror.workspace = true tokio = { workspace = true, features = ["io-util"] } tokio-util = { version = "0.7", features = ["rt"] } @@ -20,6 +20,7 @@ bytes.workspace = true futures-util.workspace = true http-body-util = "0.1" httparse = "1" +hyper = { version = "1", features = ["client", "http2"] } proptest.workspace = true rstest.workspace = true tokio = { workspace = true, features = ["io-util", "sync"] } diff --git a/litellm-rust/crates/gateway-forward/src/forward.rs b/litellm-rust/crates/gateway-forward/src/forward.rs index ceedfeb231b..5941d173a6f 100644 --- a/litellm-rust/crates/gateway-forward/src/forward.rs +++ b/litellm-rust/crates/gateway-forward/src/forward.rs @@ -96,6 +96,14 @@ async fn forward( mut request: Request, ) -> Result { let downstream = state.upgrades.take_handshake(&mut request)?; + // HTTP/2 carries the public host in :authority, which the upstream rewrite below replaces. + if let Some(authority) = request.uri().authority() + && !request.headers().contains_key(header::HOST) + { + let host = + HeaderValue::from_str(authority.as_str()).map_err(|_| StatusCode::BAD_REQUEST)?; + request.headers_mut().insert(header::HOST, host); + } let mut uri = state.upstream.into_parts(); uri.path_and_query = request.uri().path_and_query().cloned(); *request.uri_mut() = Uri::from_parts(uri).map_err(|_| StatusCode::BAD_GATEWAY)?; diff --git a/litellm-rust/crates/gateway-forward/tests/http.rs b/litellm-rust/crates/gateway-forward/tests/http.rs index 3635b795cf0..d5e70625c58 100644 --- a/litellm-rust/crates/gateway-forward/tests/http.rs +++ b/litellm-rust/crates/gateway-forward/tests/http.rs @@ -139,6 +139,34 @@ async fn absolute_form_targets_reach_python_as_origin_form() { assert_eq!(request.uri, "/v1/models?x=1"); } +#[rstest] +#[tokio::test] +async fn http2_clients_reach_python_with_their_public_host() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let stream = tokio::net::TcpStream::connect(gateway.address) + .await + .unwrap(); + let (mut sender, connection) = hyper::client::conn::http2::handshake( + hyper_util::rt::TokioExecutor::new(), + hyper_util::rt::TokioIo::new(stream), + ) + .await + .unwrap(); + tokio::spawn(connection); + let request = axum::http::Request::get("http://public.example/v1/models") + .body(Body::empty()) + .unwrap(); + + let response = within(sender.send_request(request)).await.unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + received.recv().await.unwrap().headers["host"], + "public.example" + ); +} + #[rstest] #[tokio::test] async fn keeps_repeated_response_headers_and_opaque_header_bytes() { From c2833d5fd74c9a911f867834eb8bd3312e2e8ce6 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 12:24:20 +0900 Subject: [PATCH 08/15] fix(gateway): disable Nagle on both forwarding hops uvicorn sets TCP_NODELAY on every connection through asyncio, so token-by-token SSE and realtime audio deltas went out at once. Without it the forwarder can hold each small write for an ACK on real networks Co-Authored-By: Claude Opus 5.5 --- litellm-rust/crates/gateway-forward/src/forward.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/litellm-rust/crates/gateway-forward/src/forward.rs b/litellm-rust/crates/gateway-forward/src/forward.rs index 5941d173a6f..bc12fb53271 100644 --- a/litellm-rust/crates/gateway-forward/src/forward.rs +++ b/litellm-rust/crates/gateway-forward/src/forward.rs @@ -9,6 +9,7 @@ use axum::{ extract::{ConnectInfo, DefaultBodyLimit, Request, State}, http::{HeaderValue, StatusCode, Uri, Version, header}, response::Response, + serve::ListenerExt, }; use hyper_util::{ client::legacy::{Client, connect::HttpConnector}, @@ -45,10 +46,12 @@ impl Forward { { return Err(Error::Upstream); } + let mut connector = HttpConnector::new(); + connector.set_nodelay(true); let client = Client::builder(TokioExecutor::new()) .pool_timer(TokioTimer::new()) .pool_idle_timeout(upstream_keepalive / 2) - .build_http(); + .build(connector); Ok(Self { upstream, client, @@ -72,6 +75,11 @@ impl Forward { let upgrades = self.upgrades.clone(); let _close_on_drop = upgrades.close_on_drop(); let draining = upgrades.clone(); + let listener = listener.tap_io(|stream| { + if let Err(error) = stream.set_nodelay(true) { + tracing::warn!(%error, "could not disable Nagle on a client connection"); + } + }); let result = axum::serve( listener, self.router() From ad7b8d9da5173f8b0d912434884113f54e1a9738 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 12:54:12 +0900 Subject: [PATCH 09/15] fix(gateway): bound the forwarding shutdown drain by Python's graceful timeout HTTP streams kept Rust draining with no limit and WebSockets were cut after a fixed 5s. Both now wait for Python to end them, up to GRACEFUL_SHUTDOWN_TIMEOUT plus the teardown headroom Python uses, so the closes Python sends reach clients Co-Authored-By: Claude Opus 5.5 --- .../crates/gateway-forward/src/forward.rs | 38 +++++++++++++++---- .../crates/gateway-forward/src/websocket.rs | 9 ----- .../gateway-forward/tests/connection.rs | 26 ++++++++++++- .../gateway-forward/tests/support/mod.rs | 14 +++++-- .../crates/gateway-forward/tests/websocket.rs | 36 ++++++++++++------ litellm-rust/crates/gateway/src/main.rs | 4 +- litellm-rust/crates/gateway/src/settings.rs | 31 +++++++++++++++ 7 files changed, 124 insertions(+), 34 deletions(-) diff --git a/litellm-rust/crates/gateway-forward/src/forward.rs b/litellm-rust/crates/gateway-forward/src/forward.rs index bc12fb53271..239ad2a1458 100644 --- a/litellm-rust/crates/gateway-forward/src/forward.rs +++ b/litellm-rust/crates/gateway-forward/src/forward.rs @@ -16,6 +16,7 @@ use hyper_util::{ rt::{TokioExecutor, TokioTimer}, }; use tokio::net::TcpListener; +use tokio_util::sync::CancellationToken; use crate::{Error, hop, websocket::Upgrades}; @@ -66,21 +67,26 @@ impl Forward { .with_state(self) } - /// Serves until `shutdown` resolves, then gives open WebSockets a bounded drain before closing them. + /// Serves until `shutdown` resolves, then lets open requests and WebSockets finish for up to + /// `drain` before closing whatever is left. Python shuts down alongside and ends long streams + /// itself, so `drain` only backstops a Python that never does. pub async fn serve( self, listener: TcpListener, shutdown: impl Future + Send + 'static, + drain: Duration, ) -> std::io::Result<()> { let upgrades = self.upgrades.clone(); let _close_on_drop = upgrades.close_on_drop(); + let stopping = CancellationToken::new(); + let stop = stopping.clone(); let draining = upgrades.clone(); let listener = listener.tap_io(|stream| { if let Err(error) = stream.set_nodelay(true) { tracing::warn!(%error, "could not disable Nagle on a client connection"); } }); - let result = axum::serve( + let server = axum::serve( listener, self.router() .into_make_service_with_connect_info::(), @@ -88,11 +94,29 @@ impl Forward { .with_graceful_shutdown(async move { shutdown.await; draining.begin_shutdown(); - }) - .await; - if result.is_err() { - upgrades.close(); - } + stop.cancel(); + }); + let drained = async { + let result = server.await; + if result.is_err() { + upgrades.close(); + } + upgrades.wait().await; + result + }; + let deadline = async { + stopping.cancelled().await; + tokio::time::sleep(drain).await; + tracing::warn!( + ?drain, + "closing connections still open after the shutdown drain" + ); + }; + let result = tokio::select! { + result = drained => result, + () = deadline => Ok(()), + }; + upgrades.close(); upgrades.wait().await; result } diff --git a/litellm-rust/crates/gateway-forward/src/websocket.rs b/litellm-rust/crates/gateway-forward/src/websocket.rs index 172ebc2da2f..d04b0434c19 100644 --- a/litellm-rust/crates/gateway-forward/src/websocket.rs +++ b/litellm-rust/crates/gateway-forward/src/websocket.rs @@ -1,5 +1,3 @@ -use std::time::Duration; - use axum::{ body::Body, extract::Request, @@ -16,8 +14,6 @@ use tokio_util::{ task::TaskTracker, }; -const DRAIN_TIMEOUT: Duration = Duration::from_secs(5); - #[derive(Clone, Default)] pub(crate) struct Upgrades { tasks: TaskTracker, @@ -28,11 +24,6 @@ pub(crate) struct Upgrades { impl Upgrades { pub fn begin_shutdown(&self) { self.draining.cancel(); - let closed = self.closed.clone(); - tokio::spawn(async move { - tokio::time::sleep(DRAIN_TIMEOUT).await; - closed.cancel(); - }); } pub fn take_handshake(&self, request: &mut Request) -> Result, StatusCode> { diff --git a/litellm-rust/crates/gateway-forward/tests/connection.rs b/litellm-rust/crates/gateway-forward/tests/connection.rs index dae3b08368b..5fc018c5b42 100644 --- a/litellm-rust/crates/gateway-forward/tests/connection.rs +++ b/litellm-rust/crates/gateway-forward/tests/connection.rs @@ -8,7 +8,7 @@ use std::{ use axum::{body::Body, extract::Request, http::StatusCode}; use http_body_util::BodyExt; use rstest::rstest; -use support::{Gateway, Raw, client, raw_upstream, unreachable_upstream, within}; +use support::{DRAIN, Gateway, Raw, client, raw_upstream, unreachable_upstream, within}; use tokio::sync::{Notify, mpsc}; /// Python stops a provider stream when it sees the client go away, so a client disconnect must @@ -143,3 +143,27 @@ async fn shutdown_lets_an_in_flight_stream_finish() { assert_eq!(body, "firstlast"); within(server).await.unwrap().unwrap(); } + +#[rstest] +#[tokio::test] +async fn shutdown_stops_waiting_for_a_stream_still_open_after_the_drain() { + let upstream = raw_upstream(|mut python: Raw| async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n5\r\nfirst\r\n") + .await; + std::future::pending::<()>().await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let response = + within(client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap())) + .await + .unwrap(); + let started = std::time::Instant::now(); + + within(gateway.shutdown()).await.unwrap().unwrap(); + + assert!(started.elapsed() >= DRAIN); + drop(response); +} diff --git a/litellm-rust/crates/gateway-forward/tests/support/mod.rs b/litellm-rust/crates/gateway-forward/tests/support/mod.rs index b68a42cb66b..db9da82efbf 100644 --- a/litellm-rust/crates/gateway-forward/tests/support/mod.rs +++ b/litellm-rust/crates/gateway-forward/tests/support/mod.rs @@ -26,6 +26,7 @@ use tokio::{ pub const WAIT: Duration = Duration::from_secs(5); pub const UVICORN_KEEPALIVE: Duration = Duration::from_secs(5); +pub const DRAIN: Duration = Duration::from_secs(1); pub async fn within(future: impl Future) -> T { tokio::time::timeout(WAIT, future).await.expect("timed out") @@ -48,9 +49,13 @@ impl Gateway { let public = TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = public.local_addr().unwrap(); let (stop, stopped) = oneshot::channel::<()>(); - let server = tokio::spawn(forward.serve(public, async { - stopped.await.ok(); - })); + let server = tokio::spawn(forward.serve( + public, + async { + stopped.await.ok(); + }, + DRAIN, + )); Self { address, stop, @@ -68,7 +73,8 @@ impl Gateway { Raw::new(TcpStream::connect(self.address).await.unwrap()) } - /// Starts graceful shutdown and returns the server task, which ends once in-flight work drains. + /// Starts graceful shutdown and returns the server task, which ends once in-flight work drains + /// or `DRAIN` passes. pub fn shutdown(self) -> JoinHandle> { self.stop.send(()).unwrap(); self.server diff --git a/litellm-rust/crates/gateway-forward/tests/websocket.rs b/litellm-rust/crates/gateway-forward/tests/websocket.rs index 02fbd1f9368..d992a6ab53f 100644 --- a/litellm-rust/crates/gateway-forward/tests/websocket.rs +++ b/litellm-rust/crates/gateway-forward/tests/websocket.rs @@ -13,7 +13,7 @@ use axum::{ use futures_util::{SinkExt, StreamExt}; use litellm_gateway_forward::PEER_HEADER; use rstest::rstest; -use support::{Gateway, raw_upstream, serve, within}; +use support::{DRAIN, Gateway, raw_upstream, serve, within}; use tokio::net::TcpStream; use tokio_tungstenite::{ WebSocketStream, accept_hdr_async, client_async, @@ -151,15 +151,31 @@ async fn shutdown_closes_open_sockets_after_drain() { let mut socket = within(connect(gateway.address, "Upgrade")).await.unwrap(); socket.next().await.unwrap().unwrap(); let started = Instant::now(); - tokio::time::timeout(Duration::from_secs(10), gateway.shutdown()) - .await - .unwrap() - .unwrap() - .unwrap(); - assert!(started.elapsed() >= Duration::from_secs(4)); + within(gateway.shutdown()).await.unwrap().unwrap(); + assert!(started.elapsed() >= DRAIN); assert!(!matches!(socket.next().await, Some(Ok(Message::Text(_))))); } +#[rstest] +#[tokio::test] +async fn shutdown_relays_the_close_python_sends_while_draining() { + let gateway = Gateway::start(echo().await).await; + let mut socket = within(connect(gateway.address, "Upgrade")).await.unwrap(); + socket.next().await.unwrap().unwrap(); + let server = gateway.shutdown(); + tokio::time::sleep(Duration::from_millis(100)).await; + socket.send(Message::text("bye")).await.unwrap(); + let Message::Close(Some(frame)) = within(socket.next()).await.unwrap().unwrap() else { + panic!("expected python's close frame"); + }; + assert_eq!(frame.code, CloseCode::Library(4000)); + while socket.next().await.is_some() {} + drop(socket); + let started = Instant::now(); + within(server).await.unwrap().unwrap(); + assert!(started.elapsed() < DRAIN / 2); +} + #[rstest] #[tokio::test] async fn refuses_new_sockets_while_draining() { @@ -178,9 +194,5 @@ async fn refuses_new_sockets_while_draining() { Message::text("still open") ); drop(open); - tokio::time::timeout(Duration::from_secs(10), server) - .await - .unwrap() - .unwrap() - .unwrap(); + within(server).await.unwrap().unwrap(); } diff --git a/litellm-rust/crates/gateway/src/main.rs b/litellm-rust/crates/gateway/src/main.rs index 8c8785a10ba..3413d11defa 100644 --- a/litellm-rust/crates/gateway/src/main.rs +++ b/litellm-rust/crates/gateway/src/main.rs @@ -56,7 +56,9 @@ async fn main() -> Result<(), Box> { let listener = tokio::net::TcpListener::bind((settings.host.as_str(), settings.port)).await?; tracing::info!(address = %listener.local_addr()?, "forwarding gateway listening"); - forward.serve(listener, shutdown_signal()).await?; + forward + .serve(listener, shutdown_signal(), settings.shutdown_drain()) + .await?; return Ok(()); } let settings: Settings = settings::from_iter(std::env::vars_os())?; diff --git a/litellm-rust/crates/gateway/src/settings.rs b/litellm-rust/crates/gateway/src/settings.rs index 1f63e75ff51..97f9c851d2d 100644 --- a/litellm-rust/crates/gateway/src/settings.rs +++ b/litellm-rust/crates/gateway/src/settings.rs @@ -11,6 +11,7 @@ pub(super) struct ForwardSettings { pub port: u16, rust_log: Option, keepalive_timeout: u64, + graceful_shutdown_timeout: Option, } impl Default for ForwardSettings { @@ -20,11 +21,27 @@ impl Default for ForwardSettings { port: 4000, rust_log: None, keepalive_timeout: 5, + graceful_shutdown_timeout: None, } } } +const DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30); +const GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM: Duration = Duration::from_secs(10); + impl ForwardSettings { + /// Python ends its own streams at `GRACEFUL_SHUTDOWN_TIMEOUT` and then tears down, so Rust + /// waits past both to relay the closes Python sends instead of cutting connections first. + pub fn shutdown_drain(&self) -> Duration { + let python = self + .graceful_shutdown_timeout + .as_deref() + .and_then(|value| value.trim().parse::().ok()) + .and_then(|seconds| Duration::try_from_secs_f64(seconds).ok()) + .unwrap_or(DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT); + python + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM + } + pub fn upstream_keepalive(&self) -> Duration { Duration::from_secs(self.keepalive_timeout) } @@ -119,6 +136,20 @@ mod tests { assert_eq!(settings.upstream_keepalive(), expected); } + #[rstest] + #[case::python_default(&[], Duration::from_secs(40))] + #[case::fractional(&[("GRACEFUL_SHUTDOWN_TIMEOUT", " 2.5 ")], Duration::from_millis(12_500))] + #[case::malformed_falls_back_like_python(&[("GRACEFUL_SHUTDOWN_TIMEOUT", "soon")], Duration::from_secs(40))] + #[case::negative(&[("GRACEFUL_SHUTDOWN_TIMEOUT", "-1")], Duration::from_secs(40))] + fn forwarding_outlasts_python_graceful_shutdown( + #[case] variables: &[(&str, &str)], + #[case] expected: Duration, + ) { + let settings: ForwardSettings = parse(variables).unwrap(); + + assert_eq!(settings.shutdown_drain(), expected); + } + #[rstest] fn defaults_allow_startup_without_ui_credentials() { let settings: Settings = parse(&[]).unwrap(); From 37c27147d85b137561d7ed68b92946913a6a7d56 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 12:54:17 +0900 Subject: [PATCH 10/15] fix(gateway): accept Python responses with more than 100 headers hyper's client rejects a response past 100 headers, which turned a completed request into a 502. LiteLLM copies every provider header into its response and uvicorn sets no limit Co-Authored-By: Claude Opus 5.5 --- .../crates/gateway-forward/src/forward.rs | 5 +++ .../crates/gateway-forward/tests/http.rs | 31 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/litellm-rust/crates/gateway-forward/src/forward.rs b/litellm-rust/crates/gateway-forward/src/forward.rs index 239ad2a1458..d783ecab8a1 100644 --- a/litellm-rust/crates/gateway-forward/src/forward.rs +++ b/litellm-rust/crates/gateway-forward/src/forward.rs @@ -22,6 +22,10 @@ use crate::{Error, hop, websocket::Upgrades}; pub const PEER_HEADER: &str = "x-litellm-rust-peer"; +// uvicorn sets no response header limit and Python copies every provider header into its +// response, so hyper's default of 100 would turn ordinary responses into 502s. +const PYTHON_RESPONSE_HEADERS: usize = 1000; + #[derive(Clone)] pub struct Forward { upstream: Uri, @@ -50,6 +54,7 @@ impl Forward { let mut connector = HttpConnector::new(); connector.set_nodelay(true); let client = Client::builder(TokioExecutor::new()) + .http1_max_headers(PYTHON_RESPONSE_HEADERS) .pool_timer(TokioTimer::new()) .pool_idle_timeout(upstream_keepalive / 2) .build(connector); diff --git a/litellm-rust/crates/gateway-forward/tests/http.rs b/litellm-rust/crates/gateway-forward/tests/http.rs index d5e70625c58..3714a111b75 100644 --- a/litellm-rust/crates/gateway-forward/tests/http.rs +++ b/litellm-rust/crates/gateway-forward/tests/http.rs @@ -167,6 +167,37 @@ async fn http2_clients_reach_python_with_their_public_host() { ); } +#[rstest] +#[tokio::test] +async fn relays_a_response_with_more_headers_than_hyper_allows_by_default() { + let upstream = serve(axum::Router::new().fallback(|| async { + let headers = (0..150).map(|index| (format!("llm_provider-x-{index}"), "v")); + let mut response = ok(); + for (name, value) in headers { + response.headers_mut().insert( + axum::http::HeaderName::from_bytes(name.as_bytes()).unwrap(), + HeaderValue::from_static(value), + ); + } + response + })) + .await; + let gateway = Gateway::start(upstream).await; + + let response = within( + client().request( + axum::http::Request::get(gateway.url("/v1/chat/completions")) + .body(Body::empty()) + .unwrap(), + ), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()["llm_provider-x-149"], "v"); +} + #[rstest] #[tokio::test] async fn keeps_repeated_response_headers_and_opaque_header_bytes() { From 7fbdc9a38debd72339011e68ffdb671dfddee410 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 13:00:38 +0900 Subject: [PATCH 11/15] test(gateway): read the many-header response with the raw client The hyper test client has the same 100-header default the fix lifts, so it rejected the relayed response itself Co-Authored-By: Claude Opus 5.5 --- .../crates/gateway-forward/tests/http.rs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/litellm-rust/crates/gateway-forward/tests/http.rs b/litellm-rust/crates/gateway-forward/tests/http.rs index 3714a111b75..78de426f521 100644 --- a/litellm-rust/crates/gateway-forward/tests/http.rs +++ b/litellm-rust/crates/gateway-forward/tests/http.rs @@ -183,19 +183,14 @@ async fn relays_a_response_with_more_headers_than_hyper_allows_by_default() { })) .await; let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; - let response = within( - client().request( - axum::http::Request::get(gateway.url("/v1/chat/completions")) - .body(Body::empty()) - .unwrap(), - ), - ) - .await - .unwrap(); + raw.send("GET /v1/chat/completions HTTP/1.1\r\nhost: public.example\r\n\r\n") + .await; - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(response.headers()["llm_provider-x-149"], "v"); + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::OK); + assert_eq!(head.header("llm_provider-x-149"), Some(&b"v"[..])); } #[rstest] From 5a5dad4b95c55f5980314543d57c0a55809c3386 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 13:12:01 +0900 Subject: [PATCH 12/15] refactor(gateway): move forward mode into gateway-forward behind an explicit env contract Forward mode now reads only LITELLM_FORWARD_UPSTREAM, _HOST, _PORT, _UPSTREAM_KEEPALIVE and _SHUTDOWN_DRAIN, all required. It no longer guesses Python's keepalive and graceful timeout from HOST, PORT, KEEPALIVE_TIMEOUT and GRACEFUL_SHUTDOWN_TIMEOUT, which were wrong whenever Python got its values from CLI flags. The gateway binary keeps only the dispatch and logger setup. Co-Authored-By: Claude Opus 5.5 --- litellm-rust/Cargo.lock | 2 + .../crates/gateway-forward/Cargo.toml | 4 +- .../crates/gateway-forward/src/error.rs | 6 ++ .../crates/gateway-forward/src/lib.rs | 17 ++++ .../crates/gateway-forward/src/settings.rs | 71 +++++++++++++ .../crates/gateway-forward/tests/settings.rs | 99 +++++++++++++++++++ litellm-rust/crates/gateway/src/main.rs | 21 +--- litellm-rust/crates/gateway/src/settings.rs | 84 +++------------- 8 files changed, 214 insertions(+), 90 deletions(-) create mode 100644 litellm-rust/crates/gateway-forward/src/settings.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/settings.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index e9dcd822713..960b33511a4 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3865,6 +3865,7 @@ version = "0.1.0" dependencies = [ "axum", "bytes", + "envy", "futures-util", "http-body-util", "httparse", @@ -3872,6 +3873,7 @@ dependencies = [ "hyper-util", "proptest", "rstest", + "serde", "thiserror 2.0.19", "tokio", "tokio-tungstenite", diff --git a/litellm-rust/crates/gateway-forward/Cargo.toml b/litellm-rust/crates/gateway-forward/Cargo.toml index 8e2eeea4702..9c51fb15842 100644 --- a/litellm-rust/crates/gateway-forward/Cargo.toml +++ b/litellm-rust/crates/gateway-forward/Cargo.toml @@ -8,8 +8,10 @@ repository.workspace = true [dependencies] axum.workspace = true +envy = "0.4.2" hyper = { version = "1", features = ["http1"] } hyper-util = { workspace = true, features = ["client-legacy", "http1", "http2", "tokio"] } +serde.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["io-util"] } tokio-util = { version = "0.7", features = ["rt"] } @@ -23,5 +25,5 @@ httparse = "1" hyper = { version = "1", features = ["client", "http2"] } proptest.workspace = true rstest.workspace = true -tokio = { workspace = true, features = ["io-util", "sync"] } +tokio = { workspace = true, features = ["io-util", "sync", "test-util"] } tokio-tungstenite.workspace = true diff --git a/litellm-rust/crates/gateway-forward/src/error.rs b/litellm-rust/crates/gateway-forward/src/error.rs index abbe415b39a..ffd63b2ecab 100644 --- a/litellm-rust/crates/gateway-forward/src/error.rs +++ b/litellm-rust/crates/gateway-forward/src/error.rs @@ -6,4 +6,10 @@ pub enum Error { "LITELLM_FORWARD_UPSTREAM must be a loopback IPv4 http origin without a path, query or credentials" )] Upstream, + #[error("{0} must be set in forward mode")] + MissingSetting(String), + #[error("invalid forwarding setting: {0}")] + InvalidSetting(String), + #[error("forwarding gateway I/O failed: {0}")] + Io(#[from] std::io::Error), } diff --git a/litellm-rust/crates/gateway-forward/src/lib.rs b/litellm-rust/crates/gateway-forward/src/lib.rs index 606c42200e5..b473fbca202 100644 --- a/litellm-rust/crates/gateway-forward/src/lib.rs +++ b/litellm-rust/crates/gateway-forward/src/lib.rs @@ -1,7 +1,24 @@ mod error; mod forward; mod hop; +mod settings; mod websocket; pub use error::Error; pub use forward::{Forward, PEER_HEADER}; +pub use settings::Settings; + +/// Binds the public listener and forwards to Python until `shutdown` resolves, then drains for +/// `settings.shutdown_drain`. +pub async fn run( + settings: Settings, + shutdown: impl Future + Send + 'static, +) -> Result<(), Error> { + let forward = Forward::new(&settings.upstream, settings.upstream_keepalive)?; + let listener = tokio::net::TcpListener::bind((settings.host.as_str(), settings.port)).await?; + tracing::info!(address = %listener.local_addr()?, "forwarding gateway listening"); + forward + .serve(listener, shutdown, settings.shutdown_drain) + .await?; + Ok(()) +} diff --git a/litellm-rust/crates/gateway-forward/src/settings.rs b/litellm-rust/crates/gateway-forward/src/settings.rs new file mode 100644 index 00000000000..6093ea33561 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/settings.rs @@ -0,0 +1,71 @@ +use std::{ffi::OsString, time::Duration}; + +use serde::Deserialize; + +use crate::Error; + +const UPSTREAM: &str = "LITELLM_FORWARD_UPSTREAM"; + +#[derive(Deserialize)] +struct Environment { + litellm_forward_upstream: String, + litellm_forward_host: String, + litellm_forward_port: u16, + litellm_forward_upstream_keepalive: f64, + litellm_forward_shutdown_drain: f64, +} + +/// Everything forward mode runs with. The Python supervisor resolves each value from the settings +/// Python actually runs with, so none of them is guessed or defaulted here. +#[derive(Debug, Clone, PartialEq)] +pub struct Settings { + pub upstream: String, + pub host: String, + pub port: u16, + pub upstream_keepalive: Duration, + pub shutdown_drain: Duration, +} + +impl Settings { + /// `None` when `LITELLM_FORWARD_UPSTREAM` is unset, which means the gateway is not forwarding. + pub fn from_env() -> Result, Error> { + Self::from_vars(std::env::vars_os()) + } + + pub fn from_vars( + variables: impl IntoIterator, + ) -> Result, Error> { + let variables: Vec<(String, String)> = variables + .into_iter() + .filter_map(|(key, value)| Some((key.into_string().ok()?, value.into_string().ok()?))) + .collect(); + if !variables.iter().any(|(key, _)| key == UPSTREAM) { + return Ok(None); + } + let environment: Environment = envy::from_iter(variables).map_err(|error| match error { + envy::Error::MissingValue(field) => Error::MissingSetting(field.to_uppercase()), + envy::Error::Custom(message) => Error::InvalidSetting(message), + })?; + Ok(Some(Self { + upstream: environment.litellm_forward_upstream, + host: environment.litellm_forward_host, + port: environment.litellm_forward_port, + upstream_keepalive: seconds( + "LITELLM_FORWARD_UPSTREAM_KEEPALIVE", + environment.litellm_forward_upstream_keepalive, + )?, + shutdown_drain: seconds( + "LITELLM_FORWARD_SHUTDOWN_DRAIN", + environment.litellm_forward_shutdown_drain, + )?, + })) + } +} + +fn seconds(variable: &str, value: f64) -> Result { + Duration::try_from_secs_f64(value).map_err(|_| { + Error::InvalidSetting(format!( + "{variable} must be a non-negative number of seconds, got {value}" + )) + }) +} diff --git a/litellm-rust/crates/gateway-forward/tests/settings.rs b/litellm-rust/crates/gateway-forward/tests/settings.rs new file mode 100644 index 00000000000..540e1d9844d --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/settings.rs @@ -0,0 +1,99 @@ +use std::{ffi::OsString, time::Duration}; + +use litellm_gateway_forward::{Error, Settings}; +use rstest::rstest; + +const COMPLETE: [(&str, &str); 5] = [ + ("LITELLM_FORWARD_UPSTREAM", "http://127.0.0.1:53211"), + ("LITELLM_FORWARD_HOST", "0.0.0.0"), + ("LITELLM_FORWARD_PORT", "4000"), + ("LITELLM_FORWARD_UPSTREAM_KEEPALIVE", "2"), + ("LITELLM_FORWARD_SHUTDOWN_DRAIN", "40"), +]; + +fn parse(variables: &[(&str, &str)]) -> Result, Error> { + Settings::from_vars( + variables + .iter() + .map(|(key, value)| (OsString::from(key), OsString::from(value))), + ) +} + +fn without(variable: &str) -> Vec<(&'static str, &'static str)> { + COMPLETE + .into_iter() + .filter(|(key, _)| *key != variable) + .collect() +} + +fn with<'a>(variable: &'a str, value: &'a str) -> Vec<(&'a str, &'a str)> { + without(variable) + .into_iter() + .chain([(variable, value)]) + .collect() +} + +#[rstest] +fn reads_every_variable() { + assert_eq!( + parse(&COMPLETE).unwrap(), + Some(Settings { + upstream: "http://127.0.0.1:53211".into(), + host: "0.0.0.0".into(), + port: 4000, + upstream_keepalive: Duration::from_secs(2), + shutdown_drain: Duration::from_secs(40), + }) + ); +} + +#[rstest] +fn without_an_upstream_the_gateway_is_not_forwarding() { + assert_eq!(parse(&without("LITELLM_FORWARD_UPSTREAM")).unwrap(), None); +} + +#[rstest] +#[case::host("LITELLM_FORWARD_HOST")] +#[case::port("LITELLM_FORWARD_PORT")] +#[case::upstream_keepalive("LITELLM_FORWARD_UPSTREAM_KEEPALIVE")] +#[case::shutdown_drain("LITELLM_FORWARD_SHUTDOWN_DRAIN")] +fn a_missing_variable_fails_naming_it(#[case] variable: &str) { + let error = parse(&without(variable)).unwrap_err(); + + assert!(error.to_string().contains(variable), "{error}"); +} + +#[rstest] +#[case::port_overflow("LITELLM_FORWARD_PORT", "65536")] +#[case::port_text("LITELLM_FORWARD_PORT", "http")] +#[case::keepalive_text("LITELLM_FORWARD_UPSTREAM_KEEPALIVE", "soon")] +#[case::keepalive_negative("LITELLM_FORWARD_UPSTREAM_KEEPALIVE", "-1")] +#[case::drain_text("LITELLM_FORWARD_SHUTDOWN_DRAIN", "soon")] +#[case::drain_negative("LITELLM_FORWARD_SHUTDOWN_DRAIN", "-1")] +#[case::drain_infinite("LITELLM_FORWARD_SHUTDOWN_DRAIN", "inf")] +fn an_invalid_variable_fails_naming_it(#[case] variable: &str, #[case] value: &str) { + let error = parse(&with(variable, value)).unwrap_err(); + + assert!(error.to_string().contains(variable), "{error}"); +} + +#[rstest] +fn fractional_drain_seconds_are_used_as_given() { + let settings = parse(&with("LITELLM_FORWARD_SHUTDOWN_DRAIN", "12.5")) + .unwrap() + .unwrap(); + + assert_eq!(settings.shutdown_drain, Duration::from_millis(12_500)); +} + +#[rstest] +#[case::keepalive("KEEPALIVE_TIMEOUT")] +#[case::graceful_shutdown("GRACEFUL_SHUTDOWN_TIMEOUT")] +#[case::host("HOST")] +#[case::port("PORT")] +fn python_settings_are_never_read(#[case] variable: &str) { + let mut variables = COMPLETE.to_vec(); + variables.push((variable, "9")); + + assert_eq!(parse(&variables).unwrap(), parse(&COMPLETE).unwrap()); +} diff --git a/litellm-rust/crates/gateway/src/main.rs b/litellm-rust/crates/gateway/src/main.rs index 3413d11defa..83f43f43b73 100644 --- a/litellm-rust/crates/gateway/src/main.rs +++ b/litellm-rust/crates/gateway/src/main.rs @@ -10,7 +10,7 @@ use serde_json::json; mod settings; -use settings::{Settings, UiSettings}; +use settings::{LogSettings, Settings, UiSettings}; struct StderrSink { level: Level, @@ -41,24 +41,13 @@ impl Sink for StderrSink { #[tokio::main] async fn main() -> Result<(), Box> { - if let Some(upstream) = std::env::var_os("LITELLM_FORWARD_UPSTREAM") { - let settings: settings::ForwardSettings = settings::from_iter(std::env::vars_os())?; + if let Some(forward) = litellm_gateway_forward::Settings::from_env()? { + let log: LogSettings = settings::from_iter(std::env::vars_os())?; Logger::new(StderrSink { - level: settings.log_level(), + level: log.log_level(), }) .install_global()?; - let forward = litellm_gateway_forward::Forward::new( - &upstream - .into_string() - .map_err(|_| "invalid upstream encoding")?, - settings.upstream_keepalive(), - )?; - let listener = - tokio::net::TcpListener::bind((settings.host.as_str(), settings.port)).await?; - tracing::info!(address = %listener.local_addr()?, "forwarding gateway listening"); - forward - .serve(listener, shutdown_signal(), settings.shutdown_drain()) - .await?; + litellm_gateway_forward::run(forward, shutdown_signal()).await?; return Ok(()); } let settings: Settings = settings::from_iter(std::env::vars_os())?; diff --git a/litellm-rust/crates/gateway/src/settings.rs b/litellm-rust/crates/gateway/src/settings.rs index 97f9c851d2d..c2c8bc30934 100644 --- a/litellm-rust/crates/gateway/src/settings.rs +++ b/litellm-rust/crates/gateway/src/settings.rs @@ -1,59 +1,26 @@ -use std::{ffi::OsString, time::Duration}; +use std::ffi::OsString; use litellm_secret_value::SecretValue; use litellm_tracing::Level; use serde::{Deserialize, de::DeserializeOwned}; #[derive(Deserialize)] -#[serde(default)] -pub(super) struct ForwardSettings { - pub host: String, - pub port: u16, +pub(super) struct LogSettings { rust_log: Option, - keepalive_timeout: u64, - graceful_shutdown_timeout: Option, } -impl Default for ForwardSettings { - fn default() -> Self { - Self { - host: "0.0.0.0".into(), - port: 4000, - rust_log: None, - keepalive_timeout: 5, - graceful_shutdown_timeout: None, - } - } -} - -const DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30); -const GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM: Duration = Duration::from_secs(10); - -impl ForwardSettings { - /// Python ends its own streams at `GRACEFUL_SHUTDOWN_TIMEOUT` and then tears down, so Rust - /// waits past both to relay the closes Python sends instead of cutting connections first. - pub fn shutdown_drain(&self) -> Duration { - let python = self - .graceful_shutdown_timeout - .as_deref() - .and_then(|value| value.trim().parse::().ok()) - .and_then(|seconds| Duration::try_from_secs_f64(seconds).ok()) - .unwrap_or(DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT); - python + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM - } - - pub fn upstream_keepalive(&self) -> Duration { - Duration::from_secs(self.keepalive_timeout) - } - +impl LogSettings { pub fn log_level(&self) -> Level { - self.rust_log - .as_deref() - .and_then(|value| value.parse().ok()) - .unwrap_or(Level::INFO) + log_level(self.rust_log.as_deref()) } } +fn log_level(rust_log: Option<&str>) -> Level { + rust_log + .and_then(|value| value.parse().ok()) + .unwrap_or(Level::INFO) +} + #[derive(Deserialize)] #[serde(default)] pub(super) struct Settings { @@ -76,10 +43,7 @@ impl Default for Settings { impl Settings { pub fn log_level(&self) -> Level { - self.rust_log - .as_deref() - .and_then(|value| value.parse().ok()) - .unwrap_or(Level::INFO) + log_level(self.rust_log.as_deref()) } } @@ -124,32 +88,6 @@ mod tests { ) } - #[rstest] - #[case::uvicorn_default(&[], Duration::from_secs(5))] - #[case::shared_with_python(&[("KEEPALIVE_TIMEOUT", "2")], Duration::from_secs(2))] - fn forwarding_reads_the_keepalive_python_uses( - #[case] variables: &[(&str, &str)], - #[case] expected: Duration, - ) { - let settings: ForwardSettings = parse(variables).unwrap(); - - assert_eq!(settings.upstream_keepalive(), expected); - } - - #[rstest] - #[case::python_default(&[], Duration::from_secs(40))] - #[case::fractional(&[("GRACEFUL_SHUTDOWN_TIMEOUT", " 2.5 ")], Duration::from_millis(12_500))] - #[case::malformed_falls_back_like_python(&[("GRACEFUL_SHUTDOWN_TIMEOUT", "soon")], Duration::from_secs(40))] - #[case::negative(&[("GRACEFUL_SHUTDOWN_TIMEOUT", "-1")], Duration::from_secs(40))] - fn forwarding_outlasts_python_graceful_shutdown( - #[case] variables: &[(&str, &str)], - #[case] expected: Duration, - ) { - let settings: ForwardSettings = parse(variables).unwrap(); - - assert_eq!(settings.shutdown_drain(), expected); - } - #[rstest] fn defaults_allow_startup_without_ui_credentials() { let settings: Settings = parse(&[]).unwrap(); From e34c874f91dd49474c1282ec4f7ef4ade41e95fa Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 13:16:55 +0900 Subject: [PATCH 13/15] test(gateway): organize forwarding tests by traffic shape Split http.rs into exchange, stream, upload and client_identity, rename connection.rs to lifecycle, and add slow_head, idle_stream and download. New cases cover minutes-long waits for headers and idle MCP streams on paused time, binary framed streams, 1xx, redirects with cookies and range requests. Co-Authored-By: Claude Opus 5.5 --- .../gateway-forward/tests/client_identity.rs | 68 ++++++ .../crates/gateway-forward/tests/download.rs | 80 +++++++ .../tests/{http.rs => exchange.rs} | 205 ++++++------------ .../gateway-forward/tests/idle_stream.rs | 82 +++++++ .../tests/{connection.rs => lifecycle.rs} | 0 .../crates/gateway-forward/tests/slow_head.rs | 41 ++++ .../crates/gateway-forward/tests/stream.rs | 136 ++++++++++++ .../crates/gateway-forward/tests/upload.rs | 92 ++++++++ 8 files changed, 566 insertions(+), 138 deletions(-) create mode 100644 litellm-rust/crates/gateway-forward/tests/client_identity.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/download.rs rename litellm-rust/crates/gateway-forward/tests/{http.rs => exchange.rs} (63%) create mode 100644 litellm-rust/crates/gateway-forward/tests/idle_stream.rs rename litellm-rust/crates/gateway-forward/tests/{connection.rs => lifecycle.rs} (100%) create mode 100644 litellm-rust/crates/gateway-forward/tests/slow_head.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/stream.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/upload.rs diff --git a/litellm-rust/crates/gateway-forward/tests/client_identity.rs b/litellm-rust/crates/gateway-forward/tests/client_identity.rs new file mode 100644 index 00000000000..e4d502fd01c --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/client_identity.rs @@ -0,0 +1,68 @@ +mod support; + +use axum::{body::Body, http::StatusCode, response::Response}; +use litellm_gateway_forward::PEER_HEADER; +use rstest::rstest; +use support::{Gateway, recording, within}; + +fn ok() -> Response { + Response::new(Body::from("ok")) +} + +#[rstest] +#[tokio::test] +async fn python_sees_forwarding_headers_untouched_and_the_tcp_peer_rust_saw() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send( + "GET /sso/callback HTTP/1.1\r\n\ + host: public.example\r\n\ + x-forwarded-for: 192.0.2.1\r\n\ + x-forwarded-for: 192.0.2.2\r\n\ + x-forwarded-proto: https\r\n\ + forwarded: for=192.0.2.1;proto=https\r\n\ + x-litellm-rust-peer: 10.1.2.3:1\r\n\r\n", + ) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + + let request = received.recv().await.unwrap(); + let forwarded_for: Vec<_> = request.headers.get_all("x-forwarded-for").iter().collect(); + assert_eq!(forwarded_for, ["192.0.2.1", "192.0.2.2"]); + assert_eq!(request.headers["x-forwarded-proto"], "https"); + assert_eq!(request.headers["forwarded"], "for=192.0.2.1;proto=https"); + let peers: Vec<_> = request.headers.get_all(PEER_HEADER).iter().collect(); + assert_eq!(peers, [raw.local_address().to_string().as_str()]); +} + +#[rstest] +#[tokio::test] +async fn http2_clients_reach_python_with_their_public_host() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let stream = tokio::net::TcpStream::connect(gateway.address) + .await + .unwrap(); + let (mut sender, connection) = hyper::client::conn::http2::handshake( + hyper_util::rt::TokioExecutor::new(), + hyper_util::rt::TokioIo::new(stream), + ) + .await + .unwrap(); + tokio::spawn(connection); + let request = axum::http::Request::get("http://public.example/v1/models") + .body(Body::empty()) + .unwrap(); + + let response = within(sender.send_request(request)).await.unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + received.recv().await.unwrap().headers["host"], + "public.example" + ); +} diff --git a/litellm-rust/crates/gateway-forward/tests/download.rs b/litellm-rust/crates/gateway-forward/tests/download.rs new file mode 100644 index 00000000000..ea22f6f610c --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/download.rs @@ -0,0 +1,80 @@ +mod support; + +use std::sync::Arc; + +use axum::{body::Body, extract::Request, http::StatusCode, response::Response}; +use http_body_util::BodyExt; +use rstest::rstest; +use support::{Gateway, client, raw_upstream, recording, within}; +use tokio::sync::Notify; + +#[rstest] +#[tokio::test] +async fn streams_a_body_without_a_content_length_as_python_produces_it() { + let first: &[u8] = &[0xff, 0xfb, 0x90, 0x64, 0x00, 0x0d, 0x0a]; + let rest = vec![0xa5; 64 * 1024]; + let finish = Arc::new(Notify::new()); + let finished = finish.clone(); + let tail = rest.clone(); + let upstream = raw_upstream(move |mut python| { + let finished = finished.clone(); + let tail = tail.clone(); + async move { + python.request_head().await.unwrap(); + python + .send([&b"HTTP/1.1 200 OK\r\ncontent-type: audio/mpeg\r\ntransfer-encoding: chunked\r\n\r\n7\r\n"[..], first, b"\r\n"].concat()) + .await; + finished.notified().await; + python + .send([format!("{:x}\r\n", tail.len()).as_bytes(), &tail, b"\r\n0\r\n\r\n"].concat()) + .await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut response = within( + client().request( + Request::post(gateway.url("/v1/audio/speech")) + .body(Body::empty()) + .unwrap(), + ), + ) + .await + .unwrap(); + + assert!(!response.headers().contains_key("content-length")); + let frame = within(response.body_mut().frame()).await.unwrap().unwrap(); + assert_eq!(frame.into_data().unwrap(), first); + finish.notify_one(); + let body = within(response.into_body().collect()) + .await + .unwrap() + .to_bytes(); + assert_eq!(body, rest); +} + +#[rstest] +#[tokio::test] +async fn range_requests_and_partial_content_pass_unchanged() { + let (upstream, mut received) = recording(|| -> Response { + Response::builder() + .status(StatusCode::PARTIAL_CONTENT) + .header("content-range", "bytes 2-5/10") + .header("accept-ranges", "bytes") + .body(Body::from("2345")) + .unwrap() + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET /v1/videos/video_1/content HTTP/1.1\r\nhost: x\r\nrange: bytes=2-5\r\nif-range: \"v1\"\r\n\r\n") + .await; + + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::PARTIAL_CONTENT); + assert_eq!(head.header("content-range"), Some(&b"bytes 2-5/10"[..])); + assert_eq!(raw.body(head.content_length().unwrap()).await, b"2345"); + let request = received.recv().await.unwrap(); + assert_eq!(request.headers["range"], "bytes=2-5"); + assert_eq!(request.headers["if-range"], "\"v1\""); +} diff --git a/litellm-rust/crates/gateway-forward/tests/http.rs b/litellm-rust/crates/gateway-forward/tests/exchange.rs similarity index 63% rename from litellm-rust/crates/gateway-forward/tests/http.rs rename to litellm-rust/crates/gateway-forward/tests/exchange.rs index 78de426f521..bc8564016de 100644 --- a/litellm-rust/crates/gateway-forward/tests/http.rs +++ b/litellm-rust/crates/gateway-forward/tests/exchange.rs @@ -1,19 +1,14 @@ mod support; -use std::convert::Infallible; - use axum::{ Router, - body::{Body, Bytes}, + body::Body, extract::Request, http::{HeaderValue, StatusCode}, response::Response, }; -use http_body_util::BodyExt; -use litellm_gateway_forward::PEER_HEADER; use rstest::rstest; use support::{Gateway, client, raw_upstream, recording, serve, within}; -use tokio::sync::mpsc; fn ok() -> Response { Response::new(Body::from("ok")) @@ -38,11 +33,6 @@ async fn forwards_the_request_as_the_client_sent_it_minus_hop_headers() { "PATCH /v1/messages?x=a%2Fb&x=c HTTP/1.1\r\n\ host: public.example\r\n\ authorization: Bearer test\r\n\ - x-forwarded-for: 192.0.2.1\r\n\ - x-forwarded-for: 192.0.2.2\r\n\ - x-forwarded-proto: https\r\n\ - forwarded: for=192.0.2.1;proto=https\r\n\ - x-litellm-rust-peer: 10.1.2.3:1\r\n\ connection: x-remove, keep-alive\r\n\ x-remove: remove\r\n\ proxy-authorization: remove\r\n\ @@ -62,12 +52,6 @@ async fn forwards_the_request_as_the_client_sent_it_minus_hop_headers() { assert_eq!(request.uri, "/v1/messages?x=a%2Fb&x=c"); assert_eq!(request.headers["host"], "public.example"); assert_eq!(request.headers["authorization"], "Bearer test"); - let forwarded_for: Vec<_> = request.headers.get_all("x-forwarded-for").iter().collect(); - assert_eq!(forwarded_for, ["192.0.2.1", "192.0.2.2"]); - assert_eq!(request.headers["x-forwarded-proto"], "https"); - assert_eq!(request.headers["forwarded"], "for=192.0.2.1;proto=https"); - let peers: Vec<_> = request.headers.get_all(PEER_HEADER).iter().collect(); - assert_eq!(peers, [raw.local_address().to_string().as_str()]); assert!(!request.headers.contains_key("x-remove")); assert!(!request.headers.contains_key("proxy-authorization")); assert!(!request.headers.contains_key("te")); @@ -139,34 +123,6 @@ async fn absolute_form_targets_reach_python_as_origin_form() { assert_eq!(request.uri, "/v1/models?x=1"); } -#[rstest] -#[tokio::test] -async fn http2_clients_reach_python_with_their_public_host() { - let (upstream, mut received) = recording(ok).await; - let gateway = Gateway::start(upstream).await; - let stream = tokio::net::TcpStream::connect(gateway.address) - .await - .unwrap(); - let (mut sender, connection) = hyper::client::conn::http2::handshake( - hyper_util::rt::TokioExecutor::new(), - hyper_util::rt::TokioIo::new(stream), - ) - .await - .unwrap(); - tokio::spawn(connection); - let request = axum::http::Request::get("http://public.example/v1/models") - .body(Body::empty()) - .unwrap(); - - let response = within(sender.send_request(request)).await.unwrap(); - - assert_eq!(response.status(), StatusCode::OK); - assert_eq!( - received.recv().await.unwrap().headers["host"], - "public.example" - ); -} - #[rstest] #[tokio::test] async fn relays_a_response_with_more_headers_than_hyper_allows_by_default() { @@ -224,58 +180,6 @@ async fn keeps_repeated_response_headers_and_opaque_header_bytes() { ); } -#[rstest] -#[tokio::test] -async fn streams_upload_and_response_incrementally_without_a_body_limit() { - let upstream = serve( - Router::new().fallback(|request: Request| async { Response::new(request.into_body()) }), - ) - .await; - let gateway = Gateway::start(upstream).await; - let (sender, receiver) = mpsc::channel::>(1); - let stream = futures_util::stream::unfold(receiver, |mut receiver| async { - receiver.recv().await.map(|chunk| (chunk, receiver)) - }); - sender.send(Ok(Bytes::from_static(b"first"))).await.unwrap(); - let mut response = within( - client().request( - Request::post(gateway.url("/v1/files")) - .body(Body::from_stream(stream)) - .unwrap(), - ), - ) - .await - .unwrap(); - let frame = within(response.body_mut().frame()).await.unwrap().unwrap(); - assert_eq!(frame.into_data().unwrap(), "first"); - let large = Bytes::from(vec![b'x'; 8 * 1024 * 1024]); - sender.send(Ok(large.clone())).await.unwrap(); - drop(sender); - let rest = within(response.into_body().collect()) - .await - .unwrap() - .to_bytes(); - assert_eq!(rest, large); -} - -#[rstest] -#[tokio::test] -async fn answers_expect_continue_so_large_uploads_start() { - let (upstream, mut received) = recording(ok).await; - let gateway = Gateway::start(upstream).await; - let mut raw = gateway.connect().await; - raw.send("POST /v1/audio/transcriptions HTTP/1.1\r\nhost: x\r\nexpect: 100-continue\r\ncontent-length: 4\r\n\r\n") - .await; - let interim = within(raw.response_head()).await.unwrap(); - assert_eq!(interim.status, StatusCode::CONTINUE); - raw.send("data").await; - assert_eq!( - within(raw.response_head()).await.unwrap().status, - StatusCode::OK - ); - assert_eq!(received.recv().await.unwrap().body, "data"); -} - #[rstest] #[tokio::test] async fn answers_pipelined_requests_in_order() { @@ -318,37 +222,83 @@ async fn bodiless_statuses_keep_the_connection_reusable(#[case] status: StatusCo #[rstest] #[tokio::test] -async fn relays_a_body_delimited_by_the_upstream_closing() { +async fn informational_responses_keep_the_connection_reusable() { let upstream = raw_upstream(|mut python| async move { - python.request_head().await.unwrap(); - python - .send("HTTP/1.1 200 OK\r\nconnection: close\r\n\r\nuntil eof") - .await; + while let Some(head) = python.request_head().await { + match head.path.as_str() { + "/hints" => { + python + .send("HTTP/1.1 103 Early Hints\r\nlink: ; rel=preload\r\n\r\nHTTP/1.1 200 OK\r\ncontent-length: 5\r\n\r\nfinal") + .await + } + _ => { + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 4\r\n\r\nnext") + .await + } + } + } }) .await; let gateway = Gateway::start(upstream).await; - let response = client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap()); - let body = within(response).await.unwrap().into_body().collect(); - assert_eq!(within(body).await.unwrap().to_bytes(), "until eof"); + let mut raw = gateway.connect().await; + raw.send("GET /hints HTTP/1.1\r\nhost: x\r\n\r\nGET /next HTTP/1.1\r\nhost: x\r\n\r\n") + .await; + let mut hinted = within(raw.response_head()).await.unwrap(); + while hinted.status.is_informational() { + hinted = within(raw.response_head()).await.unwrap(); + } + assert_eq!(hinted.status, StatusCode::OK); + assert_eq!(raw.body(hinted.content_length().unwrap()).await, b"final"); + let next = within(raw.response_head()).await.unwrap(); + assert_eq!(raw.body(next.content_length().unwrap()).await, b"next"); } #[rstest] +#[case::found( + StatusCode::FOUND, + "https://idp.example/authorize?state=a%2Fb&redirect_uri=https%3A%2F%2Fpublic.example%2Fsso%2Fcallback" +)] +#[case::see_other(StatusCode::SEE_OTHER, "/ui/?login=success")] +#[case::temporary(StatusCode::TEMPORARY_REDIRECT, "./mcp/oauth/authorize?client_id=x")] #[tokio::test] -async fn an_upstream_dying_mid_stream_fails_the_client_body_instead_of_ending_it() { - let upstream = raw_upstream(|mut python| async move { - python.request_head().await.unwrap(); - python - .send("HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\n\r\n6\r\ndata: \r\n") - .await; +async fn redirects_reach_the_client_with_location_and_cookies_unchanged( + #[case] status: StatusCode, + #[case] location: &'static str, +) { + let (upstream, _received) = recording(move || { + Response::builder() + .status(status) + .header("location", location) + .header( + "set-cookie", + "token=a%2Fb; Path=/; HttpOnly; Secure; SameSite=Lax", + ) + .header("set-cookie", "state=; Max-Age=0") + .body(Body::empty()) + .unwrap() }) .await; let gateway = Gateway::start(upstream).await; - let response = - within(client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap())) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::OK); - assert!(within(response.into_body().collect()).await.is_err()); + let mut raw = gateway.connect().await; + raw.send("GET /sso/key/generate HTTP/1.1\r\nhost: public.example\r\n\r\n") + .await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, status); + assert_eq!(head.header("location"), Some(location.as_bytes())); + let cookies: Vec<_> = head + .headers + .iter() + .filter(|(name, _)| name == "set-cookie") + .map(|(_, value)| value.as_slice()) + .collect(); + assert_eq!( + cookies, + [ + &b"token=a%2Fb; Path=/; HttpOnly; Secure; SameSite=Lax"[..], + b"state=; Max-Age=0" + ] + ); } #[rstest] @@ -368,24 +318,3 @@ async fn non_websocket_upgrades_reach_python_as_plain_requests() { assert!(!request.headers.contains_key("connection")); assert!(!request.headers.contains_key("http2-settings")); } - -#[rstest] -#[case::get("GET")] -#[case::delete("DELETE")] -#[case::options("OPTIONS")] -#[case::post("POST")] -#[tokio::test] -async fn forwards_a_chunked_body_on_any_method(#[case] method: &str) { - let (upstream, mut received) = recording(ok).await; - let gateway = Gateway::start(upstream).await; - let mut raw = gateway.connect().await; - raw.send(format!( - "{method} /v1/search HTTP/1.1\r\nhost: x\r\ntransfer-encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\n\r\n" - )) - .await; - assert_eq!( - within(raw.response_head()).await.unwrap().status, - StatusCode::OK - ); - assert_eq!(received.recv().await.unwrap().body, "body"); -} diff --git a/litellm-rust/crates/gateway-forward/tests/idle_stream.rs b/litellm-rust/crates/gateway-forward/tests/idle_stream.rs new file mode 100644 index 00000000000..d0ce27fa6a9 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/idle_stream.rs @@ -0,0 +1,82 @@ +mod support; + +use std::time::Duration; + +use axum::{ + body::Body, + extract::Request, + http::StatusCode, + response::{IntoResponse, Response}, +}; +use http_body_util::BodyExt; +use rstest::rstest; +use support::{Gateway, client, raw_upstream, recording, within}; +use tokio::{sync::mpsc, time::Instant}; + +/// An MCP client holds a GET stream open for server-initiated messages, which may not come for +/// many minutes, so neither hop may close a stream for being idle. +#[rstest] +#[tokio::test(start_paused = true)] +async fn an_idle_mcp_stream_stays_open_in_both_directions() { + let idle = Duration::from_secs(20 * 60); + let (session, mut sessions) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python| { + let session = session.clone(); + async move { + let head = python.request_head().await.unwrap(); + session + .send(head.header("mcp-session-id").map(<[u8]>::to_vec)) + .unwrap(); + python + .send("HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\nmcp-session-id: session-1\r\ntransfer-encoding: chunked\r\n\r\n") + .await; + tokio::time::sleep(idle).await; + python.send("c\r\ndata: ping\n\n\r\n").await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let started = Instant::now(); + + let mut response = client() + .request( + Request::get(gateway.url("/github/mcp")) + .header("accept", "text/event-stream") + .header("mcp-session-id", "session-1") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.headers()["mcp-session-id"], "session-1"); + assert_eq!( + sessions.recv().await.unwrap().as_deref(), + Some(&b"session-1"[..]) + ); + let frame = response.body_mut().frame().await.unwrap().unwrap(); + assert_eq!(frame.into_data().unwrap(), "data: ping\n\n"); + assert!(started.elapsed() >= idle); +} + +#[rstest] +#[tokio::test] +async fn ends_a_session_with_a_bodiless_delete() { + let (upstream, mut received) = recording(|| -> Response { + ([("mcp-session-id", "session-1")], StatusCode::NO_CONTENT).into_response() + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("DELETE /mcp HTTP/1.1\r\nhost: x\r\nmcp-session-id: session-1\r\n\r\n") + .await; + + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::NO_CONTENT); + assert_eq!(head.header("mcp-session-id"), Some(&b"session-1"[..])); + let request = received.recv().await.unwrap(); + assert_eq!(request.method, "DELETE"); + assert_eq!(request.headers["mcp-session-id"], "session-1"); + assert!(!request.headers.contains_key("transfer-encoding")); + assert!(request.body.is_empty()); +} diff --git a/litellm-rust/crates/gateway-forward/tests/connection.rs b/litellm-rust/crates/gateway-forward/tests/lifecycle.rs similarity index 100% rename from litellm-rust/crates/gateway-forward/tests/connection.rs rename to litellm-rust/crates/gateway-forward/tests/lifecycle.rs diff --git a/litellm-rust/crates/gateway-forward/tests/slow_head.rs b/litellm-rust/crates/gateway-forward/tests/slow_head.rs new file mode 100644 index 00000000000..764733aa524 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/slow_head.rs @@ -0,0 +1,41 @@ +mod support; + +use std::time::Duration; + +use axum::{body::Body, extract::Request, http::StatusCode}; +use http_body_util::BodyExt; +use rstest::rstest; +use support::{Gateway, client, raw_upstream}; +use tokio::time::Instant; + +/// Reasoning models with `stream=false`, image and video generation, and model health checks send +/// no response headers until the provider finishes, which can take many minutes. +#[rstest] +#[tokio::test(start_paused = true)] +async fn waits_for_response_headers_as_long_as_python_takes() { + let thinking = Duration::from_secs(30 * 60); + let upstream = raw_upstream(move |mut python| async move { + python.request_head().await.unwrap(); + tokio::time::sleep(thinking).await; + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 5\r\n\r\nimage") + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let started = Instant::now(); + + let response = client() + .request( + Request::post(gateway.url("/v1/images/generations")) + .body(Body::from("{}")) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + assert!(started.elapsed() >= thinking); + let body = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(body, "image"); +} diff --git a/litellm-rust/crates/gateway-forward/tests/stream.rs b/litellm-rust/crates/gateway-forward/tests/stream.rs new file mode 100644 index 00000000000..b533a8544a6 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/stream.rs @@ -0,0 +1,136 @@ +mod support; + +use std::sync::Arc; + +use axum::{body::Body, extract::Request, http::StatusCode}; +use http_body_util::BodyExt; +use rstest::rstest; +use support::{Gateway, client, raw_upstream, within}; +use tokio::sync::Notify; + +fn chunk(bytes: &[u8]) -> Vec { + [format!("{:x}\r\n", bytes.len()).as_bytes(), bytes, b"\r\n"].concat() +} + +#[rstest] +#[tokio::test] +async fn each_event_reaches_the_client_before_python_produces_the_next() { + let produce = Arc::new(Notify::new()); + let next = produce.clone(); + let upstream = raw_upstream(move |mut python| { + let next = next.clone(); + async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\n\r\n") + .await; + for event in ["data: 1\n\n", "data: 2\n\n", "data: [DONE]\n\n"] { + python.send(chunk(event.as_bytes())).await; + next.notified().await; + } + python.send("0\r\n\r\n").await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut response = within( + client().request( + Request::post(gateway.url("/v1/chat/completions")) + .body(Body::from(r#"{"stream":true}"#)) + .unwrap(), + ), + ) + .await + .unwrap(); + + for event in ["data: 1\n\n", "data: 2\n\n", "data: [DONE]\n\n"] { + let frame = within(response.body_mut().frame()).await.unwrap().unwrap(); + assert_eq!(frame.into_data().unwrap(), event); + produce.notify_one(); + } + assert!(within(response.body_mut().frame()).await.is_none()); +} + +#[rstest] +#[tokio::test] +async fn relays_a_binary_framed_stream_unchanged() { + let message: Vec = [ + &[ + 0x00, 0x00, 0x00, 0x3d, 0x00, 0x00, 0x00, 0x20, 0x8a, 0x0d, 0x0a, 0xff, + ][..], + b"\x0b:event-type\x07\x00\x05chunk\x0d:content-type\x07\x00\x10application/json", + b"0\r\n\r\n{\"bytes\":\"eyJ0eXBlIjoiY29udGVudCJ9\"}", + &[0xc3, 0x28, 0x00, 0x0d], + ] + .concat(); + let sent = message.clone(); + let upstream = raw_upstream(move |mut python| { + let sent = sent.clone(); + async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ncontent-type: application/vnd.amazon.eventstream\r\ntransfer-encoding: chunked\r\n\r\n") + .await; + for piece in sent.chunks(7) { + python.send(chunk(piece)).await; + } + python.send("0\r\n\r\n").await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let response = within( + client().request( + Request::post(gateway.url("/bedrock/model/m/converse-stream")) + .body(Body::empty()) + .unwrap(), + ), + ) + .await + .unwrap(); + + assert_eq!( + response.headers()["content-type"], + "application/vnd.amazon.eventstream" + ); + let body = within(response.into_body().collect()) + .await + .unwrap() + .to_bytes(); + assert_eq!(body, message); +} + +#[rstest] +#[tokio::test] +async fn relays_a_body_delimited_by_the_upstream_closing() { + let upstream = raw_upstream(|mut python| async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\nconnection: close\r\n\r\nuntil eof") + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let response = client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap()); + let body = within(response).await.unwrap().into_body().collect(); + assert_eq!(within(body).await.unwrap().to_bytes(), "until eof"); +} + +#[rstest] +#[tokio::test] +async fn an_upstream_dying_mid_stream_fails_the_client_body_instead_of_ending_it() { + let upstream = raw_upstream(|mut python| async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\n\r\n6\r\ndata: \r\n") + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let response = + within(client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap())) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert!(within(response.into_body().collect()).await.is_err()); +} diff --git a/litellm-rust/crates/gateway-forward/tests/upload.rs b/litellm-rust/crates/gateway-forward/tests/upload.rs new file mode 100644 index 00000000000..a0a20a4ce41 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/upload.rs @@ -0,0 +1,92 @@ +mod support; + +use std::convert::Infallible; + +use axum::{ + Router, + body::{Body, Bytes}, + extract::Request, + http::StatusCode, + response::Response, +}; +use http_body_util::BodyExt; +use rstest::rstest; +use support::{Gateway, client, recording, serve, within}; +use tokio::sync::mpsc; + +fn ok() -> Response { + Response::new(Body::from("ok")) +} + +#[rstest] +#[tokio::test] +async fn streams_upload_and_response_incrementally_without_a_body_limit() { + let upstream = serve( + Router::new().fallback(|request: Request| async { Response::new(request.into_body()) }), + ) + .await; + let gateway = Gateway::start(upstream).await; + let (sender, receiver) = mpsc::channel::>(1); + let stream = futures_util::stream::unfold(receiver, |mut receiver| async { + receiver.recv().await.map(|chunk| (chunk, receiver)) + }); + sender.send(Ok(Bytes::from_static(b"first"))).await.unwrap(); + let mut response = within( + client().request( + Request::post(gateway.url("/v1/files")) + .body(Body::from_stream(stream)) + .unwrap(), + ), + ) + .await + .unwrap(); + let frame = within(response.body_mut().frame()).await.unwrap().unwrap(); + assert_eq!(frame.into_data().unwrap(), "first"); + let large = Bytes::from(vec![b'x'; 8 * 1024 * 1024]); + sender.send(Ok(large.clone())).await.unwrap(); + drop(sender); + let rest = within(response.into_body().collect()) + .await + .unwrap() + .to_bytes(); + assert_eq!(rest, large); +} + +#[rstest] +#[tokio::test] +async fn answers_expect_continue_so_large_uploads_start() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("POST /v1/audio/transcriptions HTTP/1.1\r\nhost: x\r\nexpect: 100-continue\r\ncontent-length: 4\r\n\r\n") + .await; + let interim = within(raw.response_head()).await.unwrap(); + assert_eq!(interim.status, StatusCode::CONTINUE); + raw.send("data").await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(received.recv().await.unwrap().body, "data"); +} + +#[rstest] +#[case::get("GET")] +#[case::delete("DELETE")] +#[case::options("OPTIONS")] +#[case::post("POST")] +#[tokio::test] +async fn forwards_a_chunked_body_on_any_method(#[case] method: &str) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "{method} /v1/search HTTP/1.1\r\nhost: x\r\ntransfer-encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\n\r\n" + )) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(received.recv().await.unwrap().body, "body"); +} From 3c85c3e36092d77763d969e2271104872cc3f569 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 13:16:55 +0900 Subject: [PATCH 14/15] docs(gateway): document the forwarding env contract and test layout Co-Authored-By: Claude Opus 5.5 --- litellm-rust/crates/gateway-forward/AGENTS.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/litellm-rust/crates/gateway-forward/AGENTS.md b/litellm-rust/crates/gateway-forward/AGENTS.md index 5ef76f808f4..1210cc6e0c2 100644 --- a/litellm-rust/crates/gateway-forward/AGENTS.md +++ b/litellm-rust/crates/gateway-forward/AGENTS.md @@ -10,6 +10,20 @@ - Pooled upstream connections are retired well before Python's keep-alive timeout, so a request never lands on a connection Python is closing - WebSocket is the one protocol switch Python serves. Upgrades are relayed as raw bytes after the 101 and drained on shutdown +## Environment + +`Settings` reads only these variables, and only when `LITELLM_FORWARD_UPSTREAM` is set. The Python supervisor in `litellm/proxy/rust_forward/` sets every one of them from the values Python actually runs with, so all of them are required and a missing or invalid one fails startup naming it. The crate never reads Python's own settings (`HOST`, `PORT`, `KEEPALIVE_TIMEOUT`, `GRACEFUL_SHUTDOWN_TIMEOUT`) and keeps no copy of Python's defaults, since Python can get those values from CLI flags the environment never shows + +| Variable | Meaning | +|---|---| +| `LITELLM_FORWARD_UPSTREAM` | Python's loopback origin, e.g. `http://127.0.0.1:53211` | +| `LITELLM_FORWARD_HOST` | public bind host | +| `LITELLM_FORWARD_PORT` | public bind port | +| `LITELLM_FORWARD_UPSTREAM_KEEPALIVE` | seconds uvicorn keeps an idle connection open. Pooled connections are retired at half of it | +| `LITELLM_FORWARD_SHUTDOWN_DRAIN` | seconds to drain on shutdown, already including Python's graceful timeout and teardown headroom | + ## Tests -Test through real sockets with `tests/support`, never `oneshot`. `Gateway` runs the forwarder on a listener like production, `recording` is a well-behaved upstream that captures what Python would receive, and `raw_upstream` and `Raw` drive bytes by hand for framing, timing and disconnects a real server would not produce. `tests/roundtrip.rs` checks the forwarding invariant for generated requests and responses. Add a protocol edge case there or as a focused test next to its neighbors +Test through real sockets with `tests/support`, never `oneshot`. `Gateway` runs the forwarder on a listener like production, `recording` is a well-behaved upstream that captures what Python would receive, and `raw_upstream` and `Raw` drive bytes by hand for framing, timing and disconnects a real server would not produce. Waits measured in minutes use paused tokio time instead of real sleeps + +The forwarder sees HTTP/1.1 messages, not routes, so the tests are split into one file per traffic shape uvicorn can emit: `exchange` (plain request and response), `slow_head` (minutes before response headers), `stream` (incremental response bodies), `idle_stream` (long-lived streams that go quiet), `upload`, `download`, `websocket`, `client_identity`, `lifecycle` (disconnects, pooling, shutdown) and `roundtrip` (the forwarding invariant for generated messages). A new route never gets its own test. Add its behavior to the shape it already has, and when it needs a wire behavior none of the files covers, add a new shape file for it From 2e5d193cd7e988af3417e03105783d4b098682ab Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 16:19:35 +0900 Subject: [PATCH 15/15] refactor(gateway): give each forwarding decision one home in gateway-forward rewrite.rs now owns everything Python and the client receive, as pure functions over requests and responses. Rejection in error.rs is the one status table for requests Rust answers itself, and Settings parses the upstream into an Upstream that can only be a loopback origin, so Forward::new can no longer fail Co-Authored-By: Claude Opus 5.5 --- litellm-rust/crates/gateway-forward/AGENTS.md | 4 + .../crates/gateway-forward/src/error.rs | 45 +++++++ .../crates/gateway-forward/src/forward.rs | 80 ++----------- .../crates/gateway-forward/src/hop.rs | 35 ------ .../crates/gateway-forward/src/lib.rs | 10 +- .../crates/gateway-forward/src/rewrite.rs | 111 ++++++++++++++++++ .../crates/gateway-forward/src/settings.rs | 36 +++++- .../crates/gateway-forward/src/websocket.rs | 43 +++---- .../crates/gateway-forward/tests/settings.rs | 12 +- .../gateway-forward/tests/support/mod.rs | 2 +- .../crates/gateway-forward/tests/upstream.rs | 8 +- 11 files changed, 241 insertions(+), 145 deletions(-) delete mode 100644 litellm-rust/crates/gateway-forward/src/hop.rs create mode 100644 litellm-rust/crates/gateway-forward/src/rewrite.rs diff --git a/litellm-rust/crates/gateway-forward/AGENTS.md b/litellm-rust/crates/gateway-forward/AGENTS.md index 1210cc6e0c2..65410851588 100644 --- a/litellm-rust/crates/gateway-forward/AGENTS.md +++ b/litellm-rust/crates/gateway-forward/AGENTS.md @@ -10,6 +10,10 @@ - Pooled upstream connections are retired well before Python's keep-alive timeout, so a request never lands on a connection Python is closing - WebSocket is the one protocol switch Python serves. Upgrades are relayed as raw bytes after the 101 and drained on shutdown +## Layout + +Every forwarding decision has one home. `rewrite.rs` owns what Python and the client receive, as pure functions over requests and responses. `Rejection` in `error.rs` is the only place that picks a status for a request Rust answers itself. `settings.rs` parses the environment into typed values, so an `Upstream` is a loopback origin by construction. `forward.rs` and `websocket.rs` only move requests, bodies and upgraded connections and never touch a header themselves + ## Environment `Settings` reads only these variables, and only when `LITELLM_FORWARD_UPSTREAM` is set. The Python supervisor in `litellm/proxy/rust_forward/` sets every one of them from the values Python actually runs with, so all of them are required and a missing or invalid one fails startup naming it. The crate never reads Python's own settings (`HOST`, `PORT`, `KEEPALIVE_TIMEOUT`, `GRACEFUL_SHUTDOWN_TIMEOUT`) and keeps no copy of Python's defaults, since Python can get those values from CLI flags the environment never shows diff --git a/litellm-rust/crates/gateway-forward/src/error.rs b/litellm-rust/crates/gateway-forward/src/error.rs index ffd63b2ecab..bc38a0bdbf2 100644 --- a/litellm-rust/crates/gateway-forward/src/error.rs +++ b/litellm-rust/crates/gateway-forward/src/error.rs @@ -1,3 +1,8 @@ +use axum::{ + http::StatusCode, + response::{IntoResponse, Response}, +}; + #[derive(Debug, thiserror::Error)] pub enum Error { #[error("invalid forwarding upstream URI")] @@ -13,3 +18,43 @@ pub enum Error { #[error("forwarding gateway I/O failed: {0}")] Io(#[from] std::io::Error), } + +/// Why a request got an answer from Rust instead of from Python. +#[derive(Debug, thiserror::Error)] +pub(crate) enum Rejection { + #[error("request authority is not a valid Host header")] + InvalidAuthority, + #[error("python proxy unreachable: {0}")] + Unreachable(#[source] hyper_util::client::legacy::Error), + #[error("python proxy switched protocols without being asked to")] + UnrequestedUpgrade, + #[error("python proxy websocket upgrade failed: {0}")] + UpgradeFailed(#[source] hyper::Error), + #[error("forwarding gateway is shutting down")] + ShuttingDown, + #[error("could not build the request for the python proxy")] + Internal, +} + +impl Rejection { + fn status(&self) -> StatusCode { + match self { + Self::InvalidAuthority => StatusCode::BAD_REQUEST, + Self::Unreachable(_) | Self::UnrequestedUpgrade | Self::UpgradeFailed(_) => { + StatusCode::BAD_GATEWAY + } + Self::ShuttingDown => StatusCode::SERVICE_UNAVAILABLE, + Self::Internal => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +impl IntoResponse for Rejection { + fn into_response(self) -> Response { + let status = self.status(); + if status.is_server_error() { + tracing::warn!(error = %self, "request not forwarded"); + } + status.into_response() + } +} diff --git a/litellm-rust/crates/gateway-forward/src/forward.rs b/litellm-rust/crates/gateway-forward/src/forward.rs index d783ecab8a1..eaee3afe277 100644 --- a/litellm-rust/crates/gateway-forward/src/forward.rs +++ b/litellm-rust/crates/gateway-forward/src/forward.rs @@ -1,13 +1,10 @@ -use std::{ - net::{Ipv4Addr, SocketAddr}, - time::Duration, -}; +use std::{net::SocketAddr, time::Duration}; use axum::{ Router, - body::{Body, HttpBody}, + body::Body, extract::{ConnectInfo, DefaultBodyLimit, Request, State}, - http::{HeaderValue, StatusCode, Uri, Version, header}, + http::{StatusCode, Uri}, response::Response, serve::ListenerExt, }; @@ -18,9 +15,7 @@ use hyper_util::{ use tokio::net::TcpListener; use tokio_util::sync::CancellationToken; -use crate::{Error, hop, websocket::Upgrades}; - -pub const PEER_HEADER: &str = "x-litellm-rust-peer"; +use crate::{Rejection, Upstream, rewrite, websocket::Upgrades}; // uvicorn sets no response header limit and Python copies every provider header into its // response, so hyper's default of 100 would turn ordinary responses into 502s. @@ -36,21 +31,7 @@ pub struct Forward { impl Forward { /// `upstream_keepalive` is how long the upstream keeps an idle connection open. Pooled /// connections are dropped well before that, so a request never lands on one it is closing. - pub fn new(upstream: &str, upstream_keepalive: Duration) -> Result { - let upstream: Uri = upstream.parse().map_err(Error::UpstreamUri)?; - if upstream.scheme_str() != Some("http") - || !upstream - .host() - .and_then(|host| host.parse::().ok()) - .is_some_and(|ip| ip.is_loopback()) - || upstream - .authority() - .is_some_and(|authority| authority.as_str().contains('@')) - || upstream.path() != "/" - || upstream.query().is_some() - { - return Err(Error::Upstream); - } + pub fn new(upstream: &Upstream, upstream_keepalive: Duration) -> Self { let mut connector = HttpConnector::new(); connector.set_nodelay(true); let client = Client::builder(TokioExecutor::new()) @@ -58,11 +39,11 @@ impl Forward { .pool_timer(TokioTimer::new()) .pool_idle_timeout(upstream_keepalive / 2) .build(connector); - Ok(Self { - upstream, + Self { + upstream: upstream.uri().clone(), client, upgrades: Upgrades::default(), - }) + } } pub fn router(self) -> Router { @@ -131,52 +112,17 @@ async fn forward( State(state): State, ConnectInfo(peer): ConnectInfo, mut request: Request, -) -> Result { +) -> Result { let downstream = state.upgrades.take_handshake(&mut request)?; - // HTTP/2 carries the public host in :authority, which the upstream rewrite below replaces. - if let Some(authority) = request.uri().authority() - && !request.headers().contains_key(header::HOST) - { - let host = - HeaderValue::from_str(authority.as_str()).map_err(|_| StatusCode::BAD_REQUEST)?; - request.headers_mut().insert(header::HOST, host); - } - let mut uri = state.upstream.into_parts(); - uri.path_and_query = request.uri().path_and_query().cloned(); - *request.uri_mut() = Uri::from_parts(uri).map_err(|_| StatusCode::BAD_GATEWAY)?; - *request.version_mut() = Version::HTTP_11; - let chunked = request.body().size_hint().exact().is_none(); - let headers = request.headers_mut(); - hop::strip(headers); - if chunked { - // hyper sends no body for a GET with an unknown length unless told it is chunked. - headers.insert( - header::TRANSFER_ENCODING, - HeaderValue::from_static("chunked"), - ); - } - if downstream.is_some() { - hop::restore_websocket_handshake(headers); - } - // Forwarding headers pass through untouched so Python sees the request as if the client - // connected to it directly. Inserting replaces any client-sent copy: the peer is the one - // value a client must not choose. - headers.insert( - PEER_HEADER, - HeaderValue::from_str(&peer.to_string()).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?, - ); + let request = rewrite::upstream_request(request, &state.upstream, peer, downstream.is_some())?; let response = state.client.request(request); - let mut response = match downstream { + let response = match downstream { Some(_) => state.upgrades.until_closed(response).await?, None => response.await, } - .map_err(|error| { - tracing::warn!(%error, "python proxy unreachable"); - StatusCode::BAD_GATEWAY - })?; + .map_err(Rejection::Unreachable)?; if response.status() == StatusCode::SWITCHING_PROTOCOLS { return state.upgrades.switch(downstream, response).await; } - hop::strip(response.headers_mut()); - Ok(response.map(Body::new)) + Ok(rewrite::downstream_response(response).map(Body::new)) } diff --git a/litellm-rust/crates/gateway-forward/src/hop.rs b/litellm-rust/crates/gateway-forward/src/hop.rs deleted file mode 100644 index e572940bab4..00000000000 --- a/litellm-rust/crates/gateway-forward/src/hop.rs +++ /dev/null @@ -1,35 +0,0 @@ -use axum::http::{HeaderMap, HeaderValue, header}; - -// Per-connection headers (RFC 9110 section 7.6.1) describe the client's connection to Rust, not -// Rust's connection to Python, so a proxy must drop them instead of forwarding them. -pub(crate) fn strip(headers: &mut HeaderMap) { - let connection_headers: Vec<_> = headers - .get_all(header::CONNECTION) - .iter() - .filter_map(|value| value.to_str().ok()) - .flat_map(|value| value.split(',')) - .filter_map(|name| header::HeaderName::from_bytes(name.trim().as_bytes()).ok()) - .collect(); - for name in connection_headers { - headers.remove(name); - } - for name in [ - "connection", - "keep-alive", - "proxy-authenticate", - "proxy-authorization", - "te", - "trailer", - "transfer-encoding", - "upgrade", - "proxy-connection", - ] { - headers.remove(name); - } -} - -// Stripping drops the handshake's Connection and Upgrade, so each hop gets them back. -pub(crate) fn restore_websocket_handshake(headers: &mut HeaderMap) { - headers.insert(header::CONNECTION, HeaderValue::from_static("upgrade")); - headers.insert(header::UPGRADE, HeaderValue::from_static("websocket")); -} diff --git a/litellm-rust/crates/gateway-forward/src/lib.rs b/litellm-rust/crates/gateway-forward/src/lib.rs index b473fbca202..ef7b9724bb5 100644 --- a/litellm-rust/crates/gateway-forward/src/lib.rs +++ b/litellm-rust/crates/gateway-forward/src/lib.rs @@ -1,12 +1,14 @@ mod error; mod forward; -mod hop; +mod rewrite; mod settings; mod websocket; pub use error::Error; -pub use forward::{Forward, PEER_HEADER}; -pub use settings::Settings; +pub(crate) use error::Rejection; +pub use forward::Forward; +pub use rewrite::PEER_HEADER; +pub use settings::{Settings, Upstream}; /// Binds the public listener and forwards to Python until `shutdown` resolves, then drains for /// `settings.shutdown_drain`. @@ -14,7 +16,7 @@ pub async fn run( settings: Settings, shutdown: impl Future + Send + 'static, ) -> Result<(), Error> { - let forward = Forward::new(&settings.upstream, settings.upstream_keepalive)?; + let forward = Forward::new(&settings.upstream, settings.upstream_keepalive); let listener = tokio::net::TcpListener::bind((settings.host.as_str(), settings.port)).await?; tracing::info!(address = %listener.local_addr()?, "forwarding gateway listening"); forward diff --git a/litellm-rust/crates/gateway-forward/src/rewrite.rs b/litellm-rust/crates/gateway-forward/src/rewrite.rs new file mode 100644 index 00000000000..10c0eaad207 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/rewrite.rs @@ -0,0 +1,111 @@ +use std::net::SocketAddr; + +use axum::{ + body::HttpBody, + extract::Request, + http::{HeaderMap, HeaderValue, Response, Uri, Version, header}, +}; + +use crate::Rejection; + +pub const PEER_HEADER: &str = "x-litellm-rust-peer"; + +/// The request Python receives: the client's method, target, headers and body unchanged, sent to +/// `upstream` over HTTP/1.1 without the client connection's own headers, plus the client's TCP +/// peer in `PEER_HEADER`, which replaces any copy the client sent. +pub(crate) fn upstream_request( + mut request: Request, + upstream: &Uri, + peer: SocketAddr, + websocket: bool, +) -> Result { + // HTTP/2 carries the public host in :authority, which retargeting at the upstream replaces. + if let Some(authority) = request.uri().authority() + && !request.headers().contains_key(header::HOST) + { + let host = + HeaderValue::from_str(authority.as_str()).map_err(|_| Rejection::InvalidAuthority)?; + request.headers_mut().insert(header::HOST, host); + } + let mut uri = upstream.clone().into_parts(); + uri.path_and_query = request.uri().path_and_query().cloned(); + *request.uri_mut() = Uri::from_parts(uri).map_err(|_| Rejection::Internal)?; + *request.version_mut() = Version::HTTP_11; + let chunked = request.body().size_hint().exact().is_none(); + let headers = request.headers_mut(); + strip_hop_by_hop(headers); + if chunked { + // hyper sends no body for a GET with an unknown length unless told it is chunked. + headers.insert( + header::TRANSFER_ENCODING, + HeaderValue::from_static("chunked"), + ); + } + if websocket { + restore_websocket_handshake(headers); + } + headers.insert( + PEER_HEADER, + HeaderValue::from_str(&peer.to_string()).map_err(|_| Rejection::Internal)?, + ); + Ok(request) +} + +/// The response the client receives: Python's, without the upstream connection's own headers. +pub(crate) fn downstream_response(mut response: Response) -> Response { + strip_hop_by_hop(response.headers_mut()); + response +} + +/// Python's 101 as the client receives it, still naming the protocol both sides switch to. +pub(crate) fn switched_response(mut response: Response) -> Response { + strip_hop_by_hop(response.headers_mut()); + restore_websocket_handshake(response.headers_mut()); + response +} + +pub(crate) fn is_websocket_handshake(headers: &HeaderMap) -> bool { + headers + .get(header::UPGRADE) + .is_some_and(|value| value.as_bytes().eq_ignore_ascii_case(b"websocket")) + && headers + .get_all(header::CONNECTION) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .any(|token| token.trim().eq_ignore_ascii_case("upgrade")) +} + +// Per-connection headers (RFC 9110 section 7.6.1) describe the client's connection to Rust, not +// Rust's connection to Python, so a proxy must drop them instead of forwarding them. +fn strip_hop_by_hop(headers: &mut HeaderMap) { + let connection_headers: Vec<_> = headers + .get_all(header::CONNECTION) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .filter_map(|name| header::HeaderName::from_bytes(name.trim().as_bytes()).ok()) + .collect(); + for name in connection_headers { + headers.remove(name); + } + for name in [ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "transfer-encoding", + "upgrade", + "proxy-connection", + ] { + headers.remove(name); + } +} + +// Stripping drops the handshake's Connection and Upgrade, so each hop gets them back. +fn restore_websocket_handshake(headers: &mut HeaderMap) { + headers.insert(header::CONNECTION, HeaderValue::from_static("upgrade")); + headers.insert(header::UPGRADE, HeaderValue::from_static("websocket")); +} diff --git a/litellm-rust/crates/gateway-forward/src/settings.rs b/litellm-rust/crates/gateway-forward/src/settings.rs index 6093ea33561..dd229b60bf2 100644 --- a/litellm-rust/crates/gateway-forward/src/settings.rs +++ b/litellm-rust/crates/gateway-forward/src/settings.rs @@ -1,11 +1,41 @@ -use std::{ffi::OsString, time::Duration}; +use std::{ffi::OsString, net::SocketAddr, str::FromStr, time::Duration}; +use axum::http::Uri; use serde::Deserialize; use crate::Error; const UPSTREAM: &str = "LITELLM_FORWARD_UPSTREAM"; +/// The Python proxy's private origin. Only a loopback IPv4 address parses, so nothing can point +/// the forwarder at another host. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Upstream(Uri); + +impl Upstream { + pub(crate) fn uri(&self) -> &Uri { + &self.0 + } +} + +impl FromStr for Upstream { + type Err = Error; + + fn from_str(value: &str) -> Result { + let uri: Uri = value.parse().map_err(Error::UpstreamUri)?; + let address = uri + .authority() + .filter(|_| uri.scheme_str() == Some("http")) + .filter(|authority| !authority.as_str().contains('@')) + .filter(|_| uri.path() == "/" && uri.query().is_none()) + .and_then(|authority| authority.as_str().parse::().ok()); + match address { + Some(SocketAddr::V4(address)) if address.ip().is_loopback() => Ok(Self(uri)), + _ => Err(Error::Upstream), + } + } +} + #[derive(Deserialize)] struct Environment { litellm_forward_upstream: String, @@ -19,7 +49,7 @@ struct Environment { /// Python actually runs with, so none of them is guessed or defaulted here. #[derive(Debug, Clone, PartialEq)] pub struct Settings { - pub upstream: String, + pub upstream: Upstream, pub host: String, pub port: u16, pub upstream_keepalive: Duration, @@ -47,7 +77,7 @@ impl Settings { envy::Error::Custom(message) => Error::InvalidSetting(message), })?; Ok(Some(Self { - upstream: environment.litellm_forward_upstream, + upstream: environment.litellm_forward_upstream.parse()?, host: environment.litellm_forward_host, port: environment.litellm_forward_port, upstream_keepalive: seconds( diff --git a/litellm-rust/crates/gateway-forward/src/websocket.rs b/litellm-rust/crates/gateway-forward/src/websocket.rs index d04b0434c19..d632e90398b 100644 --- a/litellm-rust/crates/gateway-forward/src/websocket.rs +++ b/litellm-rust/crates/gateway-forward/src/websocket.rs @@ -1,7 +1,7 @@ use axum::{ body::Body, extract::Request, - http::{HeaderMap, Method, StatusCode, Version, header}, + http::{Method, Version}, response::Response, }; use hyper::{ @@ -14,6 +14,8 @@ use tokio_util::{ task::TaskTracker, }; +use crate::{Rejection, rewrite}; + #[derive(Clone, Default)] pub(crate) struct Upgrades { tasks: TaskTracker, @@ -26,22 +28,22 @@ impl Upgrades { self.draining.cancel(); } - pub fn take_handshake(&self, request: &mut Request) -> Result, StatusCode> { + pub fn take_handshake(&self, request: &mut Request) -> Result, Rejection> { if request.method() != Method::GET || request.version() != Version::HTTP_11 - || !is_handshake(request.headers()) + || !rewrite::is_websocket_handshake(request.headers()) { return Ok(None); } if self.draining.is_cancelled() { - return Err(StatusCode::SERVICE_UNAVAILABLE); + return Err(Rejection::ShuttingDown); } Ok(Some(hyper::upgrade::on(request))) } - pub async fn until_closed(&self, future: impl Future) -> Result { + pub async fn until_closed(&self, future: impl Future) -> Result { tokio::select! { - _ = self.closed.cancelled() => Err(StatusCode::SERVICE_UNAVAILABLE), + _ = self.closed.cancelled() => Err(Rejection::ShuttingDown), output = future => Ok(output), } } @@ -50,24 +52,19 @@ impl Upgrades { &self, downstream: Option, mut response: Response, - ) -> Result { + ) -> Result { let Some(downstream) = downstream else { - return Err(StatusCode::BAD_GATEWAY); + return Err(Rejection::UnrequestedUpgrade); }; - if !is_handshake(response.headers()) { - return Err(StatusCode::BAD_GATEWAY); + if !rewrite::is_websocket_handshake(response.headers()) { + return Err(Rejection::UnrequestedUpgrade); } let upstream = self .until_closed(hyper::upgrade::on(&mut response)) .await? - .map_err(|error| { - tracing::warn!(%error, "python proxy websocket upgrade failed"); - StatusCode::BAD_GATEWAY - })?; - crate::hop::strip(response.headers_mut()); - crate::hop::restore_websocket_handshake(response.headers_mut()); + .map_err(Rejection::UpgradeFailed)?; self.relay(downstream, upstream); - Ok(response.map(|_| Body::empty())) + Ok(rewrite::switched_response(response).map(|_| Body::empty())) } pub fn close(&self) { @@ -103,15 +100,3 @@ impl Upgrades { }); } } - -fn is_handshake(headers: &HeaderMap) -> bool { - headers - .get(header::UPGRADE) - .is_some_and(|value| value.as_bytes().eq_ignore_ascii_case(b"websocket")) - && headers - .get_all(header::CONNECTION) - .iter() - .filter_map(|value| value.to_str().ok()) - .flat_map(|value| value.split(',')) - .any(|token| token.trim().eq_ignore_ascii_case("upgrade")) -} diff --git a/litellm-rust/crates/gateway-forward/tests/settings.rs b/litellm-rust/crates/gateway-forward/tests/settings.rs index 540e1d9844d..7069528449b 100644 --- a/litellm-rust/crates/gateway-forward/tests/settings.rs +++ b/litellm-rust/crates/gateway-forward/tests/settings.rs @@ -38,7 +38,7 @@ fn reads_every_variable() { assert_eq!( parse(&COMPLETE).unwrap(), Some(Settings { - upstream: "http://127.0.0.1:53211".into(), + upstream: "http://127.0.0.1:53211".parse().unwrap(), host: "0.0.0.0".into(), port: 4000, upstream_keepalive: Duration::from_secs(2), @@ -97,3 +97,13 @@ fn python_settings_are_never_read(#[case] variable: &str) { assert_eq!(parse(&variables).unwrap(), parse(&COMPLETE).unwrap()); } + +#[rstest] +fn a_non_loopback_upstream_fails_startup() { + let error = parse(&with("LITELLM_FORWARD_UPSTREAM", "http://192.0.2.1:4000")).unwrap_err(); + + assert!( + error.to_string().contains("LITELLM_FORWARD_UPSTREAM"), + "{error}" + ); +} diff --git a/litellm-rust/crates/gateway-forward/tests/support/mod.rs b/litellm-rust/crates/gateway-forward/tests/support/mod.rs index db9da82efbf..507bc47b702 100644 --- a/litellm-rust/crates/gateway-forward/tests/support/mod.rs +++ b/litellm-rust/crates/gateway-forward/tests/support/mod.rs @@ -45,7 +45,7 @@ impl Gateway { } pub async fn with_keepalive(upstream: SocketAddr, keepalive: Duration) -> Self { - let forward = Forward::new(&format!("http://{upstream}"), keepalive).unwrap(); + let forward = Forward::new(&format!("http://{upstream}").parse().unwrap(), keepalive); let public = TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = public.local_addr().unwrap(); let (stop, stopped) = oneshot::channel::<()>(); diff --git a/litellm-rust/crates/gateway-forward/tests/upstream.rs b/litellm-rust/crates/gateway-forward/tests/upstream.rs index 941208d5223..25e4768a54e 100644 --- a/litellm-rust/crates/gateway-forward/tests/upstream.rs +++ b/litellm-rust/crates/gateway-forward/tests/upstream.rs @@ -1,6 +1,4 @@ -use std::time::Duration; - -use litellm_gateway_forward::Forward; +use litellm_gateway_forward::Upstream; use rstest::rstest; #[rstest] @@ -12,12 +10,12 @@ use rstest::rstest; #[case::query("http://127.0.0.1:4000/?x=y")] #[case::credentials("http://user@127.0.0.1:4000")] fn rejects_anything_but_a_loopback_http_origin(#[case] upstream: &str) { - assert!(Forward::new(upstream, Duration::from_secs(5)).is_err()); + assert!(upstream.parse::().is_err()); } #[rstest] #[case::standard("http://127.0.0.1:4000")] #[case::loopback_range("http://127.0.0.2:4000/")] fn accepts_a_loopback_http_origin(#[case] upstream: &str) { - assert!(Forward::new(upstream, Duration::from_secs(5)).is_ok()); + assert!(upstream.parse::().is_ok()); }