From e361f7acaee7876e7bad58f7772e0474b9874033 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 23:00:07 +0000 Subject: [PATCH] feat(gateway): forward HTTP requests to the Python proxy --- litellm-rust/Cargo.lock | 1 + litellm-rust/crates/gateway/Cargo.toml | 1 + litellm-rust/crates/gateway/src/error.rs | 6 + litellm-rust/crates/gateway/src/forward.rs | 308 ++++++++++++++++++++ litellm-rust/crates/gateway/src/lib.rs | 1 + litellm-rust/crates/gateway/src/main.rs | 24 ++ litellm-rust/crates/gateway/src/settings.rs | 27 ++ 7 files changed, 368 insertions(+) create mode 100644 litellm-rust/crates/gateway/src/forward.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index f91deaaa401..f571a49b0c4 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3808,6 +3808,7 @@ dependencies = [ "envy", "futures-util", "http-body-util", + "hyper-util", "litellm-circuit-breaker", "litellm-config", "litellm-error", diff --git a/litellm-rust/crates/gateway/Cargo.toml b/litellm-rust/crates/gateway/Cargo.toml index 6eeaa43e013..e1b1e8d486c 100644 --- a/litellm-rust/crates/gateway/Cargo.toml +++ b/litellm-rust/crates/gateway/Cargo.toml @@ -10,6 +10,7 @@ litellm-secret-value.workspace = true axum.workspace = true envy = "0.4.2" http-body-util = "0.1" +hyper-util = { workspace = true, features = ["client-legacy", "http1", "tokio"] } litellm-inference.workspace = true litellm-gateway-inference.workspace = true litellm-gateway-mcp.workspace = true diff --git a/litellm-rust/crates/gateway/src/error.rs b/litellm-rust/crates/gateway/src/error.rs index e2f646c5448..6911ce17df9 100644 --- a/litellm-rust/crates/gateway/src/error.rs +++ b/litellm-rust/crates/gateway/src/error.rs @@ -1,5 +1,11 @@ #[derive(Debug, thiserror::Error)] pub enum Error { + #[error("invalid forwarding upstream URI")] + ForwardUri(#[source] axum::http::uri::InvalidUri), + #[error( + "LITELLM_FORWARD_UPSTREAM must be a loopback IPv4 http origin without a path, query or credentials" + )] + ForwardUpstream, #[error(transparent)] Http(#[from] litellm_http::Error), #[error("environment_variables values must be strings, numbers or booleans")] diff --git a/litellm-rust/crates/gateway/src/forward.rs b/litellm-rust/crates/gateway/src/forward.rs new file mode 100644 index 00000000000..c88536d8e34 --- /dev/null +++ b/litellm-rust/crates/gateway/src/forward.rs @@ -0,0 +1,308 @@ +use std::net::{Ipv4Addr, SocketAddr}; + +use axum::{ + Router, + body::Body, + extract::{ConnectInfo, DefaultBodyLimit, Request, State}, + http::{HeaderMap, HeaderValue, StatusCode, Uri, header}, + response::Response, +}; +use hyper_util::{ + client::legacy::{Client, connect::HttpConnector}, + rt::TokioExecutor, +}; + +use crate::Error; + +#[derive(Clone)] +pub struct Forward { + upstream: Uri, + client: Client, +} + +impl Forward { + pub fn new(upstream: &str) -> Result { + let upstream: Uri = upstream.parse().map_err(Error::ForwardUri)?; + if upstream.scheme_str() != Some("http") + || !upstream + .host() + .and_then(|host| host.parse::().ok()) + .is_some_and(|ip| ip.is_loopback()) + || upstream + .authority() + .is_some_and(|authority| authority.as_str().contains('@')) + || upstream.path() != "/" + || upstream.query().is_some() + { + return Err(Error::ForwardUpstream); + } + let client = Client::builder(TokioExecutor::new()) + .retry_canceled_requests(false) + .build_http(); + Ok(Self { upstream, client }) + } + + pub fn router(self) -> Router { + Router::new() + .fallback(forward) + .layer(DefaultBodyLimit::disable()) + .with_state(self) + } +} + +async fn forward( + State(state): State, + ConnectInfo(peer): ConnectInfo, + mut request: Request, +) -> Result { + let mut uri = state.upstream.into_parts(); + uri.path_and_query = request.uri().path_and_query().cloned(); + *request.uri_mut() = Uri::from_parts(uri).map_err(|_| StatusCode::BAD_GATEWAY)?; + *request.version_mut() = axum::http::Version::HTTP_11; + let headers = request.headers_mut(); + strip_hop_headers(headers); + let forwarded = headers + .get_all("x-forwarded-for") + .iter() + .map(|value| value.as_bytes()) + .collect::>(); + let mut forwarded = forwarded.join(b", ".as_slice()); + if !forwarded.is_empty() { + forwarded.extend_from_slice(b", "); + } + forwarded.extend_from_slice(peer.ip().to_string().as_bytes()); + headers.insert( + "x-forwarded-for", + HeaderValue::from_bytes(&forwarded).map_err(|_| StatusCode::BAD_REQUEST)?, + ); + if !headers.contains_key("x-forwarded-proto") { + headers.insert("x-forwarded-proto", HeaderValue::from_static("http")); + } + let mut response = state + .client + .request(request) + .await + .map_err(|_| StatusCode::BAD_GATEWAY)?; + strip_hop_headers(response.headers_mut()); + Ok(response.map(Body::new)) +} + +fn strip_hop_headers(headers: &mut HeaderMap) { + let connection_headers: Vec<_> = headers + .get_all(header::CONNECTION) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .filter_map(|name| header::HeaderName::from_bytes(name.trim().as_bytes()).ok()) + .collect(); + for name in connection_headers { + headers.remove(name); + } + for name in [ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "transfer-encoding", + "upgrade", + "proxy-connection", + ] { + headers.remove(name); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{body::Bytes, http::Version}; + use http_body_util::BodyExt; + use rstest::rstest; + use std::{convert::Infallible, time::Duration}; + use tokio::{net::TcpListener, sync::mpsc}; + use tower::ServiceExt; + + #[rstest] + #[case::https("https://127.0.0.1:4000")] + #[case::public("http://192.0.2.1:4000")] + #[case::hostname("http://localhost:4000")] + #[case::ipv6("http://[::1]:4000")] + #[case::path("http://127.0.0.1:4000/api")] + #[case::query("http://127.0.0.1:4000/?x=y")] + #[case::credentials("http://user@127.0.0.1:4000")] + fn rejects_invalid_upstream(#[case] upstream: &str) { + assert!(Forward::new(upstream).is_err()); + } + + #[rstest] + #[case::standard("http://127.0.0.1:4000")] + #[case::loopback_range("http://127.0.0.2:4000/")] + fn accepts_loopback_origin(#[case] upstream: &str) { + assert!(Forward::new(upstream).is_ok()); + } + + #[rstest] + #[tokio::test] + async fn preserves_request_and_response_without_hop_headers() { + let upstream = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}", upstream.local_addr().unwrap()); + let task = tokio::spawn(async move { + axum::serve( + upstream, + Router::new().fallback(|request: Request| async { + assert_eq!(request.method(), "PATCH"); + assert_eq!(request.uri(), "/v1/messages?x=a%2Fb&x=c"); + assert_eq!(request.headers()["host"], "public.example"); + assert_eq!(request.headers()["authorization"], "Bearer test"); + assert_eq!( + request.headers()["x-forwarded-for"], + "192.0.2.1, 192.0.2.2, 198.51.100.5" + ); + assert_eq!(request.headers()["x-forwarded-proto"], "https"); + assert!(!request.headers().contains_key("x-remove")); + assert!(!request.headers().contains_key("proxy-authorization")); + assert_eq!( + request.into_body().collect().await.unwrap().to_bytes(), + "payload" + ); + Response::builder() + .status(StatusCode::UNAUTHORIZED) + .header("connection", "x-upstream-hop") + .header("x-upstream-hop", "remove") + .header("x-response", "keep") + .body(Body::from("python error")) + .unwrap() + }), + ) + .await + .unwrap(); + }); + let request = Request::builder() + .method("PATCH") + .version(Version::HTTP_2) + .uri("/v1/messages?x=a%2Fb&x=c") + .header("host", "public.example") + .header("authorization", "Bearer test") + .header("x-forwarded-for", "192.0.2.1") + .header("x-forwarded-for", "192.0.2.2") + .header("x-forwarded-proto", "https") + .header("connection", "x-remove, keep-alive") + .header("x-remove", "remove") + .header("proxy-authorization", "remove") + .extension(ConnectInfo( + "198.51.100.5:1234".parse::().unwrap(), + )) + .body(Body::from("payload")) + .unwrap(); + let response = Forward::new(&origin) + .unwrap() + .router() + .oneshot(request) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert_eq!(response.headers()["x-response"], "keep"); + assert!(!response.headers().contains_key("connection")); + assert!(!response.headers().contains_key("x-upstream-hop")); + assert_eq!( + response.into_body().collect().await.unwrap().to_bytes(), + "python error" + ); + task.abort(); + } + + #[rstest] + #[case::health("/health/liveliness")] + #[case::readiness("/health/readiness")] + #[case::messages("/v1/messages")] + #[tokio::test] + async fn unreachable_upstream_is_502(#[case] path: &str) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + drop(listener); + let response = Forward::new(&origin) + .unwrap() + .router() + .oneshot( + Request::builder() + .uri(path) + .extension(ConnectInfo( + "198.51.100.5:1234".parse::().unwrap(), + )) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + } + + #[rstest] + #[tokio::test] + async fn streams_upload_and_response_before_eof_without_body_limit() { + let upstream = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}", upstream.local_addr().unwrap()); + let upstream_task = tokio::spawn(async move { + axum::serve( + upstream, + Router::new().fallback(|request: Request| async { + assert_eq!(request.headers()["x-forwarded-for"], "127.0.0.1"); + assert_eq!(request.headers()["x-forwarded-proto"], "http"); + Response::new(request.into_body()) + }), + ) + .await + .unwrap(); + }); + let public = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let public_uri = format!("http://{}/chat/completions", public.local_addr().unwrap()); + let forward_task = tokio::spawn(async move { + axum::serve( + public, + Forward::new(&origin) + .unwrap() + .router() + .into_make_service_with_connect_info::(), + ) + .await + .unwrap(); + }); + let (sender, receiver) = mpsc::channel::>(1); + let stream = futures_util::stream::unfold(receiver, |mut receiver| async { + receiver.recv().await.map(|chunk| (chunk, receiver)) + }); + sender.send(Ok(Bytes::from_static(b"first"))).await.unwrap(); + let client: Client = + Client::builder(TokioExecutor::new()).build_http(); + let mut response = tokio::time::timeout( + Duration::from_secs(5), + client.request( + Request::post(public_uri) + .body(Body::from_stream(stream)) + .unwrap(), + ), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let frame = tokio::time::timeout(Duration::from_secs(5), response.body_mut().frame()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(frame.into_data().unwrap(), "first"); + let large = Bytes::from(vec![b'x'; 3 * 1024 * 1024]); + sender.send(Ok(large.clone())).await.unwrap(); + drop(sender); + let rest = tokio::time::timeout(Duration::from_secs(5), response.into_body().collect()) + .await + .unwrap() + .unwrap() + .to_bytes(); + assert_eq!(rest, large); + upstream_task.abort(); + forward_task.abort(); + } +} diff --git a/litellm-rust/crates/gateway/src/lib.rs b/litellm-rust/crates/gateway/src/lib.rs index 12a9bf9110a..8fa322628a7 100644 --- a/litellm-rust/crates/gateway/src/lib.rs +++ b/litellm-rust/crates/gateway/src/lib.rs @@ -1,5 +1,6 @@ mod auth; mod error; +pub mod forward; mod redis; mod secrets; pub use error::Error; diff --git a/litellm-rust/crates/gateway/src/main.rs b/litellm-rust/crates/gateway/src/main.rs index 47c1fe83f50..168a4966ef7 100644 --- a/litellm-rust/crates/gateway/src/main.rs +++ b/litellm-rust/crates/gateway/src/main.rs @@ -41,6 +41,30 @@ impl Sink for StderrSink { #[tokio::main] async fn main() -> Result<(), Box> { + if let Some(upstream) = std::env::var_os("LITELLM_FORWARD_UPSTREAM") { + let settings: settings::ForwardSettings = settings::from_iter(std::env::vars_os())?; + Logger::new(StderrSink { + level: settings.log_level(), + }) + .install_global()?; + let forward = litellm_gateway::forward::Forward::new( + &upstream + .into_string() + .map_err(|_| "invalid upstream encoding")?, + )?; + let listener = + tokio::net::TcpListener::bind((settings.host.as_str(), settings.port)).await?; + tracing::info!(address = %listener.local_addr()?, "forwarding gateway listening"); + axum::serve( + listener, + forward + .router() + .into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(shutdown_signal()) + .await?; + return Ok(()); + } let settings: Settings = settings::from_iter(std::env::vars_os())?; let level = settings.log_level(); Logger::new(StderrSink { level }).install_global()?; diff --git a/litellm-rust/crates/gateway/src/settings.rs b/litellm-rust/crates/gateway/src/settings.rs index 445071504ed..332c774b600 100644 --- a/litellm-rust/crates/gateway/src/settings.rs +++ b/litellm-rust/crates/gateway/src/settings.rs @@ -4,6 +4,33 @@ use litellm_secret_value::SecretValue; use litellm_tracing::Level; use serde::{Deserialize, de::DeserializeOwned}; +#[derive(Deserialize)] +#[serde(default)] +pub(super) struct ForwardSettings { + pub host: String, + pub port: u16, + rust_log: Option, +} + +impl Default for ForwardSettings { + fn default() -> Self { + Self { + host: "0.0.0.0".into(), + port: 4000, + rust_log: None, + } + } +} + +impl ForwardSettings { + pub fn log_level(&self) -> Level { + self.rust_log + .as_deref() + .and_then(|value| value.parse().ok()) + .unwrap_or(Level::INFO) + } +} + #[derive(Deserialize)] #[serde(default)] pub(super) struct Settings {