feat(gateway): forward HTTP requests to the Python proxy

This commit is contained in:
Devin AI 2026-11-01 23:00:07 +00:00
parent 60fc121ce8
commit e361f7acae
7 changed files with 368 additions and 0 deletions

View file

@ -3808,6 +3808,7 @@ dependencies = [
"envy",
"futures-util",
"http-body-util",
"hyper-util",
"litellm-circuit-breaker",
"litellm-config",
"litellm-error",

View file

@ -10,6 +10,7 @@ litellm-secret-value.workspace = true
axum.workspace = true
envy = "0.4.2"
http-body-util = "0.1"
hyper-util = { workspace = true, features = ["client-legacy", "http1", "tokio"] }
litellm-inference.workspace = true
litellm-gateway-inference.workspace = true
litellm-gateway-mcp.workspace = true

View file

@ -1,5 +1,11 @@
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("invalid forwarding upstream URI")]
ForwardUri(#[source] axum::http::uri::InvalidUri),
#[error(
"LITELLM_FORWARD_UPSTREAM must be a loopback IPv4 http origin without a path, query or credentials"
)]
ForwardUpstream,
#[error(transparent)]
Http(#[from] litellm_http::Error),
#[error("environment_variables values must be strings, numbers or booleans")]

View file

@ -0,0 +1,308 @@
use std::net::{Ipv4Addr, SocketAddr};
use axum::{
Router,
body::Body,
extract::{ConnectInfo, DefaultBodyLimit, Request, State},
http::{HeaderMap, HeaderValue, StatusCode, Uri, header},
response::Response,
};
use hyper_util::{
client::legacy::{Client, connect::HttpConnector},
rt::TokioExecutor,
};
use crate::Error;
#[derive(Clone)]
pub struct Forward {
upstream: Uri,
client: Client<HttpConnector, Body>,
}
impl Forward {
pub fn new(upstream: &str) -> Result<Self, Error> {
let upstream: Uri = upstream.parse().map_err(Error::ForwardUri)?;
if upstream.scheme_str() != Some("http")
|| !upstream
.host()
.and_then(|host| host.parse::<Ipv4Addr>().ok())
.is_some_and(|ip| ip.is_loopback())
|| upstream
.authority()
.is_some_and(|authority| authority.as_str().contains('@'))
|| upstream.path() != "/"
|| upstream.query().is_some()
{
return Err(Error::ForwardUpstream);
}
let client = Client::builder(TokioExecutor::new())
.retry_canceled_requests(false)
.build_http();
Ok(Self { upstream, client })
}
pub fn router(self) -> Router {
Router::new()
.fallback(forward)
.layer(DefaultBodyLimit::disable())
.with_state(self)
}
}
async fn forward(
State(state): State<Forward>,
ConnectInfo(peer): ConnectInfo<SocketAddr>,
mut request: Request,
) -> Result<Response, StatusCode> {
let mut uri = state.upstream.into_parts();
uri.path_and_query = request.uri().path_and_query().cloned();
*request.uri_mut() = Uri::from_parts(uri).map_err(|_| StatusCode::BAD_GATEWAY)?;
*request.version_mut() = axum::http::Version::HTTP_11;
let headers = request.headers_mut();
strip_hop_headers(headers);
let forwarded = headers
.get_all("x-forwarded-for")
.iter()
.map(|value| value.as_bytes())
.collect::<Vec<_>>();
let mut forwarded = forwarded.join(b", ".as_slice());
if !forwarded.is_empty() {
forwarded.extend_from_slice(b", ");
}
forwarded.extend_from_slice(peer.ip().to_string().as_bytes());
headers.insert(
"x-forwarded-for",
HeaderValue::from_bytes(&forwarded).map_err(|_| StatusCode::BAD_REQUEST)?,
);
if !headers.contains_key("x-forwarded-proto") {
headers.insert("x-forwarded-proto", HeaderValue::from_static("http"));
}
let mut response = state
.client
.request(request)
.await
.map_err(|_| StatusCode::BAD_GATEWAY)?;
strip_hop_headers(response.headers_mut());
Ok(response.map(Body::new))
}
fn strip_hop_headers(headers: &mut HeaderMap) {
let connection_headers: Vec<_> = headers
.get_all(header::CONNECTION)
.iter()
.filter_map(|value| value.to_str().ok())
.flat_map(|value| value.split(','))
.filter_map(|name| header::HeaderName::from_bytes(name.trim().as_bytes()).ok())
.collect();
for name in connection_headers {
headers.remove(name);
}
for name in [
"connection",
"keep-alive",
"proxy-authenticate",
"proxy-authorization",
"te",
"trailer",
"transfer-encoding",
"upgrade",
"proxy-connection",
] {
headers.remove(name);
}
}
#[cfg(test)]
mod tests {
use super::*;
use axum::{body::Bytes, http::Version};
use http_body_util::BodyExt;
use rstest::rstest;
use std::{convert::Infallible, time::Duration};
use tokio::{net::TcpListener, sync::mpsc};
use tower::ServiceExt;
#[rstest]
#[case::https("https://127.0.0.1:4000")]
#[case::public("http://192.0.2.1:4000")]
#[case::hostname("http://localhost:4000")]
#[case::ipv6("http://[::1]:4000")]
#[case::path("http://127.0.0.1:4000/api")]
#[case::query("http://127.0.0.1:4000/?x=y")]
#[case::credentials("http://user@127.0.0.1:4000")]
fn rejects_invalid_upstream(#[case] upstream: &str) {
assert!(Forward::new(upstream).is_err());
}
#[rstest]
#[case::standard("http://127.0.0.1:4000")]
#[case::loopback_range("http://127.0.0.2:4000/")]
fn accepts_loopback_origin(#[case] upstream: &str) {
assert!(Forward::new(upstream).is_ok());
}
#[rstest]
#[tokio::test]
async fn preserves_request_and_response_without_hop_headers() {
let upstream = TcpListener::bind("127.0.0.1:0").await.unwrap();
let origin = format!("http://{}", upstream.local_addr().unwrap());
let task = tokio::spawn(async move {
axum::serve(
upstream,
Router::new().fallback(|request: Request| async {
assert_eq!(request.method(), "PATCH");
assert_eq!(request.uri(), "/v1/messages?x=a%2Fb&x=c");
assert_eq!(request.headers()["host"], "public.example");
assert_eq!(request.headers()["authorization"], "Bearer test");
assert_eq!(
request.headers()["x-forwarded-for"],
"192.0.2.1, 192.0.2.2, 198.51.100.5"
);
assert_eq!(request.headers()["x-forwarded-proto"], "https");
assert!(!request.headers().contains_key("x-remove"));
assert!(!request.headers().contains_key("proxy-authorization"));
assert_eq!(
request.into_body().collect().await.unwrap().to_bytes(),
"payload"
);
Response::builder()
.status(StatusCode::UNAUTHORIZED)
.header("connection", "x-upstream-hop")
.header("x-upstream-hop", "remove")
.header("x-response", "keep")
.body(Body::from("python error"))
.unwrap()
}),
)
.await
.unwrap();
});
let request = Request::builder()
.method("PATCH")
.version(Version::HTTP_2)
.uri("/v1/messages?x=a%2Fb&x=c")
.header("host", "public.example")
.header("authorization", "Bearer test")
.header("x-forwarded-for", "192.0.2.1")
.header("x-forwarded-for", "192.0.2.2")
.header("x-forwarded-proto", "https")
.header("connection", "x-remove, keep-alive")
.header("x-remove", "remove")
.header("proxy-authorization", "remove")
.extension(ConnectInfo(
"198.51.100.5:1234".parse::<SocketAddr>().unwrap(),
))
.body(Body::from("payload"))
.unwrap();
let response = Forward::new(&origin)
.unwrap()
.router()
.oneshot(request)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
assert_eq!(response.headers()["x-response"], "keep");
assert!(!response.headers().contains_key("connection"));
assert!(!response.headers().contains_key("x-upstream-hop"));
assert_eq!(
response.into_body().collect().await.unwrap().to_bytes(),
"python error"
);
task.abort();
}
#[rstest]
#[case::health("/health/liveliness")]
#[case::readiness("/health/readiness")]
#[case::messages("/v1/messages")]
#[tokio::test]
async fn unreachable_upstream_is_502(#[case] path: &str) {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let origin = format!("http://{}", listener.local_addr().unwrap());
drop(listener);
let response = Forward::new(&origin)
.unwrap()
.router()
.oneshot(
Request::builder()
.uri(path)
.extension(ConnectInfo(
"198.51.100.5:1234".parse::<SocketAddr>().unwrap(),
))
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::BAD_GATEWAY);
}
#[rstest]
#[tokio::test]
async fn streams_upload_and_response_before_eof_without_body_limit() {
let upstream = TcpListener::bind("127.0.0.1:0").await.unwrap();
let origin = format!("http://{}", upstream.local_addr().unwrap());
let upstream_task = tokio::spawn(async move {
axum::serve(
upstream,
Router::new().fallback(|request: Request| async {
assert_eq!(request.headers()["x-forwarded-for"], "127.0.0.1");
assert_eq!(request.headers()["x-forwarded-proto"], "http");
Response::new(request.into_body())
}),
)
.await
.unwrap();
});
let public = TcpListener::bind("127.0.0.1:0").await.unwrap();
let public_uri = format!("http://{}/chat/completions", public.local_addr().unwrap());
let forward_task = tokio::spawn(async move {
axum::serve(
public,
Forward::new(&origin)
.unwrap()
.router()
.into_make_service_with_connect_info::<SocketAddr>(),
)
.await
.unwrap();
});
let (sender, receiver) = mpsc::channel::<Result<Bytes, Infallible>>(1);
let stream = futures_util::stream::unfold(receiver, |mut receiver| async {
receiver.recv().await.map(|chunk| (chunk, receiver))
});
sender.send(Ok(Bytes::from_static(b"first"))).await.unwrap();
let client: Client<HttpConnector, Body> =
Client::builder(TokioExecutor::new()).build_http();
let mut response = tokio::time::timeout(
Duration::from_secs(5),
client.request(
Request::post(public_uri)
.body(Body::from_stream(stream))
.unwrap(),
),
)
.await
.unwrap()
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let frame = tokio::time::timeout(Duration::from_secs(5), response.body_mut().frame())
.await
.unwrap()
.unwrap()
.unwrap();
assert_eq!(frame.into_data().unwrap(), "first");
let large = Bytes::from(vec![b'x'; 3 * 1024 * 1024]);
sender.send(Ok(large.clone())).await.unwrap();
drop(sender);
let rest = tokio::time::timeout(Duration::from_secs(5), response.into_body().collect())
.await
.unwrap()
.unwrap()
.to_bytes();
assert_eq!(rest, large);
upstream_task.abort();
forward_task.abort();
}
}

View file

@ -1,5 +1,6 @@
mod auth;
mod error;
pub mod forward;
mod redis;
mod secrets;
pub use error::Error;

View file

@ -41,6 +41,30 @@ impl Sink for StderrSink {
#[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
if let Some(upstream) = std::env::var_os("LITELLM_FORWARD_UPSTREAM") {
let settings: settings::ForwardSettings = settings::from_iter(std::env::vars_os())?;
Logger::new(StderrSink {
level: settings.log_level(),
})
.install_global()?;
let forward = litellm_gateway::forward::Forward::new(
&upstream
.into_string()
.map_err(|_| "invalid upstream encoding")?,
)?;
let listener =
tokio::net::TcpListener::bind((settings.host.as_str(), settings.port)).await?;
tracing::info!(address = %listener.local_addr()?, "forwarding gateway listening");
axum::serve(
listener,
forward
.router()
.into_make_service_with_connect_info::<std::net::SocketAddr>(),
)
.with_graceful_shutdown(shutdown_signal())
.await?;
return Ok(());
}
let settings: Settings = settings::from_iter(std::env::vars_os())?;
let level = settings.log_level();
Logger::new(StderrSink { level }).install_global()?;

View file

@ -4,6 +4,33 @@ use litellm_secret_value::SecretValue;
use litellm_tracing::Level;
use serde::{Deserialize, de::DeserializeOwned};
#[derive(Deserialize)]
#[serde(default)]
pub(super) struct ForwardSettings {
pub host: String,
pub port: u16,
rust_log: Option<String>,
}
impl Default for ForwardSettings {
fn default() -> Self {
Self {
host: "0.0.0.0".into(),
port: 4000,
rust_log: None,
}
}
}
impl ForwardSettings {
pub fn log_level(&self) -> Level {
self.rust_log
.as_deref()
.and_then(|value| value.parse().ok())
.unwrap_or(Level::INFO)
}
}
#[derive(Deserialize)]
#[serde(default)]
pub(super) struct Settings {