This commit is contained in:
yujonglee 2026-10-26 12:12:03 -04:00 • committed by GitHub
commit a80d97d2d6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 901 additions and 16 deletions

View file

@ -3765,6 +3765,7 @@ dependencies = [
"envy",
"futures-util",
"http-body-util",
"hyper 1.10.1",
"hyper-util",
"litellm-auth-types",
"litellm-config",
@ -3786,6 +3787,7 @@ dependencies = [
"tempfile",
"thiserror 2.0.19",
"tokio",
"tokio-tungstenite",
"tokio-util",
"tower",
"tower-sessions-moka-store",
@ -7444,6 +7446,7 @@ dependencies = [
"bytes",
"futures-core",
"futures-sink",
"futures-util",
"pin-project-lite",
"tokio",
]

View file

@ -10,12 +10,13 @@ axum = { workspace = true, features = ["original-uri"] }
bytes.workspace = true
envy = "0.4.2"
http-body-util = "0.1"
hyper = { version = "1", features = ["http1"] }
hyper-util = { workspace = true, features = ["client-legacy", "http1", "tokio"] }
litellm-core-utils.workspace = true
litellm-inference.workspace = true
litellm-gateway-inference.workspace = true
litellm-gateway-mcp.workspace = true
tokio-util = "0.7"
tokio-util = { version = "0.7", features = ["rt"] }
thiserror.workspace = true
futures-util.workspace = true
litellm-gateway-auth.workspace = true
@ -42,4 +43,5 @@ base64.workspace = true
rstest.workspace = true
tempfile.workspace = true
tokio = { workspace = true, features = ["sync"] }
tokio-tungstenite.workspace = true
tower = { version = "0.5", features = ["util"] }

View file

@ -4,7 +4,7 @@ use axum::{
body::Body,
extract::{ConnectInfo, OriginalUri, Request},
http::{
HeaderMap, HeaderName, HeaderValue, StatusCode, Uri, Version,
HeaderMap, HeaderName, HeaderValue, Method, StatusCode, Uri, Version,
header::{
CONNECTION, PROXY_AUTHENTICATE, PROXY_AUTHORIZATION, TE, TRAILER, TRANSFER_ENCODING,
UPGRADE,
@ -19,6 +19,8 @@ use hyper_util::{
};
use litellm_core_utils::decline::Decline;
use crate::upgrades::{HANDSHAKE_TIMEOUT, Upgrades};
/// Python reads it to log why the gateway handed the request over.
const DECLINED_HEADER: HeaderName = HeaderName::from_static("x-litellm-gateway-declined");
@ -73,7 +75,36 @@ impl Forward {
})
}
pub(super) async fn send(&self, request: Request, decline: Decline) -> Response {
pub(super) async fn send(&self, mut request: Request, decline: Decline) -> Response {
let websocket = if request.method() == Method::GET
&& request.version() == Version::HTTP_11
&& is_websocket(request.headers())
{
let Some(lifecycle) = request.extensions().get::<Upgrades>().cloned() else {
return (
StatusCode::SERVICE_UNAVAILABLE,
"WebSocket forwarding requires litellm_gateway::serve",
)
.into_response();
};
if lifecycle.is_draining() {
return StatusCode::SERVICE_UNAVAILABLE.into_response();
}
if request
.extensions()
.get::<hyper::upgrade::OnUpgrade>()
.is_none()
{
return (
StatusCode::SERVICE_UNAVAILABLE,
"WebSocket forwarding requires an HTTP upgrade transport",
)
.into_response();
}
Some((hyper::upgrade::on(&mut request), lifecycle))
} else {
None
};
let (mut parts, body) = request.into_parts();
let path = parts
.extensions
@ -92,6 +123,9 @@ impl Forward {
.expect("scheme, authority and path make an absolute uri");
parts.version = Version::HTTP_11;
strip_hop_by_hop(&mut parts.headers);
if websocket.is_some() {
websocket_headers(&mut parts.headers);
}
if let Some(ConnectInfo(client)) = parts.extensions.get::<ConnectInfo<SocketAddr>>() {
append_forwarded_for(&mut parts.headers, *client);
}
@ -103,8 +137,46 @@ impl Forward {
path = parts.uri.path(),
"forwarded to python"
);
match self.client.request(Request::from_parts(parts, body)).await {
Ok(response) => {
let response = self.client.request(Request::from_parts(parts, body));
let response = if let Some((_, lifecycle)) = &websocket {
tokio::select! {
_ = lifecycle.cancelled() => return StatusCode::SERVICE_UNAVAILABLE.into_response(),
result = tokio::time::timeout(HANDSHAKE_TIMEOUT, response) => match result {
Ok(result) => result,
Err(_) => return StatusCode::GATEWAY_TIMEOUT.into_response(),
}
}
} else {
response.await
};
match response {
Ok(mut response) => {
if response.status() == StatusCode::SWITCHING_PROTOCOLS {
let Some((downstream, lifecycle)) = websocket else {
return StatusCode::BAD_GATEWAY.into_response();
};
if !is_websocket(response.headers()) {
return StatusCode::BAD_GATEWAY.into_response();
}
let upstream = hyper::upgrade::on(&mut response);
let upstream = tokio::select! {
_ = lifecycle.cancelled() => {
return StatusCode::SERVICE_UNAVAILABLE.into_response();
}
result = tokio::time::timeout(HANDSHAKE_TIMEOUT, upstream) => {
match result {
Ok(Ok(upstream)) => upstream,
Ok(Err(_)) => return StatusCode::BAD_GATEWAY.into_response(),
Err(_) => return StatusCode::GATEWAY_TIMEOUT.into_response(),
}
}
};
let (mut parts, _) = response.into_parts();
strip_hop_by_hop(&mut parts.headers);
websocket_headers(&mut parts.headers);
lifecycle.relay(downstream, upstream);
return Response::from_parts(parts, Body::empty());
}
let (mut parts, body) = response.into_parts();
strip_hop_by_hop(&mut parts.headers);
Response::from_parts(parts, Body::new(body))
@ -117,6 +189,24 @@ impl Forward {
}
}
fn is_websocket(headers: &HeaderMap) -> bool {
headers
.get(UPGRADE)
.is_some_and(|value| value.as_bytes().eq_ignore_ascii_case(b"websocket"))
&& headers.get_all(CONNECTION).iter().any(|value| {
value.to_str().is_ok_and(|value| {
value
.split(',')
.any(|token| token.trim().eq_ignore_ascii_case("upgrade"))
})
})
}
fn websocket_headers(headers: &mut HeaderMap) {
headers.insert(CONNECTION, HeaderValue::from_static("upgrade"));
headers.insert(UPGRADE, HeaderValue::from_static("websocket"));
}
/// Python sees the gateway as its peer, so the client's address goes on the end of the chain
/// any proxy in front of the gateway already started.
fn append_forwarded_for(headers: &mut HeaderMap, client: SocketAddr) {

View file

@ -74,5 +74,11 @@ fn record(parts: &Parts, body: Bytes) -> Request<Bytes> {
if let Some(&client) = parts.extensions.get::<ConnectInfo<SocketAddr>>() {
recorded.extensions_mut().insert(client);
}
if let Some(upgrade) = parts.extensions.get::<hyper::upgrade::OnUpgrade>() {
recorded.extensions_mut().insert(upgrade.clone());
}
if let Some(lifecycle) = parts.extensions.get::<crate::upgrades::Upgrades>() {
recorded.extensions_mut().insert(lifecycle.clone());
}
recorded
}

View file

@ -2,6 +2,7 @@ mod auth;
mod dispatch;
mod error;
mod secrets;
mod upgrades;
pub use dispatch::{Fallback, Forward, InvalidUpstream};
pub use error::Error;
@ -141,12 +142,25 @@ pub async fn serve(
app: Router,
shutdown: impl Future<Output = ()> + Send + 'static,
) -> std::io::Result<()> {
axum::serve(
let upgrades = upgrades::Upgrades::new();
let _cancel_upgrades = upgrades.cancel.clone().drop_guard();
let draining = upgrades.clone();
let result = axum::serve(
listener,
app.into_make_service_with_connect_info::<std::net::SocketAddr>(),
app.layer(axum::Extension(upgrades.clone()))
.into_make_service_with_connect_info::<std::net::SocketAddr>(),
)
.with_graceful_shutdown(shutdown)
.await
.with_graceful_shutdown(async move {
shutdown.await;
draining.begin_shutdown();
})
.await;
if result.is_err() {
upgrades.cancel.cancel();
}
upgrades.tasks.close();
upgrades.tasks.wait().await;
result
}
async fn log_request(request: Request, next: Next) -> Response {

View file

@ -0,0 +1,71 @@
use std::time::Duration;
use hyper::upgrade::{OnUpgrade, Upgraded};
use hyper_util::rt::TokioIo;
use tokio::{sync::watch, time::Instant};
use tokio_util::{sync::CancellationToken, task::TaskTracker};
pub(crate) const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10);
const DRAIN_TIMEOUT: Duration = Duration::from_secs(5);
#[derive(Clone)]
pub(crate) struct Upgrades {
pub tasks: TaskTracker,
pub cancel: CancellationToken,
deadline: watch::Sender<Option<Instant>>,
}
impl Upgrades {
pub fn new() -> Self {
Self {
tasks: TaskTracker::new(),
cancel: CancellationToken::new(),
deadline: watch::Sender::new(None),
}
}
pub fn begin_shutdown(&self) {
self.deadline
.send_replace(Some(Instant::now() + DRAIN_TIMEOUT));
}
pub fn is_draining(&self) -> bool {
self.deadline.borrow().is_some() || self.cancel.is_cancelled()
}
pub async fn cancelled(&self) {
let mut deadline = self.deadline.subscribe();
tokio::select! {
_ = self.cancel.cancelled() => {}
_ = async {
let at = *deadline.wait_for(Option::is_some).await.expect("sender is owned by self");
tokio::time::sleep_until(at.expect("waited for shutdown deadline")).await;
} => {}
}
}
pub fn relay(&self, downstream: OnUpgrade, upstream: Upgraded) {
let lifecycle = self.clone();
self.tasks.spawn(async move {
tokio::select! {
_ = lifecycle.cancelled() => {}
_ = async {
let upgraded = tokio::time::timeout(HANDSHAKE_TIMEOUT, downstream).await;
let Ok(Ok(downstream)) = upgraded else {
tracing::debug!("websocket upgrade did not complete");
return;
};
let (mut downstream_read, mut downstream_write) = tokio::io::split(TokioIo::new(downstream));
let (mut upstream_read, mut upstream_write) = tokio::io::split(TokioIo::new(upstream));
let result = tokio::select! {
result = tokio::io::copy(&mut downstream_read, &mut upstream_write) => result,
result = tokio::io::copy(&mut upstream_read, &mut downstream_write) => result,
};
if result.is_err() {
tracing::debug!("websocket relay disconnected");
}
} => {}
}
});
}
}

View file

@ -9,20 +9,33 @@ use axum::{
Router,
body::{Body, Bytes, to_bytes},
extract::{ConnectInfo, DefaultBodyLimit},
http::{HeaderMap, Request, StatusCode},
http::{
HeaderMap, HeaderValue, Request, StatusCode,
header::{
AUTHORIZATION, CONNECTION, HOST, ORIGIN, SEC_WEBSOCKET_KEY, SEC_WEBSOCKET_PROTOCOL,
SEC_WEBSOCKET_VERSION, UPGRADE,
},
},
response::{IntoResponse, Response},
};
use futures_util::{StreamExt, stream};
use futures_util::{SinkExt, StreamExt, stream};
use litellm_config::Config;
use litellm_gateway::{Fallback, Forward};
use litellm_gateway_inference::MAX_BODY_BYTES;
use rstest::rstest;
use rstest::{fixture, rstest};
use serde_json::{Value, json};
use tokio::{
net::TcpListener,
net::{TcpListener, TcpStream},
sync::{mpsc, oneshot},
time::timeout,
};
use tokio_tungstenite::{
accept_async, accept_hdr_async, client_async,
tungstenite::{
Message,
handshake::server::{Request as WebSocketRequest, Response as WebSocketResponse},
},
};
use tower::ServiceExt;
use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any};
@ -212,6 +225,383 @@ async fn gateway_in_front_of(python: SocketAddr) -> SocketAddr {
.await
}
#[fixture]
async fn websocket_python() -> SocketAddr {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
tokio::spawn(async move {
let (stream, _) = listener.accept().await.unwrap();
let Ok(mut socket) = accept_async(stream).await else {
return;
};
socket.send(Message::text("from python")).await.unwrap();
if let Some(Ok(message)) = socket.next().await {
socket.send(message).await.unwrap();
socket.close(None).await.unwrap();
}
});
address
}
#[rstest]
#[case::direct_text(false, Message::text("hello"))]
#[case::direct_binary(false, Message::binary(vec![0, 1, 255]))]
#[case::forwarded_text(true, Message::text("hello"))]
#[case::forwarded_binary(true, Message::binary(vec![0, 1, 255]))]
#[tokio::test]
async fn python_serves_websocket_messages(
#[future(awt)] websocket_python: SocketAddr,
#[case] forwarded: bool,
#[case] message: Message,
) {
timeout(Duration::from_secs(5), async {
let address = if forwarded {
gateway_in_front_of(websocket_python).await
} else {
websocket_python
};
let stream = TcpStream::connect(address).await.unwrap();
let (mut socket, response) = client_async(
format!("ws://{address}/realtime?model=public/model"),
stream,
)
.await
.expect("the client should receive a successful WebSocket upgrade");
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
assert_eq!(
socket.next().await.unwrap().unwrap(),
Message::text("from python")
);
socket.send(message.clone()).await.unwrap();
assert_eq!(socket.next().await.unwrap().unwrap(), message);
assert_eq!(socket.next().await.unwrap().unwrap(), Message::Close(None));
})
.await
.expect("the WebSocket exchange should finish without hanging");
}
#[rstest]
#[tokio::test]
// Tungstenite's handshake callback uses a full HTTP response as its error type.
#[allow(clippy::result_large_err)]
async fn forwarded_websocket_preserves_handshake_headers_and_control_frames() {
timeout(Duration::from_secs(5), async {
const WEBSOCKET_EXTENSION: &str =
"permessage-deflate; client_no_context_takeover; server_no_context_takeover";
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let python = listener.local_addr().unwrap();
let (seen_sender, seen_receiver) = oneshot::channel();
let upstream = tokio::spawn(async move {
let (stream, _) = listener.accept().await.unwrap();
let mut socket = accept_hdr_async(
stream,
move |request: &WebSocketRequest, mut response: WebSocketResponse| {
seen_sender
.send((request.uri().to_string(), request.headers().clone()))
.unwrap();
response
.headers_mut()
.insert(SEC_WEBSOCKET_PROTOCOL, HeaderValue::from_static("chat"));
response.headers_mut().insert(
"sec-websocket-extensions",
HeaderValue::from_static(WEBSOCKET_EXTENSION),
);
Ok(response)
},
)
.await
.unwrap();
assert_eq!(
timeout(Duration::from_secs(2), socket.next())
.await
.unwrap()
.unwrap()
.unwrap(),
Message::Ping(vec![7, 8])
);
socket.send(Message::Pong(vec![7, 8])).await.unwrap();
socket.send(Message::text("from python")).await.unwrap();
assert_eq!(
timeout(Duration::from_secs(2), socket.next())
.await
.unwrap()
.unwrap()
.unwrap(),
Message::text("to python")
);
socket.send(Message::text("echo")).await.unwrap();
});
let gateway = gateway_in_front_of(python).await;
let stream = TcpStream::connect(gateway).await.unwrap();
let request = Request::builder()
.uri(format!(
"ws://{gateway}/realtime?model=public/model&trace=1"
))
.header(HOST, gateway.to_string())
.header(AUTHORIZATION, "Bearer gateway-key")
.header(ORIGIN, "https://client.example")
.header(SEC_WEBSOCKET_PROTOCOL, "chat, superchat")
.header("sec-websocket-extensions", WEBSOCKET_EXTENSION)
.header(SEC_WEBSOCKET_KEY, "dGhlIHNhbXBsZSBub25jZQ==")
.header(SEC_WEBSOCKET_VERSION, "13")
.header(UPGRADE, "websocket")
.header(CONNECTION, "Upgrade, x-hop")
.header("x-hop", "remove")
.body(())
.unwrap();
let (mut socket, response) = client_async(request, stream).await.unwrap();
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
assert_eq!(
response.headers().get(SEC_WEBSOCKET_PROTOCOL).unwrap(),
"chat"
);
assert_eq!(
response.headers().get("sec-websocket-extensions").unwrap(),
WEBSOCKET_EXTENSION
);
let (uri, headers) = timeout(Duration::from_secs(2), seen_receiver)
.await
.unwrap()
.unwrap();
assert_eq!(uri, "/realtime?model=public/model&trace=1");
assert_eq!(headers.get(AUTHORIZATION).unwrap(), "Bearer gateway-key");
assert_eq!(headers.get(ORIGIN).unwrap(), "https://client.example");
assert_eq!(
headers.get(SEC_WEBSOCKET_PROTOCOL).unwrap(),
"chat, superchat"
);
assert_eq!(
headers.get("sec-websocket-extensions").unwrap(),
WEBSOCKET_EXTENSION
);
assert_eq!(headers.get(CONNECTION).unwrap(), "upgrade");
assert!(!headers.contains_key("x-hop"));
socket.send(Message::Ping(vec![7, 8])).await.unwrap();
assert_eq!(
timeout(Duration::from_secs(2), socket.next())
.await
.unwrap()
.unwrap()
.unwrap(),
Message::Pong(vec![7, 8])
);
assert_eq!(
timeout(Duration::from_secs(2), socket.next())
.await
.unwrap()
.unwrap()
.unwrap(),
Message::text("from python")
);
socket.send(Message::text("to python")).await.unwrap();
assert_eq!(
timeout(Duration::from_secs(2), socket.next())
.await
.unwrap()
.unwrap()
.unwrap(),
Message::text("echo")
);
drop(socket);
upstream.await.unwrap();
})
.await
.expect("the forwarded WebSocket should preserve its negotiated behavior");
}
#[rstest]
#[tokio::test]
async fn forwarded_websocket_rejection_preserves_status_body_challenge_and_filters_hops() {
let python = MockServer::start().await;
Mock::given(any())
.respond_with(
ResponseTemplate::new(401)
.insert_header("www-authenticate", "Bearer realm=upstream")
.insert_header("connection", "x-upstream-hop")
.insert_header("x-upstream-hop", "remove")
.set_body_string("upstream denied"),
)
.mount(&python)
.await;
let gateway = gateway_in_front_of(*python.address()).await;
let stream = TcpStream::connect(gateway).await.unwrap();
let request = Request::builder()
.uri(format!("ws://{gateway}/realtime?model=public/model"))
.header(HOST, gateway.to_string())
.header(AUTHORIZATION, "Bearer gateway-key")
.header(CONNECTION, "Upgrade")
.header(SEC_WEBSOCKET_KEY, "dGhlIHNhbXBsZSBub25jZQ==")
.header(SEC_WEBSOCKET_VERSION, "13")
.header(UPGRADE, "websocket")
.body(())
.unwrap();
let error = timeout(Duration::from_secs(5), client_async(request, stream))
.await
.unwrap()
.unwrap_err();
let response = match error {
tokio_tungstenite::tungstenite::Error::Http(response) => response,
error => panic!("expected an HTTP rejection, got {error}"),
};
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
assert_eq!(
response.headers().get("www-authenticate").unwrap(),
"Bearer realm=upstream"
);
assert!(!response.headers().contains_key("x-upstream-hop"));
assert_eq!(
response.body().as_deref(),
Some(b"upstream denied".as_slice())
);
let requests = python.received_requests().await.unwrap();
assert_eq!(requests.len(), 1);
assert_eq!(
requests[0].headers.get("authorization").unwrap(),
"Bearer gateway-key"
);
}
#[rstest]
#[case::missing_connection("websocket", "keep-alive")]
#[case::non_websocket_upgrade("h2c", "upgrade")]
#[tokio::test]
async fn malformed_upgrade_is_forwarded_as_http(#[case] upgrade: &str, #[case] connection: &str) {
let python = python().await;
let response = app(&python.uri(), "anthropic/test-model", "http://127.0.0.1:1")
.oneshot(
Request::get("/realtime?model=public/model")
.header("upgrade", upgrade)
.header("connection", connection)
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
to_bytes(response.into_body(), 1024).await.unwrap(),
"from python"
);
let requests = python.received_requests().await.unwrap();
assert_eq!(requests.len(), 1);
assert!(!requests[0].headers.contains_key("upgrade"));
assert!(!requests[0].headers.contains_key("connection"));
}
#[rstest]
#[tokio::test]
async fn downstream_disconnect_reaches_upstream_as_eof() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let python = listener.local_addr().unwrap();
let (ready_sender, ready_receiver) = oneshot::channel();
let (eof_sender, eof_receiver) = oneshot::channel();
let upstream = tokio::spawn(async move {
let (stream, _) = listener.accept().await.unwrap();
let mut socket = accept_async(stream).await.unwrap();
ready_sender.send(()).unwrap();
let result = timeout(Duration::from_secs(2), socket.next())
.await
.unwrap();
eof_sender
.send(result.is_none_or(|message| message.is_err()))
.unwrap();
});
let gateway = gateway_in_front_of(python).await;
let stream = TcpStream::connect(gateway).await.unwrap();
let (socket, response) = timeout(
Duration::from_secs(2),
client_async(format!("ws://{gateway}/realtime"), stream),
)
.await
.unwrap()
.unwrap();
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
timeout(Duration::from_secs(2), ready_receiver)
.await
.unwrap()
.unwrap();
drop(socket);
assert!(
timeout(Duration::from_secs(2), eof_receiver)
.await
.unwrap()
.unwrap()
);
upstream.await.unwrap();
}
#[rstest]
#[tokio::test]
async fn upstream_disconnect_reaches_downstream_as_eof() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let python = listener.local_addr().unwrap();
let upstream = tokio::spawn(async move {
let (stream, _) = listener.accept().await.unwrap();
let socket = accept_async(stream).await.unwrap();
drop(socket);
});
let gateway = gateway_in_front_of(python).await;
let stream = TcpStream::connect(gateway).await.unwrap();
let (mut socket, response) = timeout(
Duration::from_secs(2),
client_async(format!("ws://{gateway}/realtime"), stream),
)
.await
.unwrap()
.unwrap();
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
let result = timeout(Duration::from_secs(2), socket.next())
.await
.unwrap();
assert!(result.is_none_or(|message| message.is_err()));
upstream.await.unwrap();
}
#[rstest]
#[tokio::test]
async fn established_websocket_survives_the_handshake_timeout() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let python = listener.local_addr().unwrap();
let upstream = tokio::spawn(async move {
let (stream, _) = listener.accept().await.unwrap();
let mut socket = accept_async(stream).await.unwrap();
tokio::time::sleep(Duration::from_secs(11)).await;
socket.send(Message::text("still alive")).await.unwrap();
});
let gateway = gateway_in_front_of(python).await;
let stream = TcpStream::connect(gateway).await.unwrap();
let (mut socket, response) = timeout(
Duration::from_secs(5),
client_async(format!("ws://{gateway}/realtime"), stream),
)
.await
.unwrap()
.unwrap();
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
assert_eq!(
timeout(Duration::from_secs(12), socket.next())
.await
.unwrap()
.unwrap()
.unwrap(),
Message::text("still alive")
);
upstream.await.unwrap();
}
async fn recording_python(
respond: fn() -> Response,
) -> (SocketAddr, mpsc::UnboundedReceiver<(HeaderMap, Bytes)>) {

View file

@ -1,10 +1,15 @@
use std::{
net::SocketAddr,
sync::{Arc, mpsc},
time::Duration,
time::{Duration, Instant},
};
use axum::{body::Body, http::Request};
use axum::{
body::Body,
http::{Request, StatusCode},
};
use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
use futures_util::StreamExt;
use litellm_config::Config;
use litellm_gateway::{Fallback, Forward};
use litellm_gateway_inference::{Error, Gateway};
@ -12,7 +17,14 @@ use litellm_http::ClientVariant;
use litellm_tracing::{Logger, Metadata, Record, Sink};
use rstest::{fixture, rstest};
use serde_json::{Value, json};
use tokio::{net::TcpListener, sync::oneshot, time::timeout};
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
net::{TcpListener, TcpStream},
sync::oneshot,
task::JoinHandle,
time::timeout,
};
use tokio_tungstenite::{accept_async, client_async, tungstenite::handshake::derive_accept_key};
use tower::ServiceExt;
struct LogSink(mpsc::Sender<Value>);
@ -34,6 +46,39 @@ fn inference() -> Arc<Gateway> {
litellm_gateway::build_inference(&Config::from_yaml("model_list: []").unwrap()).unwrap()
}
async fn managed_gateway(
inference: Arc<Gateway>,
upstream: SocketAddr,
) -> (
SocketAddr,
oneshot::Sender<()>,
JoinHandle<std::io::Result<()>>,
) {
let config =
Config::from_yaml("model_list: []\ngeneral_settings:\n master_key: gateway-key\n")
.unwrap();
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let fallback = Fallback::Forward(Forward::new(&format!("http://{upstream}")).unwrap());
let app = litellm_gateway::router(inference, &config, None, None, fallback);
let (shutdown, stopped) = oneshot::channel();
let server = tokio::spawn(litellm_gateway::serve(listener, app, async move {
let _ = stopped.await;
}));
(address, shutdown, server)
}
fn websocket_key(request: &[u8]) -> String {
String::from_utf8_lossy(request)
.lines()
.find_map(|line| {
let (name, value) = line.split_once(':')?;
name.eq_ignore_ascii_case("sec-websocket-key")
.then(|| value.trim().to_owned())
})
.unwrap()
}
#[rstest]
#[case::authorized("/v1/messages", Some("Bearer gateway-key"), Some("gateway-key"), 400)]
#[case::missing_token("/v1/messages", None, Some("gateway-key"), 401)]
@ -353,3 +398,267 @@ async fn python_serves_every_request_the_gateway_does_not_route(
assert_eq!(response.status().as_u16(), 401);
}
}
#[rstest]
#[tokio::test]
async fn serve_drains_an_open_websocket_and_closes_both_peers(inference: Arc<Gateway>) {
let upstream_listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let upstream = upstream_listener.local_addr().unwrap();
let (ready_sender, ready_receiver) = oneshot::channel();
let (eof_sender, eof_receiver) = oneshot::channel();
let upstream_task = tokio::spawn(async move {
let (stream, _) = upstream_listener.accept().await.unwrap();
let mut socket = accept_async(stream).await.unwrap();
ready_sender.send(()).unwrap();
let result = timeout(Duration::from_secs(7), socket.next())
.await
.unwrap();
eof_sender
.send(result.is_none_or(|message| message.is_err()))
.unwrap();
});
let (gateway, shutdown, server) = managed_gateway(inference, upstream).await;
let stream = TcpStream::connect(gateway).await.unwrap();
let (mut socket, response) = timeout(
Duration::from_secs(2),
client_async(format!("ws://{gateway}/realtime"), stream),
)
.await
.unwrap()
.unwrap();
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
timeout(Duration::from_secs(2), ready_receiver)
.await
.unwrap()
.unwrap();
let started = Instant::now();
shutdown.send(()).unwrap();
timeout(Duration::from_secs(6), server)
.await
.unwrap()
.unwrap()
.unwrap();
assert!(started.elapsed() <= Duration::from_secs(6));
let result = timeout(Duration::from_secs(2), socket.next())
.await
.unwrap();
assert!(result.is_none_or(|message| message.is_err()));
assert!(
timeout(Duration::from_secs(2), eof_receiver)
.await
.unwrap()
.unwrap()
);
upstream_task.await.unwrap();
}
#[rstest]
#[tokio::test]
async fn shutdown_deadline_closes_a_handshake_released_after_shutdown(inference: Arc<Gateway>) {
let upstream_listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let upstream = upstream_listener.local_addr().unwrap();
let (request_sender, request_receiver) = oneshot::channel();
let (release_sender, release_receiver) = oneshot::channel();
let (closed_sender, closed_receiver) = oneshot::channel();
let upstream_task = tokio::spawn(async move {
let (mut stream, _) = upstream_listener.accept().await.unwrap();
let mut request = Vec::new();
let mut chunk = [0; 4096];
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
let read = stream.read(&mut chunk).await.unwrap();
if read == 0 {
return;
}
request.extend_from_slice(&chunk[..read]);
}
let key = websocket_key(&request);
request_sender.send(()).unwrap();
release_receiver.await.unwrap();
let accept = derive_accept_key(key.as_bytes());
let response = format!(
"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Accept: {accept}\r\n\r\n"
);
stream.write_all(response.as_bytes()).await.unwrap();
let mut bytes = [0; 64];
let closed = matches!(
timeout(Duration::from_secs(3), stream.read(&mut bytes)).await,
Ok(Ok(0))
);
closed_sender.send(closed).unwrap();
});
let config =
Config::from_yaml("model_list: []\ngeneral_settings:\n master_key: gateway-key\n")
.unwrap();
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let gateway = listener.local_addr().unwrap();
let fallback = Fallback::Forward(Forward::new(&format!("http://{upstream}")).unwrap());
let app = litellm_gateway::router(inference, &config, None, None, fallback);
let (shutdown, stopped) = oneshot::channel();
let (began_sender, began_receiver) = oneshot::channel();
let server = tokio::spawn(litellm_gateway::serve(listener, app, async move {
let _ = stopped.await;
began_sender.send(()).unwrap();
}));
let client = tokio::spawn(async move {
let mut stream = TcpStream::connect(gateway).await.unwrap();
let request = format!(
"GET /realtime HTTP/1.1\r\nHost: {gateway}\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: 13\r\n\r\n"
);
stream.write_all(request.as_bytes()).await.unwrap();
let mut response = Vec::new();
let mut chunk = [0; 1024];
while !response.windows(4).any(|window| window == b"\r\n\r\n") {
let read = stream.read(&mut chunk).await.unwrap();
if read == 0 {
break;
}
response.extend_from_slice(&chunk[..read]);
}
(stream, response)
});
timeout(Duration::from_secs(2), request_receiver)
.await
.unwrap()
.unwrap();
shutdown.send(()).unwrap();
timeout(Duration::from_secs(2), began_receiver)
.await
.unwrap()
.unwrap();
tokio::time::sleep(Duration::from_secs(4)).await;
release_sender.send(()).unwrap();
let (mut socket, response) = timeout(Duration::from_secs(3), client)
.await
.unwrap()
.unwrap();
assert!(response.starts_with(b"HTTP/1.1 101"));
let result = timeout(Duration::from_secs(3), socket.read(&mut [0; 64]))
.await
.unwrap();
assert_eq!(result.unwrap(), 0);
timeout(Duration::from_secs(3), server)
.await
.unwrap()
.unwrap()
.unwrap();
assert!(
timeout(Duration::from_secs(2), closed_receiver)
.await
.unwrap()
.unwrap()
);
upstream_task.await.unwrap();
}
#[rstest]
#[tokio::test]
async fn dropping_serve_cancels_an_open_websocket(inference: Arc<Gateway>) {
let upstream_listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let upstream = upstream_listener.local_addr().unwrap();
let (ready_sender, ready_receiver) = oneshot::channel();
let (eof_sender, eof_receiver) = oneshot::channel();
let upstream_task = tokio::spawn(async move {
let (stream, _) = upstream_listener.accept().await.unwrap();
let mut socket = accept_async(stream).await.unwrap();
ready_sender.send(()).unwrap();
let result = timeout(Duration::from_secs(2), socket.next())
.await
.unwrap();
eof_sender
.send(result.is_none_or(|message| message.is_err()))
.unwrap();
});
let (gateway, _shutdown, server) = managed_gateway(inference, upstream).await;
let stream = TcpStream::connect(gateway).await.unwrap();
let (mut socket, response) = timeout(
Duration::from_secs(2),
client_async(format!("ws://{gateway}/realtime"), stream),
)
.await
.unwrap()
.unwrap();
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
timeout(Duration::from_secs(2), ready_receiver)
.await
.unwrap()
.unwrap();
server.abort();
assert!(
timeout(Duration::from_secs(2), server)
.await
.unwrap()
.unwrap_err()
.is_cancelled()
);
let result = timeout(Duration::from_secs(2), socket.next())
.await
.unwrap();
assert!(result.is_none_or(|message| message.is_err()));
assert!(
timeout(Duration::from_secs(2), eof_receiver)
.await
.unwrap()
.unwrap()
);
upstream_task.await.unwrap();
}
#[rstest]
#[tokio::test]
async fn unmanaged_axum_server_rejects_websocket_forwarding(inference: Arc<Gateway>) {
let upstream = TcpListener::bind("127.0.0.1:0").await.unwrap();
let upstream_address = upstream.local_addr().unwrap();
let config =
Config::from_yaml("model_list: []\ngeneral_settings:\n master_key: gateway-key\n")
.unwrap();
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let gateway = listener.local_addr().unwrap();
let fallback = Fallback::Forward(Forward::new(&format!("http://{upstream_address}")).unwrap());
let app = litellm_gateway::router(inference, &config, None, None, fallback);
let (shutdown, stopped) = oneshot::channel();
let server = tokio::spawn(async move {
axum::serve(listener, app)
.with_graceful_shutdown(async move {
let _ = stopped.await;
})
.await
});
let stream = TcpStream::connect(gateway).await.unwrap();
let request = axum::http::Request::builder()
.uri(format!("ws://{gateway}/realtime"))
.header("Host", gateway.to_string())
.header("Connection", "Upgrade")
.header("Upgrade", "websocket")
.header("Sec-WebSocket-Key", "dGhlIHNhbXBsZSBub25jZQ==")
.header("Sec-WebSocket-Version", "13")
.body(())
.unwrap();
let error = timeout(Duration::from_secs(2), client_async(request, stream))
.await
.unwrap()
.unwrap_err();
let response = match error {
tokio_tungstenite::tungstenite::Error::Http(response) => response,
error => panic!("expected an HTTP rejection, got {error}"),
};
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
assert_ne!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
let body = String::from_utf8(response.body().clone().unwrap()).unwrap();
assert!(body.contains("requires litellm_gateway::serve"));
shutdown.send(()).unwrap();
timeout(Duration::from_secs(5), server)
.await
.unwrap()
.unwrap()
.unwrap();
}