diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 4f40c4e1f32..159c43aa717 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3765,6 +3765,7 @@ dependencies = [ "envy", "futures-util", "http-body-util", + "hyper 1.10.1", "hyper-util", "litellm-auth-types", "litellm-config", @@ -3786,6 +3787,7 @@ dependencies = [ "tempfile", "thiserror 2.0.19", "tokio", + "tokio-tungstenite", "tokio-util", "tower", "tower-sessions-moka-store", @@ -7444,6 +7446,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", "pin-project-lite", "tokio", ] diff --git a/litellm-rust/crates/gateway/Cargo.toml b/litellm-rust/crates/gateway/Cargo.toml index 4135ee4b0ff..c87824c459a 100644 --- a/litellm-rust/crates/gateway/Cargo.toml +++ b/litellm-rust/crates/gateway/Cargo.toml @@ -10,12 +10,13 @@ axum = { workspace = true, features = ["original-uri"] } bytes.workspace = true envy = "0.4.2" http-body-util = "0.1" +hyper = { version = "1", features = ["http1"] } hyper-util = { workspace = true, features = ["client-legacy", "http1", "tokio"] } litellm-core-utils.workspace = true litellm-inference.workspace = true litellm-gateway-inference.workspace = true litellm-gateway-mcp.workspace = true -tokio-util = "0.7" +tokio-util = { version = "0.7", features = ["rt"] } thiserror.workspace = true futures-util.workspace = true litellm-gateway-auth.workspace = true @@ -42,4 +43,5 @@ base64.workspace = true rstest.workspace = true tempfile.workspace = true tokio = { workspace = true, features = ["sync"] } +tokio-tungstenite.workspace = true tower = { version = "0.5", features = ["util"] } diff --git a/litellm-rust/crates/gateway/src/dispatch/forward.rs b/litellm-rust/crates/gateway/src/dispatch/forward.rs index 255df04a0a4..b31587c96f8 100644 --- a/litellm-rust/crates/gateway/src/dispatch/forward.rs +++ b/litellm-rust/crates/gateway/src/dispatch/forward.rs @@ -4,7 +4,7 @@ use axum::{ body::Body, extract::{ConnectInfo, OriginalUri, Request}, http::{ - HeaderMap, HeaderName, HeaderValue, StatusCode, Uri, Version, + HeaderMap, HeaderName, HeaderValue, Method, StatusCode, Uri, Version, header::{ CONNECTION, PROXY_AUTHENTICATE, PROXY_AUTHORIZATION, TE, TRAILER, TRANSFER_ENCODING, UPGRADE, @@ -19,6 +19,8 @@ use hyper_util::{ }; use litellm_core_utils::decline::Decline; +use crate::upgrades::{HANDSHAKE_TIMEOUT, Upgrades}; + /// Python reads it to log why the gateway handed the request over. const DECLINED_HEADER: HeaderName = HeaderName::from_static("x-litellm-gateway-declined"); @@ -73,7 +75,36 @@ impl Forward { }) } - pub(super) async fn send(&self, request: Request, decline: Decline) -> Response { + pub(super) async fn send(&self, mut request: Request, decline: Decline) -> Response { + let websocket = if request.method() == Method::GET + && request.version() == Version::HTTP_11 + && is_websocket(request.headers()) + { + let Some(lifecycle) = request.extensions().get::().cloned() else { + return ( + StatusCode::SERVICE_UNAVAILABLE, + "WebSocket forwarding requires litellm_gateway::serve", + ) + .into_response(); + }; + if lifecycle.is_draining() { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + if request + .extensions() + .get::() + .is_none() + { + return ( + StatusCode::SERVICE_UNAVAILABLE, + "WebSocket forwarding requires an HTTP upgrade transport", + ) + .into_response(); + } + Some((hyper::upgrade::on(&mut request), lifecycle)) + } else { + None + }; let (mut parts, body) = request.into_parts(); let path = parts .extensions @@ -92,6 +123,9 @@ impl Forward { .expect("scheme, authority and path make an absolute uri"); parts.version = Version::HTTP_11; strip_hop_by_hop(&mut parts.headers); + if websocket.is_some() { + websocket_headers(&mut parts.headers); + } if let Some(ConnectInfo(client)) = parts.extensions.get::>() { append_forwarded_for(&mut parts.headers, *client); } @@ -103,8 +137,46 @@ impl Forward { path = parts.uri.path(), "forwarded to python" ); - match self.client.request(Request::from_parts(parts, body)).await { - Ok(response) => { + let response = self.client.request(Request::from_parts(parts, body)); + let response = if let Some((_, lifecycle)) = &websocket { + tokio::select! { + _ = lifecycle.cancelled() => return StatusCode::SERVICE_UNAVAILABLE.into_response(), + result = tokio::time::timeout(HANDSHAKE_TIMEOUT, response) => match result { + Ok(result) => result, + Err(_) => return StatusCode::GATEWAY_TIMEOUT.into_response(), + } + } + } else { + response.await + }; + match response { + Ok(mut response) => { + if response.status() == StatusCode::SWITCHING_PROTOCOLS { + let Some((downstream, lifecycle)) = websocket else { + return StatusCode::BAD_GATEWAY.into_response(); + }; + if !is_websocket(response.headers()) { + return StatusCode::BAD_GATEWAY.into_response(); + } + let upstream = hyper::upgrade::on(&mut response); + let upstream = tokio::select! { + _ = lifecycle.cancelled() => { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + result = tokio::time::timeout(HANDSHAKE_TIMEOUT, upstream) => { + match result { + Ok(Ok(upstream)) => upstream, + Ok(Err(_)) => return StatusCode::BAD_GATEWAY.into_response(), + Err(_) => return StatusCode::GATEWAY_TIMEOUT.into_response(), + } + } + }; + let (mut parts, _) = response.into_parts(); + strip_hop_by_hop(&mut parts.headers); + websocket_headers(&mut parts.headers); + lifecycle.relay(downstream, upstream); + return Response::from_parts(parts, Body::empty()); + } let (mut parts, body) = response.into_parts(); strip_hop_by_hop(&mut parts.headers); Response::from_parts(parts, Body::new(body)) @@ -117,6 +189,24 @@ impl Forward { } } +fn is_websocket(headers: &HeaderMap) -> bool { + headers + .get(UPGRADE) + .is_some_and(|value| value.as_bytes().eq_ignore_ascii_case(b"websocket")) + && headers.get_all(CONNECTION).iter().any(|value| { + value.to_str().is_ok_and(|value| { + value + .split(',') + .any(|token| token.trim().eq_ignore_ascii_case("upgrade")) + }) + }) +} + +fn websocket_headers(headers: &mut HeaderMap) { + headers.insert(CONNECTION, HeaderValue::from_static("upgrade")); + headers.insert(UPGRADE, HeaderValue::from_static("websocket")); +} + /// Python sees the gateway as its peer, so the client's address goes on the end of the chain /// any proxy in front of the gateway already started. fn append_forwarded_for(headers: &mut HeaderMap, client: SocketAddr) { diff --git a/litellm-rust/crates/gateway/src/dispatch/replay.rs b/litellm-rust/crates/gateway/src/dispatch/replay.rs index 0d371cd7902..bced070729e 100644 --- a/litellm-rust/crates/gateway/src/dispatch/replay.rs +++ b/litellm-rust/crates/gateway/src/dispatch/replay.rs @@ -74,5 +74,11 @@ fn record(parts: &Parts, body: Bytes) -> Request { if let Some(&client) = parts.extensions.get::>() { recorded.extensions_mut().insert(client); } + if let Some(upgrade) = parts.extensions.get::() { + recorded.extensions_mut().insert(upgrade.clone()); + } + if let Some(lifecycle) = parts.extensions.get::() { + recorded.extensions_mut().insert(lifecycle.clone()); + } recorded } diff --git a/litellm-rust/crates/gateway/src/lib.rs b/litellm-rust/crates/gateway/src/lib.rs index 63fb0bd7c42..fc67879a9b2 100644 --- a/litellm-rust/crates/gateway/src/lib.rs +++ b/litellm-rust/crates/gateway/src/lib.rs @@ -2,6 +2,7 @@ mod auth; mod dispatch; mod error; mod secrets; +mod upgrades; pub use dispatch::{Fallback, Forward, InvalidUpstream}; pub use error::Error; @@ -141,12 +142,25 @@ pub async fn serve( app: Router, shutdown: impl Future + Send + 'static, ) -> std::io::Result<()> { - axum::serve( + let upgrades = upgrades::Upgrades::new(); + let _cancel_upgrades = upgrades.cancel.clone().drop_guard(); + let draining = upgrades.clone(); + let result = axum::serve( listener, - app.into_make_service_with_connect_info::(), + app.layer(axum::Extension(upgrades.clone())) + .into_make_service_with_connect_info::(), ) - .with_graceful_shutdown(shutdown) - .await + .with_graceful_shutdown(async move { + shutdown.await; + draining.begin_shutdown(); + }) + .await; + if result.is_err() { + upgrades.cancel.cancel(); + } + upgrades.tasks.close(); + upgrades.tasks.wait().await; + result } async fn log_request(request: Request, next: Next) -> Response { diff --git a/litellm-rust/crates/gateway/src/upgrades.rs b/litellm-rust/crates/gateway/src/upgrades.rs new file mode 100644 index 00000000000..199b16bfa03 --- /dev/null +++ b/litellm-rust/crates/gateway/src/upgrades.rs @@ -0,0 +1,71 @@ +use std::time::Duration; + +use hyper::upgrade::{OnUpgrade, Upgraded}; +use hyper_util::rt::TokioIo; +use tokio::{sync::watch, time::Instant}; +use tokio_util::{sync::CancellationToken, task::TaskTracker}; + +pub(crate) const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10); +const DRAIN_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(Clone)] +pub(crate) struct Upgrades { + pub tasks: TaskTracker, + pub cancel: CancellationToken, + deadline: watch::Sender>, +} + +impl Upgrades { + pub fn new() -> Self { + Self { + tasks: TaskTracker::new(), + cancel: CancellationToken::new(), + deadline: watch::Sender::new(None), + } + } + + pub fn begin_shutdown(&self) { + self.deadline + .send_replace(Some(Instant::now() + DRAIN_TIMEOUT)); + } + + pub fn is_draining(&self) -> bool { + self.deadline.borrow().is_some() || self.cancel.is_cancelled() + } + + pub async fn cancelled(&self) { + let mut deadline = self.deadline.subscribe(); + tokio::select! { + _ = self.cancel.cancelled() => {} + _ = async { + let at = *deadline.wait_for(Option::is_some).await.expect("sender is owned by self"); + tokio::time::sleep_until(at.expect("waited for shutdown deadline")).await; + } => {} + } + } + + pub fn relay(&self, downstream: OnUpgrade, upstream: Upgraded) { + let lifecycle = self.clone(); + self.tasks.spawn(async move { + tokio::select! { + _ = lifecycle.cancelled() => {} + _ = async { + let upgraded = tokio::time::timeout(HANDSHAKE_TIMEOUT, downstream).await; + let Ok(Ok(downstream)) = upgraded else { + tracing::debug!("websocket upgrade did not complete"); + return; + }; + let (mut downstream_read, mut downstream_write) = tokio::io::split(TokioIo::new(downstream)); + let (mut upstream_read, mut upstream_write) = tokio::io::split(TokioIo::new(upstream)); + let result = tokio::select! { + result = tokio::io::copy(&mut downstream_read, &mut upstream_write) => result, + result = tokio::io::copy(&mut upstream_read, &mut downstream_write) => result, + }; + if result.is_err() { + tracing::debug!("websocket relay disconnected"); + } + } => {} + } + }); + } +} diff --git a/litellm-rust/crates/gateway/tests/dispatch.rs b/litellm-rust/crates/gateway/tests/dispatch.rs index f4a00a185a0..779baf743b2 100644 --- a/litellm-rust/crates/gateway/tests/dispatch.rs +++ b/litellm-rust/crates/gateway/tests/dispatch.rs @@ -9,20 +9,33 @@ use axum::{ Router, body::{Body, Bytes, to_bytes}, extract::{ConnectInfo, DefaultBodyLimit}, - http::{HeaderMap, Request, StatusCode}, + http::{ + HeaderMap, HeaderValue, Request, StatusCode, + header::{ + AUTHORIZATION, CONNECTION, HOST, ORIGIN, SEC_WEBSOCKET_KEY, SEC_WEBSOCKET_PROTOCOL, + SEC_WEBSOCKET_VERSION, UPGRADE, + }, + }, response::{IntoResponse, Response}, }; -use futures_util::{StreamExt, stream}; +use futures_util::{SinkExt, StreamExt, stream}; use litellm_config::Config; use litellm_gateway::{Fallback, Forward}; use litellm_gateway_inference::MAX_BODY_BYTES; -use rstest::rstest; +use rstest::{fixture, rstest}; use serde_json::{Value, json}; use tokio::{ - net::TcpListener, + net::{TcpListener, TcpStream}, sync::{mpsc, oneshot}, time::timeout, }; +use tokio_tungstenite::{ + accept_async, accept_hdr_async, client_async, + tungstenite::{ + Message, + handshake::server::{Request as WebSocketRequest, Response as WebSocketResponse}, + }, +}; use tower::ServiceExt; use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any}; @@ -212,6 +225,383 @@ async fn gateway_in_front_of(python: SocketAddr) -> SocketAddr { .await } +#[fixture] +async fn websocket_python() -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let Ok(mut socket) = accept_async(stream).await else { + return; + }; + socket.send(Message::text("from python")).await.unwrap(); + if let Some(Ok(message)) = socket.next().await { + socket.send(message).await.unwrap(); + socket.close(None).await.unwrap(); + } + }); + address +} + +#[rstest] +#[case::direct_text(false, Message::text("hello"))] +#[case::direct_binary(false, Message::binary(vec![0, 1, 255]))] +#[case::forwarded_text(true, Message::text("hello"))] +#[case::forwarded_binary(true, Message::binary(vec![0, 1, 255]))] +#[tokio::test] +async fn python_serves_websocket_messages( + #[future(awt)] websocket_python: SocketAddr, + #[case] forwarded: bool, + #[case] message: Message, +) { + timeout(Duration::from_secs(5), async { + let address = if forwarded { + gateway_in_front_of(websocket_python).await + } else { + websocket_python + }; + let stream = TcpStream::connect(address).await.unwrap(); + let (mut socket, response) = client_async( + format!("ws://{address}/realtime?model=public/model"), + stream, + ) + .await + .expect("the client should receive a successful WebSocket upgrade"); + + assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS); + assert_eq!( + socket.next().await.unwrap().unwrap(), + Message::text("from python") + ); + socket.send(message.clone()).await.unwrap(); + assert_eq!(socket.next().await.unwrap().unwrap(), message); + assert_eq!(socket.next().await.unwrap().unwrap(), Message::Close(None)); + }) + .await + .expect("the WebSocket exchange should finish without hanging"); +} + +#[rstest] +#[tokio::test] +// Tungstenite's handshake callback uses a full HTTP response as its error type. +#[allow(clippy::result_large_err)] +async fn forwarded_websocket_preserves_handshake_headers_and_control_frames() { + timeout(Duration::from_secs(5), async { + const WEBSOCKET_EXTENSION: &str = + "permessage-deflate; client_no_context_takeover; server_no_context_takeover"; + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let python = listener.local_addr().unwrap(); + let (seen_sender, seen_receiver) = oneshot::channel(); + let upstream = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_hdr_async( + stream, + move |request: &WebSocketRequest, mut response: WebSocketResponse| { + seen_sender + .send((request.uri().to_string(), request.headers().clone())) + .unwrap(); + response + .headers_mut() + .insert(SEC_WEBSOCKET_PROTOCOL, HeaderValue::from_static("chat")); + response.headers_mut().insert( + "sec-websocket-extensions", + HeaderValue::from_static(WEBSOCKET_EXTENSION), + ); + Ok(response) + }, + ) + .await + .unwrap(); + + assert_eq!( + timeout(Duration::from_secs(2), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(), + Message::Ping(vec![7, 8]) + ); + socket.send(Message::Pong(vec![7, 8])).await.unwrap(); + socket.send(Message::text("from python")).await.unwrap(); + assert_eq!( + timeout(Duration::from_secs(2), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(), + Message::text("to python") + ); + socket.send(Message::text("echo")).await.unwrap(); + }); + + let gateway = gateway_in_front_of(python).await; + let stream = TcpStream::connect(gateway).await.unwrap(); + let request = Request::builder() + .uri(format!( + "ws://{gateway}/realtime?model=public/model&trace=1" + )) + .header(HOST, gateway.to_string()) + .header(AUTHORIZATION, "Bearer gateway-key") + .header(ORIGIN, "https://client.example") + .header(SEC_WEBSOCKET_PROTOCOL, "chat, superchat") + .header("sec-websocket-extensions", WEBSOCKET_EXTENSION) + .header(SEC_WEBSOCKET_KEY, "dGhlIHNhbXBsZSBub25jZQ==") + .header(SEC_WEBSOCKET_VERSION, "13") + .header(UPGRADE, "websocket") + .header(CONNECTION, "Upgrade, x-hop") + .header("x-hop", "remove") + .body(()) + .unwrap(); + let (mut socket, response) = client_async(request, stream).await.unwrap(); + + assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS); + assert_eq!( + response.headers().get(SEC_WEBSOCKET_PROTOCOL).unwrap(), + "chat" + ); + assert_eq!( + response.headers().get("sec-websocket-extensions").unwrap(), + WEBSOCKET_EXTENSION + ); + let (uri, headers) = timeout(Duration::from_secs(2), seen_receiver) + .await + .unwrap() + .unwrap(); + assert_eq!(uri, "/realtime?model=public/model&trace=1"); + assert_eq!(headers.get(AUTHORIZATION).unwrap(), "Bearer gateway-key"); + assert_eq!(headers.get(ORIGIN).unwrap(), "https://client.example"); + assert_eq!( + headers.get(SEC_WEBSOCKET_PROTOCOL).unwrap(), + "chat, superchat" + ); + assert_eq!( + headers.get("sec-websocket-extensions").unwrap(), + WEBSOCKET_EXTENSION + ); + assert_eq!(headers.get(CONNECTION).unwrap(), "upgrade"); + assert!(!headers.contains_key("x-hop")); + + socket.send(Message::Ping(vec![7, 8])).await.unwrap(); + assert_eq!( + timeout(Duration::from_secs(2), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(), + Message::Pong(vec![7, 8]) + ); + assert_eq!( + timeout(Duration::from_secs(2), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(), + Message::text("from python") + ); + socket.send(Message::text("to python")).await.unwrap(); + assert_eq!( + timeout(Duration::from_secs(2), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(), + Message::text("echo") + ); + drop(socket); + upstream.await.unwrap(); + }) + .await + .expect("the forwarded WebSocket should preserve its negotiated behavior"); +} + +#[rstest] +#[tokio::test] +async fn forwarded_websocket_rejection_preserves_status_body_challenge_and_filters_hops() { + let python = MockServer::start().await; + Mock::given(any()) + .respond_with( + ResponseTemplate::new(401) + .insert_header("www-authenticate", "Bearer realm=upstream") + .insert_header("connection", "x-upstream-hop") + .insert_header("x-upstream-hop", "remove") + .set_body_string("upstream denied"), + ) + .mount(&python) + .await; + let gateway = gateway_in_front_of(*python.address()).await; + let stream = TcpStream::connect(gateway).await.unwrap(); + let request = Request::builder() + .uri(format!("ws://{gateway}/realtime?model=public/model")) + .header(HOST, gateway.to_string()) + .header(AUTHORIZATION, "Bearer gateway-key") + .header(CONNECTION, "Upgrade") + .header(SEC_WEBSOCKET_KEY, "dGhlIHNhbXBsZSBub25jZQ==") + .header(SEC_WEBSOCKET_VERSION, "13") + .header(UPGRADE, "websocket") + .body(()) + .unwrap(); + + let error = timeout(Duration::from_secs(5), client_async(request, stream)) + .await + .unwrap() + .unwrap_err(); + let response = match error { + tokio_tungstenite::tungstenite::Error::Http(response) => response, + error => panic!("expected an HTTP rejection, got {error}"), + }; + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert_eq!( + response.headers().get("www-authenticate").unwrap(), + "Bearer realm=upstream" + ); + assert!(!response.headers().contains_key("x-upstream-hop")); + assert_eq!( + response.body().as_deref(), + Some(b"upstream denied".as_slice()) + ); + + let requests = python.received_requests().await.unwrap(); + assert_eq!(requests.len(), 1); + assert_eq!( + requests[0].headers.get("authorization").unwrap(), + "Bearer gateway-key" + ); +} + +#[rstest] +#[case::missing_connection("websocket", "keep-alive")] +#[case::non_websocket_upgrade("h2c", "upgrade")] +#[tokio::test] +async fn malformed_upgrade_is_forwarded_as_http(#[case] upgrade: &str, #[case] connection: &str) { + let python = python().await; + let response = app(&python.uri(), "anthropic/test-model", "http://127.0.0.1:1") + .oneshot( + Request::get("/realtime?model=public/model") + .header("upgrade", upgrade) + .header("connection", connection) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + to_bytes(response.into_body(), 1024).await.unwrap(), + "from python" + ); + let requests = python.received_requests().await.unwrap(); + assert_eq!(requests.len(), 1); + assert!(!requests[0].headers.contains_key("upgrade")); + assert!(!requests[0].headers.contains_key("connection")); +} + +#[rstest] +#[tokio::test] +async fn downstream_disconnect_reaches_upstream_as_eof() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let python = listener.local_addr().unwrap(); + let (ready_sender, ready_receiver) = oneshot::channel(); + let (eof_sender, eof_receiver) = oneshot::channel(); + let upstream = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + ready_sender.send(()).unwrap(); + let result = timeout(Duration::from_secs(2), socket.next()) + .await + .unwrap(); + eof_sender + .send(result.is_none_or(|message| message.is_err())) + .unwrap(); + }); + + let gateway = gateway_in_front_of(python).await; + let stream = TcpStream::connect(gateway).await.unwrap(); + let (socket, response) = timeout( + Duration::from_secs(2), + client_async(format!("ws://{gateway}/realtime"), stream), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS); + timeout(Duration::from_secs(2), ready_receiver) + .await + .unwrap() + .unwrap(); + drop(socket); + + assert!( + timeout(Duration::from_secs(2), eof_receiver) + .await + .unwrap() + .unwrap() + ); + upstream.await.unwrap(); +} + +#[rstest] +#[tokio::test] +async fn upstream_disconnect_reaches_downstream_as_eof() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let python = listener.local_addr().unwrap(); + let upstream = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let socket = accept_async(stream).await.unwrap(); + drop(socket); + }); + + let gateway = gateway_in_front_of(python).await; + let stream = TcpStream::connect(gateway).await.unwrap(); + let (mut socket, response) = timeout( + Duration::from_secs(2), + client_async(format!("ws://{gateway}/realtime"), stream), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS); + let result = timeout(Duration::from_secs(2), socket.next()) + .await + .unwrap(); + assert!(result.is_none_or(|message| message.is_err())); + upstream.await.unwrap(); +} + +#[rstest] +#[tokio::test] +async fn established_websocket_survives_the_handshake_timeout() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let python = listener.local_addr().unwrap(); + let upstream = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + tokio::time::sleep(Duration::from_secs(11)).await; + socket.send(Message::text("still alive")).await.unwrap(); + }); + + let gateway = gateway_in_front_of(python).await; + let stream = TcpStream::connect(gateway).await.unwrap(); + let (mut socket, response) = timeout( + Duration::from_secs(5), + client_async(format!("ws://{gateway}/realtime"), stream), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS); + assert_eq!( + timeout(Duration::from_secs(12), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(), + Message::text("still alive") + ); + upstream.await.unwrap(); +} + async fn recording_python( respond: fn() -> Response, ) -> (SocketAddr, mpsc::UnboundedReceiver<(HeaderMap, Bytes)>) { diff --git a/litellm-rust/crates/gateway/tests/server.rs b/litellm-rust/crates/gateway/tests/server.rs index 1f86ff8825a..c445a2a4d35 100644 --- a/litellm-rust/crates/gateway/tests/server.rs +++ b/litellm-rust/crates/gateway/tests/server.rs @@ -1,10 +1,15 @@ use std::{ + net::SocketAddr, sync::{Arc, mpsc}, - time::Duration, + time::{Duration, Instant}, }; -use axum::{body::Body, http::Request}; +use axum::{ + body::Body, + http::{Request, StatusCode}, +}; use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; +use futures_util::StreamExt; use litellm_config::Config; use litellm_gateway::{Fallback, Forward}; use litellm_gateway_inference::{Error, Gateway}; @@ -12,7 +17,14 @@ use litellm_http::ClientVariant; use litellm_tracing::{Logger, Metadata, Record, Sink}; use rstest::{fixture, rstest}; use serde_json::{Value, json}; -use tokio::{net::TcpListener, sync::oneshot, time::timeout}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, + sync::oneshot, + task::JoinHandle, + time::timeout, +}; +use tokio_tungstenite::{accept_async, client_async, tungstenite::handshake::derive_accept_key}; use tower::ServiceExt; struct LogSink(mpsc::Sender); @@ -34,6 +46,39 @@ fn inference() -> Arc { litellm_gateway::build_inference(&Config::from_yaml("model_list: []").unwrap()).unwrap() } +async fn managed_gateway( + inference: Arc, + upstream: SocketAddr, +) -> ( + SocketAddr, + oneshot::Sender<()>, + JoinHandle>, +) { + let config = + Config::from_yaml("model_list: []\ngeneral_settings:\n master_key: gateway-key\n") + .unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let fallback = Fallback::Forward(Forward::new(&format!("http://{upstream}")).unwrap()); + let app = litellm_gateway::router(inference, &config, None, None, fallback); + let (shutdown, stopped) = oneshot::channel(); + let server = tokio::spawn(litellm_gateway::serve(listener, app, async move { + let _ = stopped.await; + })); + (address, shutdown, server) +} + +fn websocket_key(request: &[u8]) -> String { + String::from_utf8_lossy(request) + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("sec-websocket-key") + .then(|| value.trim().to_owned()) + }) + .unwrap() +} + #[rstest] #[case::authorized("/v1/messages", Some("Bearer gateway-key"), Some("gateway-key"), 400)] #[case::missing_token("/v1/messages", None, Some("gateway-key"), 401)] @@ -353,3 +398,267 @@ async fn python_serves_every_request_the_gateway_does_not_route( assert_eq!(response.status().as_u16(), 401); } } + +#[rstest] +#[tokio::test] +async fn serve_drains_an_open_websocket_and_closes_both_peers(inference: Arc) { + let upstream_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let upstream = upstream_listener.local_addr().unwrap(); + let (ready_sender, ready_receiver) = oneshot::channel(); + let (eof_sender, eof_receiver) = oneshot::channel(); + let upstream_task = tokio::spawn(async move { + let (stream, _) = upstream_listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + ready_sender.send(()).unwrap(); + let result = timeout(Duration::from_secs(7), socket.next()) + .await + .unwrap(); + eof_sender + .send(result.is_none_or(|message| message.is_err())) + .unwrap(); + }); + + let (gateway, shutdown, server) = managed_gateway(inference, upstream).await; + let stream = TcpStream::connect(gateway).await.unwrap(); + let (mut socket, response) = timeout( + Duration::from_secs(2), + client_async(format!("ws://{gateway}/realtime"), stream), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS); + timeout(Duration::from_secs(2), ready_receiver) + .await + .unwrap() + .unwrap(); + + let started = Instant::now(); + shutdown.send(()).unwrap(); + timeout(Duration::from_secs(6), server) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(started.elapsed() <= Duration::from_secs(6)); + + let result = timeout(Duration::from_secs(2), socket.next()) + .await + .unwrap(); + assert!(result.is_none_or(|message| message.is_err())); + assert!( + timeout(Duration::from_secs(2), eof_receiver) + .await + .unwrap() + .unwrap() + ); + upstream_task.await.unwrap(); +} + +#[rstest] +#[tokio::test] +async fn shutdown_deadline_closes_a_handshake_released_after_shutdown(inference: Arc) { + let upstream_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let upstream = upstream_listener.local_addr().unwrap(); + let (request_sender, request_receiver) = oneshot::channel(); + let (release_sender, release_receiver) = oneshot::channel(); + let (closed_sender, closed_receiver) = oneshot::channel(); + let upstream_task = tokio::spawn(async move { + let (mut stream, _) = upstream_listener.accept().await.unwrap(); + let mut request = Vec::new(); + let mut chunk = [0; 4096]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let read = stream.read(&mut chunk).await.unwrap(); + if read == 0 { + return; + } + request.extend_from_slice(&chunk[..read]); + } + let key = websocket_key(&request); + request_sender.send(()).unwrap(); + release_receiver.await.unwrap(); + let accept = derive_accept_key(key.as_bytes()); + let response = format!( + "HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Accept: {accept}\r\n\r\n" + ); + stream.write_all(response.as_bytes()).await.unwrap(); + let mut bytes = [0; 64]; + let closed = matches!( + timeout(Duration::from_secs(3), stream.read(&mut bytes)).await, + Ok(Ok(0)) + ); + closed_sender.send(closed).unwrap(); + }); + + let config = + Config::from_yaml("model_list: []\ngeneral_settings:\n master_key: gateway-key\n") + .unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let gateway = listener.local_addr().unwrap(); + let fallback = Fallback::Forward(Forward::new(&format!("http://{upstream}")).unwrap()); + let app = litellm_gateway::router(inference, &config, None, None, fallback); + let (shutdown, stopped) = oneshot::channel(); + let (began_sender, began_receiver) = oneshot::channel(); + let server = tokio::spawn(litellm_gateway::serve(listener, app, async move { + let _ = stopped.await; + began_sender.send(()).unwrap(); + })); + + let client = tokio::spawn(async move { + let mut stream = TcpStream::connect(gateway).await.unwrap(); + let request = format!( + "GET /realtime HTTP/1.1\r\nHost: {gateway}\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: 13\r\n\r\n" + ); + stream.write_all(request.as_bytes()).await.unwrap(); + let mut response = Vec::new(); + let mut chunk = [0; 1024]; + while !response.windows(4).any(|window| window == b"\r\n\r\n") { + let read = stream.read(&mut chunk).await.unwrap(); + if read == 0 { + break; + } + response.extend_from_slice(&chunk[..read]); + } + (stream, response) + }); + timeout(Duration::from_secs(2), request_receiver) + .await + .unwrap() + .unwrap(); + shutdown.send(()).unwrap(); + timeout(Duration::from_secs(2), began_receiver) + .await + .unwrap() + .unwrap(); + tokio::time::sleep(Duration::from_secs(4)).await; + release_sender.send(()).unwrap(); + + let (mut socket, response) = timeout(Duration::from_secs(3), client) + .await + .unwrap() + .unwrap(); + assert!(response.starts_with(b"HTTP/1.1 101")); + let result = timeout(Duration::from_secs(3), socket.read(&mut [0; 64])) + .await + .unwrap(); + assert_eq!(result.unwrap(), 0); + timeout(Duration::from_secs(3), server) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!( + timeout(Duration::from_secs(2), closed_receiver) + .await + .unwrap() + .unwrap() + ); + upstream_task.await.unwrap(); +} + +#[rstest] +#[tokio::test] +async fn dropping_serve_cancels_an_open_websocket(inference: Arc) { + let upstream_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let upstream = upstream_listener.local_addr().unwrap(); + let (ready_sender, ready_receiver) = oneshot::channel(); + let (eof_sender, eof_receiver) = oneshot::channel(); + let upstream_task = tokio::spawn(async move { + let (stream, _) = upstream_listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + ready_sender.send(()).unwrap(); + let result = timeout(Duration::from_secs(2), socket.next()) + .await + .unwrap(); + eof_sender + .send(result.is_none_or(|message| message.is_err())) + .unwrap(); + }); + + let (gateway, _shutdown, server) = managed_gateway(inference, upstream).await; + let stream = TcpStream::connect(gateway).await.unwrap(); + let (mut socket, response) = timeout( + Duration::from_secs(2), + client_async(format!("ws://{gateway}/realtime"), stream), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS); + timeout(Duration::from_secs(2), ready_receiver) + .await + .unwrap() + .unwrap(); + + server.abort(); + assert!( + timeout(Duration::from_secs(2), server) + .await + .unwrap() + .unwrap_err() + .is_cancelled() + ); + let result = timeout(Duration::from_secs(2), socket.next()) + .await + .unwrap(); + assert!(result.is_none_or(|message| message.is_err())); + assert!( + timeout(Duration::from_secs(2), eof_receiver) + .await + .unwrap() + .unwrap() + ); + upstream_task.await.unwrap(); +} + +#[rstest] +#[tokio::test] +async fn unmanaged_axum_server_rejects_websocket_forwarding(inference: Arc) { + let upstream = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let upstream_address = upstream.local_addr().unwrap(); + let config = + Config::from_yaml("model_list: []\ngeneral_settings:\n master_key: gateway-key\n") + .unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let gateway = listener.local_addr().unwrap(); + let fallback = Fallback::Forward(Forward::new(&format!("http://{upstream_address}")).unwrap()); + let app = litellm_gateway::router(inference, &config, None, None, fallback); + let (shutdown, stopped) = oneshot::channel(); + let server = tokio::spawn(async move { + axum::serve(listener, app) + .with_graceful_shutdown(async move { + let _ = stopped.await; + }) + .await + }); + + let stream = TcpStream::connect(gateway).await.unwrap(); + let request = axum::http::Request::builder() + .uri(format!("ws://{gateway}/realtime")) + .header("Host", gateway.to_string()) + .header("Connection", "Upgrade") + .header("Upgrade", "websocket") + .header("Sec-WebSocket-Key", "dGhlIHNhbXBsZSBub25jZQ==") + .header("Sec-WebSocket-Version", "13") + .body(()) + .unwrap(); + let error = timeout(Duration::from_secs(2), client_async(request, stream)) + .await + .unwrap() + .unwrap_err(); + let response = match error { + tokio_tungstenite::tungstenite::Error::Http(response) => response, + error => panic!("expected an HTTP rejection, got {error}"), + }; + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_ne!(response.status(), StatusCode::SWITCHING_PROTOCOLS); + let body = String::from_utf8(response.body().clone().unwrap()).unwrap(); + assert!(body.contains("requires litellm_gateway::serve")); + + shutdown.send(()).unwrap(); + timeout(Duration::from_secs(5), server) + .await + .unwrap() + .unwrap() + .unwrap(); +}