mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 62714ea9b4 into 5d489579f7
This commit is contained in:
commit
a80d97d2d6
8 changed files with 901 additions and 16 deletions
3
litellm-rust/Cargo.lock
generated
3
litellm-rust/Cargo.lock
generated
|
|
@ -3765,6 +3765,7 @@ dependencies = [
|
|||
"envy",
|
||||
"futures-util",
|
||||
"http-body-util",
|
||||
"hyper 1.10.1",
|
||||
"hyper-util",
|
||||
"litellm-auth-types",
|
||||
"litellm-config",
|
||||
|
|
@ -3786,6 +3787,7 @@ dependencies = [
|
|||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"tokio-tungstenite",
|
||||
"tokio-util",
|
||||
"tower",
|
||||
"tower-sessions-moka-store",
|
||||
|
|
@ -7444,6 +7446,7 @@ dependencies = [
|
|||
"bytes",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"futures-util",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -10,12 +10,13 @@ axum = { workspace = true, features = ["original-uri"] }
|
|||
bytes.workspace = true
|
||||
envy = "0.4.2"
|
||||
http-body-util = "0.1"
|
||||
hyper = { version = "1", features = ["http1"] }
|
||||
hyper-util = { workspace = true, features = ["client-legacy", "http1", "tokio"] }
|
||||
litellm-core-utils.workspace = true
|
||||
litellm-inference.workspace = true
|
||||
litellm-gateway-inference.workspace = true
|
||||
litellm-gateway-mcp.workspace = true
|
||||
tokio-util = "0.7"
|
||||
tokio-util = { version = "0.7", features = ["rt"] }
|
||||
thiserror.workspace = true
|
||||
futures-util.workspace = true
|
||||
litellm-gateway-auth.workspace = true
|
||||
|
|
@ -42,4 +43,5 @@ base64.workspace = true
|
|||
rstest.workspace = true
|
||||
tempfile.workspace = true
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
tokio-tungstenite.workspace = true
|
||||
tower = { version = "0.5", features = ["util"] }
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ use axum::{
|
|||
body::Body,
|
||||
extract::{ConnectInfo, OriginalUri, Request},
|
||||
http::{
|
||||
HeaderMap, HeaderName, HeaderValue, StatusCode, Uri, Version,
|
||||
HeaderMap, HeaderName, HeaderValue, Method, StatusCode, Uri, Version,
|
||||
header::{
|
||||
CONNECTION, PROXY_AUTHENTICATE, PROXY_AUTHORIZATION, TE, TRAILER, TRANSFER_ENCODING,
|
||||
UPGRADE,
|
||||
|
|
@ -19,6 +19,8 @@ use hyper_util::{
|
|||
};
|
||||
use litellm_core_utils::decline::Decline;
|
||||
|
||||
use crate::upgrades::{HANDSHAKE_TIMEOUT, Upgrades};
|
||||
|
||||
/// Python reads it to log why the gateway handed the request over.
|
||||
const DECLINED_HEADER: HeaderName = HeaderName::from_static("x-litellm-gateway-declined");
|
||||
|
||||
|
|
@ -73,7 +75,36 @@ impl Forward {
|
|||
})
|
||||
}
|
||||
|
||||
pub(super) async fn send(&self, request: Request, decline: Decline) -> Response {
|
||||
pub(super) async fn send(&self, mut request: Request, decline: Decline) -> Response {
|
||||
let websocket = if request.method() == Method::GET
|
||||
&& request.version() == Version::HTTP_11
|
||||
&& is_websocket(request.headers())
|
||||
{
|
||||
let Some(lifecycle) = request.extensions().get::<Upgrades>().cloned() else {
|
||||
return (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"WebSocket forwarding requires litellm_gateway::serve",
|
||||
)
|
||||
.into_response();
|
||||
};
|
||||
if lifecycle.is_draining() {
|
||||
return StatusCode::SERVICE_UNAVAILABLE.into_response();
|
||||
}
|
||||
if request
|
||||
.extensions()
|
||||
.get::<hyper::upgrade::OnUpgrade>()
|
||||
.is_none()
|
||||
{
|
||||
return (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"WebSocket forwarding requires an HTTP upgrade transport",
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
Some((hyper::upgrade::on(&mut request), lifecycle))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let (mut parts, body) = request.into_parts();
|
||||
let path = parts
|
||||
.extensions
|
||||
|
|
@ -92,6 +123,9 @@ impl Forward {
|
|||
.expect("scheme, authority and path make an absolute uri");
|
||||
parts.version = Version::HTTP_11;
|
||||
strip_hop_by_hop(&mut parts.headers);
|
||||
if websocket.is_some() {
|
||||
websocket_headers(&mut parts.headers);
|
||||
}
|
||||
if let Some(ConnectInfo(client)) = parts.extensions.get::<ConnectInfo<SocketAddr>>() {
|
||||
append_forwarded_for(&mut parts.headers, *client);
|
||||
}
|
||||
|
|
@ -103,8 +137,46 @@ impl Forward {
|
|||
path = parts.uri.path(),
|
||||
"forwarded to python"
|
||||
);
|
||||
match self.client.request(Request::from_parts(parts, body)).await {
|
||||
Ok(response) => {
|
||||
let response = self.client.request(Request::from_parts(parts, body));
|
||||
let response = if let Some((_, lifecycle)) = &websocket {
|
||||
tokio::select! {
|
||||
_ = lifecycle.cancelled() => return StatusCode::SERVICE_UNAVAILABLE.into_response(),
|
||||
result = tokio::time::timeout(HANDSHAKE_TIMEOUT, response) => match result {
|
||||
Ok(result) => result,
|
||||
Err(_) => return StatusCode::GATEWAY_TIMEOUT.into_response(),
|
||||
}
|
||||
}
|
||||
} else {
|
||||
response.await
|
||||
};
|
||||
match response {
|
||||
Ok(mut response) => {
|
||||
if response.status() == StatusCode::SWITCHING_PROTOCOLS {
|
||||
let Some((downstream, lifecycle)) = websocket else {
|
||||
return StatusCode::BAD_GATEWAY.into_response();
|
||||
};
|
||||
if !is_websocket(response.headers()) {
|
||||
return StatusCode::BAD_GATEWAY.into_response();
|
||||
}
|
||||
let upstream = hyper::upgrade::on(&mut response);
|
||||
let upstream = tokio::select! {
|
||||
_ = lifecycle.cancelled() => {
|
||||
return StatusCode::SERVICE_UNAVAILABLE.into_response();
|
||||
}
|
||||
result = tokio::time::timeout(HANDSHAKE_TIMEOUT, upstream) => {
|
||||
match result {
|
||||
Ok(Ok(upstream)) => upstream,
|
||||
Ok(Err(_)) => return StatusCode::BAD_GATEWAY.into_response(),
|
||||
Err(_) => return StatusCode::GATEWAY_TIMEOUT.into_response(),
|
||||
}
|
||||
}
|
||||
};
|
||||
let (mut parts, _) = response.into_parts();
|
||||
strip_hop_by_hop(&mut parts.headers);
|
||||
websocket_headers(&mut parts.headers);
|
||||
lifecycle.relay(downstream, upstream);
|
||||
return Response::from_parts(parts, Body::empty());
|
||||
}
|
||||
let (mut parts, body) = response.into_parts();
|
||||
strip_hop_by_hop(&mut parts.headers);
|
||||
Response::from_parts(parts, Body::new(body))
|
||||
|
|
@ -117,6 +189,24 @@ impl Forward {
|
|||
}
|
||||
}
|
||||
|
||||
fn is_websocket(headers: &HeaderMap) -> bool {
|
||||
headers
|
||||
.get(UPGRADE)
|
||||
.is_some_and(|value| value.as_bytes().eq_ignore_ascii_case(b"websocket"))
|
||||
&& headers.get_all(CONNECTION).iter().any(|value| {
|
||||
value.to_str().is_ok_and(|value| {
|
||||
value
|
||||
.split(',')
|
||||
.any(|token| token.trim().eq_ignore_ascii_case("upgrade"))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn websocket_headers(headers: &mut HeaderMap) {
|
||||
headers.insert(CONNECTION, HeaderValue::from_static("upgrade"));
|
||||
headers.insert(UPGRADE, HeaderValue::from_static("websocket"));
|
||||
}
|
||||
|
||||
/// Python sees the gateway as its peer, so the client's address goes on the end of the chain
|
||||
/// any proxy in front of the gateway already started.
|
||||
fn append_forwarded_for(headers: &mut HeaderMap, client: SocketAddr) {
|
||||
|
|
|
|||
|
|
@ -74,5 +74,11 @@ fn record(parts: &Parts, body: Bytes) -> Request<Bytes> {
|
|||
if let Some(&client) = parts.extensions.get::<ConnectInfo<SocketAddr>>() {
|
||||
recorded.extensions_mut().insert(client);
|
||||
}
|
||||
if let Some(upgrade) = parts.extensions.get::<hyper::upgrade::OnUpgrade>() {
|
||||
recorded.extensions_mut().insert(upgrade.clone());
|
||||
}
|
||||
if let Some(lifecycle) = parts.extensions.get::<crate::upgrades::Upgrades>() {
|
||||
recorded.extensions_mut().insert(lifecycle.clone());
|
||||
}
|
||||
recorded
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ mod auth;
|
|||
mod dispatch;
|
||||
mod error;
|
||||
mod secrets;
|
||||
mod upgrades;
|
||||
pub use dispatch::{Fallback, Forward, InvalidUpstream};
|
||||
pub use error::Error;
|
||||
|
||||
|
|
@ -141,12 +142,25 @@ pub async fn serve(
|
|||
app: Router,
|
||||
shutdown: impl Future<Output = ()> + Send + 'static,
|
||||
) -> std::io::Result<()> {
|
||||
axum::serve(
|
||||
let upgrades = upgrades::Upgrades::new();
|
||||
let _cancel_upgrades = upgrades.cancel.clone().drop_guard();
|
||||
let draining = upgrades.clone();
|
||||
let result = axum::serve(
|
||||
listener,
|
||||
app.into_make_service_with_connect_info::<std::net::SocketAddr>(),
|
||||
app.layer(axum::Extension(upgrades.clone()))
|
||||
.into_make_service_with_connect_info::<std::net::SocketAddr>(),
|
||||
)
|
||||
.with_graceful_shutdown(shutdown)
|
||||
.await
|
||||
.with_graceful_shutdown(async move {
|
||||
shutdown.await;
|
||||
draining.begin_shutdown();
|
||||
})
|
||||
.await;
|
||||
if result.is_err() {
|
||||
upgrades.cancel.cancel();
|
||||
}
|
||||
upgrades.tasks.close();
|
||||
upgrades.tasks.wait().await;
|
||||
result
|
||||
}
|
||||
|
||||
async fn log_request(request: Request, next: Next) -> Response {
|
||||
|
|
|
|||
71
litellm-rust/crates/gateway/src/upgrades.rs
Normal file
71
litellm-rust/crates/gateway/src/upgrades.rs
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use hyper::upgrade::{OnUpgrade, Upgraded};
|
||||
use hyper_util::rt::TokioIo;
|
||||
use tokio::{sync::watch, time::Instant};
|
||||
use tokio_util::{sync::CancellationToken, task::TaskTracker};
|
||||
|
||||
pub(crate) const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const DRAIN_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct Upgrades {
|
||||
pub tasks: TaskTracker,
|
||||
pub cancel: CancellationToken,
|
||||
deadline: watch::Sender<Option<Instant>>,
|
||||
}
|
||||
|
||||
impl Upgrades {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
tasks: TaskTracker::new(),
|
||||
cancel: CancellationToken::new(),
|
||||
deadline: watch::Sender::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn begin_shutdown(&self) {
|
||||
self.deadline
|
||||
.send_replace(Some(Instant::now() + DRAIN_TIMEOUT));
|
||||
}
|
||||
|
||||
pub fn is_draining(&self) -> bool {
|
||||
self.deadline.borrow().is_some() || self.cancel.is_cancelled()
|
||||
}
|
||||
|
||||
pub async fn cancelled(&self) {
|
||||
let mut deadline = self.deadline.subscribe();
|
||||
tokio::select! {
|
||||
_ = self.cancel.cancelled() => {}
|
||||
_ = async {
|
||||
let at = *deadline.wait_for(Option::is_some).await.expect("sender is owned by self");
|
||||
tokio::time::sleep_until(at.expect("waited for shutdown deadline")).await;
|
||||
} => {}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn relay(&self, downstream: OnUpgrade, upstream: Upgraded) {
|
||||
let lifecycle = self.clone();
|
||||
self.tasks.spawn(async move {
|
||||
tokio::select! {
|
||||
_ = lifecycle.cancelled() => {}
|
||||
_ = async {
|
||||
let upgraded = tokio::time::timeout(HANDSHAKE_TIMEOUT, downstream).await;
|
||||
let Ok(Ok(downstream)) = upgraded else {
|
||||
tracing::debug!("websocket upgrade did not complete");
|
||||
return;
|
||||
};
|
||||
let (mut downstream_read, mut downstream_write) = tokio::io::split(TokioIo::new(downstream));
|
||||
let (mut upstream_read, mut upstream_write) = tokio::io::split(TokioIo::new(upstream));
|
||||
let result = tokio::select! {
|
||||
result = tokio::io::copy(&mut downstream_read, &mut upstream_write) => result,
|
||||
result = tokio::io::copy(&mut upstream_read, &mut downstream_write) => result,
|
||||
};
|
||||
if result.is_err() {
|
||||
tracing::debug!("websocket relay disconnected");
|
||||
}
|
||||
} => {}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -9,20 +9,33 @@ use axum::{
|
|||
Router,
|
||||
body::{Body, Bytes, to_bytes},
|
||||
extract::{ConnectInfo, DefaultBodyLimit},
|
||||
http::{HeaderMap, Request, StatusCode},
|
||||
http::{
|
||||
HeaderMap, HeaderValue, Request, StatusCode,
|
||||
header::{
|
||||
AUTHORIZATION, CONNECTION, HOST, ORIGIN, SEC_WEBSOCKET_KEY, SEC_WEBSOCKET_PROTOCOL,
|
||||
SEC_WEBSOCKET_VERSION, UPGRADE,
|
||||
},
|
||||
},
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use futures_util::{StreamExt, stream};
|
||||
use futures_util::{SinkExt, StreamExt, stream};
|
||||
use litellm_config::Config;
|
||||
use litellm_gateway::{Fallback, Forward};
|
||||
use litellm_gateway_inference::MAX_BODY_BYTES;
|
||||
use rstest::rstest;
|
||||
use rstest::{fixture, rstest};
|
||||
use serde_json::{Value, json};
|
||||
use tokio::{
|
||||
net::TcpListener,
|
||||
net::{TcpListener, TcpStream},
|
||||
sync::{mpsc, oneshot},
|
||||
time::timeout,
|
||||
};
|
||||
use tokio_tungstenite::{
|
||||
accept_async, accept_hdr_async, client_async,
|
||||
tungstenite::{
|
||||
Message,
|
||||
handshake::server::{Request as WebSocketRequest, Response as WebSocketResponse},
|
||||
},
|
||||
};
|
||||
use tower::ServiceExt;
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any};
|
||||
|
||||
|
|
@ -212,6 +225,383 @@ async fn gateway_in_front_of(python: SocketAddr) -> SocketAddr {
|
|||
.await
|
||||
}
|
||||
|
||||
#[fixture]
|
||||
async fn websocket_python() -> SocketAddr {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let Ok(mut socket) = accept_async(stream).await else {
|
||||
return;
|
||||
};
|
||||
socket.send(Message::text("from python")).await.unwrap();
|
||||
if let Some(Ok(message)) = socket.next().await {
|
||||
socket.send(message).await.unwrap();
|
||||
socket.close(None).await.unwrap();
|
||||
}
|
||||
});
|
||||
address
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::direct_text(false, Message::text("hello"))]
|
||||
#[case::direct_binary(false, Message::binary(vec![0, 1, 255]))]
|
||||
#[case::forwarded_text(true, Message::text("hello"))]
|
||||
#[case::forwarded_binary(true, Message::binary(vec![0, 1, 255]))]
|
||||
#[tokio::test]
|
||||
async fn python_serves_websocket_messages(
|
||||
#[future(awt)] websocket_python: SocketAddr,
|
||||
#[case] forwarded: bool,
|
||||
#[case] message: Message,
|
||||
) {
|
||||
timeout(Duration::from_secs(5), async {
|
||||
let address = if forwarded {
|
||||
gateway_in_front_of(websocket_python).await
|
||||
} else {
|
||||
websocket_python
|
||||
};
|
||||
let stream = TcpStream::connect(address).await.unwrap();
|
||||
let (mut socket, response) = client_async(
|
||||
format!("ws://{address}/realtime?model=public/model"),
|
||||
stream,
|
||||
)
|
||||
.await
|
||||
.expect("the client should receive a successful WebSocket upgrade");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
|
||||
assert_eq!(
|
||||
socket.next().await.unwrap().unwrap(),
|
||||
Message::text("from python")
|
||||
);
|
||||
socket.send(message.clone()).await.unwrap();
|
||||
assert_eq!(socket.next().await.unwrap().unwrap(), message);
|
||||
assert_eq!(socket.next().await.unwrap().unwrap(), Message::Close(None));
|
||||
})
|
||||
.await
|
||||
.expect("the WebSocket exchange should finish without hanging");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
// Tungstenite's handshake callback uses a full HTTP response as its error type.
|
||||
#[allow(clippy::result_large_err)]
|
||||
async fn forwarded_websocket_preserves_handshake_headers_and_control_frames() {
|
||||
timeout(Duration::from_secs(5), async {
|
||||
const WEBSOCKET_EXTENSION: &str =
|
||||
"permessage-deflate; client_no_context_takeover; server_no_context_takeover";
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let python = listener.local_addr().unwrap();
|
||||
let (seen_sender, seen_receiver) = oneshot::channel();
|
||||
let upstream = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut socket = accept_hdr_async(
|
||||
stream,
|
||||
move |request: &WebSocketRequest, mut response: WebSocketResponse| {
|
||||
seen_sender
|
||||
.send((request.uri().to_string(), request.headers().clone()))
|
||||
.unwrap();
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(SEC_WEBSOCKET_PROTOCOL, HeaderValue::from_static("chat"));
|
||||
response.headers_mut().insert(
|
||||
"sec-websocket-extensions",
|
||||
HeaderValue::from_static(WEBSOCKET_EXTENSION),
|
||||
);
|
||||
Ok(response)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
Message::Ping(vec![7, 8])
|
||||
);
|
||||
socket.send(Message::Pong(vec![7, 8])).await.unwrap();
|
||||
socket.send(Message::text("from python")).await.unwrap();
|
||||
assert_eq!(
|
||||
timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
Message::text("to python")
|
||||
);
|
||||
socket.send(Message::text("echo")).await.unwrap();
|
||||
});
|
||||
|
||||
let gateway = gateway_in_front_of(python).await;
|
||||
let stream = TcpStream::connect(gateway).await.unwrap();
|
||||
let request = Request::builder()
|
||||
.uri(format!(
|
||||
"ws://{gateway}/realtime?model=public/model&trace=1"
|
||||
))
|
||||
.header(HOST, gateway.to_string())
|
||||
.header(AUTHORIZATION, "Bearer gateway-key")
|
||||
.header(ORIGIN, "https://client.example")
|
||||
.header(SEC_WEBSOCKET_PROTOCOL, "chat, superchat")
|
||||
.header("sec-websocket-extensions", WEBSOCKET_EXTENSION)
|
||||
.header(SEC_WEBSOCKET_KEY, "dGhlIHNhbXBsZSBub25jZQ==")
|
||||
.header(SEC_WEBSOCKET_VERSION, "13")
|
||||
.header(UPGRADE, "websocket")
|
||||
.header(CONNECTION, "Upgrade, x-hop")
|
||||
.header("x-hop", "remove")
|
||||
.body(())
|
||||
.unwrap();
|
||||
let (mut socket, response) = client_async(request, stream).await.unwrap();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
|
||||
assert_eq!(
|
||||
response.headers().get(SEC_WEBSOCKET_PROTOCOL).unwrap(),
|
||||
"chat"
|
||||
);
|
||||
assert_eq!(
|
||||
response.headers().get("sec-websocket-extensions").unwrap(),
|
||||
WEBSOCKET_EXTENSION
|
||||
);
|
||||
let (uri, headers) = timeout(Duration::from_secs(2), seen_receiver)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(uri, "/realtime?model=public/model&trace=1");
|
||||
assert_eq!(headers.get(AUTHORIZATION).unwrap(), "Bearer gateway-key");
|
||||
assert_eq!(headers.get(ORIGIN).unwrap(), "https://client.example");
|
||||
assert_eq!(
|
||||
headers.get(SEC_WEBSOCKET_PROTOCOL).unwrap(),
|
||||
"chat, superchat"
|
||||
);
|
||||
assert_eq!(
|
||||
headers.get("sec-websocket-extensions").unwrap(),
|
||||
WEBSOCKET_EXTENSION
|
||||
);
|
||||
assert_eq!(headers.get(CONNECTION).unwrap(), "upgrade");
|
||||
assert!(!headers.contains_key("x-hop"));
|
||||
|
||||
socket.send(Message::Ping(vec![7, 8])).await.unwrap();
|
||||
assert_eq!(
|
||||
timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
Message::Pong(vec![7, 8])
|
||||
);
|
||||
assert_eq!(
|
||||
timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
Message::text("from python")
|
||||
);
|
||||
socket.send(Message::text("to python")).await.unwrap();
|
||||
assert_eq!(
|
||||
timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
Message::text("echo")
|
||||
);
|
||||
drop(socket);
|
||||
upstream.await.unwrap();
|
||||
})
|
||||
.await
|
||||
.expect("the forwarded WebSocket should preserve its negotiated behavior");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn forwarded_websocket_rejection_preserves_status_body_challenge_and_filters_hops() {
|
||||
let python = MockServer::start().await;
|
||||
Mock::given(any())
|
||||
.respond_with(
|
||||
ResponseTemplate::new(401)
|
||||
.insert_header("www-authenticate", "Bearer realm=upstream")
|
||||
.insert_header("connection", "x-upstream-hop")
|
||||
.insert_header("x-upstream-hop", "remove")
|
||||
.set_body_string("upstream denied"),
|
||||
)
|
||||
.mount(&python)
|
||||
.await;
|
||||
let gateway = gateway_in_front_of(*python.address()).await;
|
||||
let stream = TcpStream::connect(gateway).await.unwrap();
|
||||
let request = Request::builder()
|
||||
.uri(format!("ws://{gateway}/realtime?model=public/model"))
|
||||
.header(HOST, gateway.to_string())
|
||||
.header(AUTHORIZATION, "Bearer gateway-key")
|
||||
.header(CONNECTION, "Upgrade")
|
||||
.header(SEC_WEBSOCKET_KEY, "dGhlIHNhbXBsZSBub25jZQ==")
|
||||
.header(SEC_WEBSOCKET_VERSION, "13")
|
||||
.header(UPGRADE, "websocket")
|
||||
.body(())
|
||||
.unwrap();
|
||||
|
||||
let error = timeout(Duration::from_secs(5), client_async(request, stream))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap_err();
|
||||
let response = match error {
|
||||
tokio_tungstenite::tungstenite::Error::Http(response) => response,
|
||||
error => panic!("expected an HTTP rejection, got {error}"),
|
||||
};
|
||||
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||
assert_eq!(
|
||||
response.headers().get("www-authenticate").unwrap(),
|
||||
"Bearer realm=upstream"
|
||||
);
|
||||
assert!(!response.headers().contains_key("x-upstream-hop"));
|
||||
assert_eq!(
|
||||
response.body().as_deref(),
|
||||
Some(b"upstream denied".as_slice())
|
||||
);
|
||||
|
||||
let requests = python.received_requests().await.unwrap();
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert_eq!(
|
||||
requests[0].headers.get("authorization").unwrap(),
|
||||
"Bearer gateway-key"
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::missing_connection("websocket", "keep-alive")]
|
||||
#[case::non_websocket_upgrade("h2c", "upgrade")]
|
||||
#[tokio::test]
|
||||
async fn malformed_upgrade_is_forwarded_as_http(#[case] upgrade: &str, #[case] connection: &str) {
|
||||
let python = python().await;
|
||||
let response = app(&python.uri(), "anthropic/test-model", "http://127.0.0.1:1")
|
||||
.oneshot(
|
||||
Request::get("/realtime?model=public/model")
|
||||
.header("upgrade", upgrade)
|
||||
.header("connection", connection)
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
to_bytes(response.into_body(), 1024).await.unwrap(),
|
||||
"from python"
|
||||
);
|
||||
let requests = python.received_requests().await.unwrap();
|
||||
assert_eq!(requests.len(), 1);
|
||||
assert!(!requests[0].headers.contains_key("upgrade"));
|
||||
assert!(!requests[0].headers.contains_key("connection"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn downstream_disconnect_reaches_upstream_as_eof() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let python = listener.local_addr().unwrap();
|
||||
let (ready_sender, ready_receiver) = oneshot::channel();
|
||||
let (eof_sender, eof_receiver) = oneshot::channel();
|
||||
let upstream = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut socket = accept_async(stream).await.unwrap();
|
||||
ready_sender.send(()).unwrap();
|
||||
let result = timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.unwrap();
|
||||
eof_sender
|
||||
.send(result.is_none_or(|message| message.is_err()))
|
||||
.unwrap();
|
||||
});
|
||||
|
||||
let gateway = gateway_in_front_of(python).await;
|
||||
let stream = TcpStream::connect(gateway).await.unwrap();
|
||||
let (socket, response) = timeout(
|
||||
Duration::from_secs(2),
|
||||
client_async(format!("ws://{gateway}/realtime"), stream),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
|
||||
timeout(Duration::from_secs(2), ready_receiver)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
drop(socket);
|
||||
|
||||
assert!(
|
||||
timeout(Duration::from_secs(2), eof_receiver)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
);
|
||||
upstream.await.unwrap();
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn upstream_disconnect_reaches_downstream_as_eof() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let python = listener.local_addr().unwrap();
|
||||
let upstream = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let socket = accept_async(stream).await.unwrap();
|
||||
drop(socket);
|
||||
});
|
||||
|
||||
let gateway = gateway_in_front_of(python).await;
|
||||
let stream = TcpStream::connect(gateway).await.unwrap();
|
||||
let (mut socket, response) = timeout(
|
||||
Duration::from_secs(2),
|
||||
client_async(format!("ws://{gateway}/realtime"), stream),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
|
||||
let result = timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(result.is_none_or(|message| message.is_err()));
|
||||
upstream.await.unwrap();
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn established_websocket_survives_the_handshake_timeout() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let python = listener.local_addr().unwrap();
|
||||
let upstream = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut socket = accept_async(stream).await.unwrap();
|
||||
tokio::time::sleep(Duration::from_secs(11)).await;
|
||||
socket.send(Message::text("still alive")).await.unwrap();
|
||||
});
|
||||
|
||||
let gateway = gateway_in_front_of(python).await;
|
||||
let stream = TcpStream::connect(gateway).await.unwrap();
|
||||
let (mut socket, response) = timeout(
|
||||
Duration::from_secs(5),
|
||||
client_async(format!("ws://{gateway}/realtime"), stream),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
|
||||
assert_eq!(
|
||||
timeout(Duration::from_secs(12), socket.next())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
Message::text("still alive")
|
||||
);
|
||||
upstream.await.unwrap();
|
||||
}
|
||||
|
||||
async fn recording_python(
|
||||
respond: fn() -> Response,
|
||||
) -> (SocketAddr, mpsc::UnboundedReceiver<(HeaderMap, Bytes)>) {
|
||||
|
|
|
|||
|
|
@ -1,10 +1,15 @@
|
|||
use std::{
|
||||
net::SocketAddr,
|
||||
sync::{Arc, mpsc},
|
||||
time::Duration,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use axum::{body::Body, http::Request};
|
||||
use axum::{
|
||||
body::Body,
|
||||
http::{Request, StatusCode},
|
||||
};
|
||||
use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use futures_util::StreamExt;
|
||||
use litellm_config::Config;
|
||||
use litellm_gateway::{Fallback, Forward};
|
||||
use litellm_gateway_inference::{Error, Gateway};
|
||||
|
|
@ -12,7 +17,14 @@ use litellm_http::ClientVariant;
|
|||
use litellm_tracing::{Logger, Metadata, Record, Sink};
|
||||
use rstest::{fixture, rstest};
|
||||
use serde_json::{Value, json};
|
||||
use tokio::{net::TcpListener, sync::oneshot, time::timeout};
|
||||
use tokio::{
|
||||
io::{AsyncReadExt, AsyncWriteExt},
|
||||
net::{TcpListener, TcpStream},
|
||||
sync::oneshot,
|
||||
task::JoinHandle,
|
||||
time::timeout,
|
||||
};
|
||||
use tokio_tungstenite::{accept_async, client_async, tungstenite::handshake::derive_accept_key};
|
||||
use tower::ServiceExt;
|
||||
|
||||
struct LogSink(mpsc::Sender<Value>);
|
||||
|
|
@ -34,6 +46,39 @@ fn inference() -> Arc<Gateway> {
|
|||
litellm_gateway::build_inference(&Config::from_yaml("model_list: []").unwrap()).unwrap()
|
||||
}
|
||||
|
||||
async fn managed_gateway(
|
||||
inference: Arc<Gateway>,
|
||||
upstream: SocketAddr,
|
||||
) -> (
|
||||
SocketAddr,
|
||||
oneshot::Sender<()>,
|
||||
JoinHandle<std::io::Result<()>>,
|
||||
) {
|
||||
let config =
|
||||
Config::from_yaml("model_list: []\ngeneral_settings:\n master_key: gateway-key\n")
|
||||
.unwrap();
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let fallback = Fallback::Forward(Forward::new(&format!("http://{upstream}")).unwrap());
|
||||
let app = litellm_gateway::router(inference, &config, None, None, fallback);
|
||||
let (shutdown, stopped) = oneshot::channel();
|
||||
let server = tokio::spawn(litellm_gateway::serve(listener, app, async move {
|
||||
let _ = stopped.await;
|
||||
}));
|
||||
(address, shutdown, server)
|
||||
}
|
||||
|
||||
fn websocket_key(request: &[u8]) -> String {
|
||||
String::from_utf8_lossy(request)
|
||||
.lines()
|
||||
.find_map(|line| {
|
||||
let (name, value) = line.split_once(':')?;
|
||||
name.eq_ignore_ascii_case("sec-websocket-key")
|
||||
.then(|| value.trim().to_owned())
|
||||
})
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::authorized("/v1/messages", Some("Bearer gateway-key"), Some("gateway-key"), 400)]
|
||||
#[case::missing_token("/v1/messages", None, Some("gateway-key"), 401)]
|
||||
|
|
@ -353,3 +398,267 @@ async fn python_serves_every_request_the_gateway_does_not_route(
|
|||
assert_eq!(response.status().as_u16(), 401);
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn serve_drains_an_open_websocket_and_closes_both_peers(inference: Arc<Gateway>) {
|
||||
let upstream_listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let upstream = upstream_listener.local_addr().unwrap();
|
||||
let (ready_sender, ready_receiver) = oneshot::channel();
|
||||
let (eof_sender, eof_receiver) = oneshot::channel();
|
||||
let upstream_task = tokio::spawn(async move {
|
||||
let (stream, _) = upstream_listener.accept().await.unwrap();
|
||||
let mut socket = accept_async(stream).await.unwrap();
|
||||
ready_sender.send(()).unwrap();
|
||||
let result = timeout(Duration::from_secs(7), socket.next())
|
||||
.await
|
||||
.unwrap();
|
||||
eof_sender
|
||||
.send(result.is_none_or(|message| message.is_err()))
|
||||
.unwrap();
|
||||
});
|
||||
|
||||
let (gateway, shutdown, server) = managed_gateway(inference, upstream).await;
|
||||
let stream = TcpStream::connect(gateway).await.unwrap();
|
||||
let (mut socket, response) = timeout(
|
||||
Duration::from_secs(2),
|
||||
client_async(format!("ws://{gateway}/realtime"), stream),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
|
||||
timeout(Duration::from_secs(2), ready_receiver)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
let started = Instant::now();
|
||||
shutdown.send(()).unwrap();
|
||||
timeout(Duration::from_secs(6), server)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(started.elapsed() <= Duration::from_secs(6));
|
||||
|
||||
let result = timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(result.is_none_or(|message| message.is_err()));
|
||||
assert!(
|
||||
timeout(Duration::from_secs(2), eof_receiver)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
);
|
||||
upstream_task.await.unwrap();
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn shutdown_deadline_closes_a_handshake_released_after_shutdown(inference: Arc<Gateway>) {
|
||||
let upstream_listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let upstream = upstream_listener.local_addr().unwrap();
|
||||
let (request_sender, request_receiver) = oneshot::channel();
|
||||
let (release_sender, release_receiver) = oneshot::channel();
|
||||
let (closed_sender, closed_receiver) = oneshot::channel();
|
||||
let upstream_task = tokio::spawn(async move {
|
||||
let (mut stream, _) = upstream_listener.accept().await.unwrap();
|
||||
let mut request = Vec::new();
|
||||
let mut chunk = [0; 4096];
|
||||
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
let read = stream.read(&mut chunk).await.unwrap();
|
||||
if read == 0 {
|
||||
return;
|
||||
}
|
||||
request.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
let key = websocket_key(&request);
|
||||
request_sender.send(()).unwrap();
|
||||
release_receiver.await.unwrap();
|
||||
let accept = derive_accept_key(key.as_bytes());
|
||||
let response = format!(
|
||||
"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Accept: {accept}\r\n\r\n"
|
||||
);
|
||||
stream.write_all(response.as_bytes()).await.unwrap();
|
||||
let mut bytes = [0; 64];
|
||||
let closed = matches!(
|
||||
timeout(Duration::from_secs(3), stream.read(&mut bytes)).await,
|
||||
Ok(Ok(0))
|
||||
);
|
||||
closed_sender.send(closed).unwrap();
|
||||
});
|
||||
|
||||
let config =
|
||||
Config::from_yaml("model_list: []\ngeneral_settings:\n master_key: gateway-key\n")
|
||||
.unwrap();
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let gateway = listener.local_addr().unwrap();
|
||||
let fallback = Fallback::Forward(Forward::new(&format!("http://{upstream}")).unwrap());
|
||||
let app = litellm_gateway::router(inference, &config, None, None, fallback);
|
||||
let (shutdown, stopped) = oneshot::channel();
|
||||
let (began_sender, began_receiver) = oneshot::channel();
|
||||
let server = tokio::spawn(litellm_gateway::serve(listener, app, async move {
|
||||
let _ = stopped.await;
|
||||
began_sender.send(()).unwrap();
|
||||
}));
|
||||
|
||||
let client = tokio::spawn(async move {
|
||||
let mut stream = TcpStream::connect(gateway).await.unwrap();
|
||||
let request = format!(
|
||||
"GET /realtime HTTP/1.1\r\nHost: {gateway}\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: 13\r\n\r\n"
|
||||
);
|
||||
stream.write_all(request.as_bytes()).await.unwrap();
|
||||
let mut response = Vec::new();
|
||||
let mut chunk = [0; 1024];
|
||||
while !response.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
let read = stream.read(&mut chunk).await.unwrap();
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
response.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
(stream, response)
|
||||
});
|
||||
timeout(Duration::from_secs(2), request_receiver)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
shutdown.send(()).unwrap();
|
||||
timeout(Duration::from_secs(2), began_receiver)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
tokio::time::sleep(Duration::from_secs(4)).await;
|
||||
release_sender.send(()).unwrap();
|
||||
|
||||
let (mut socket, response) = timeout(Duration::from_secs(3), client)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(response.starts_with(b"HTTP/1.1 101"));
|
||||
let result = timeout(Duration::from_secs(3), socket.read(&mut [0; 64]))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(result.unwrap(), 0);
|
||||
timeout(Duration::from_secs(3), server)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(
|
||||
timeout(Duration::from_secs(2), closed_receiver)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
);
|
||||
upstream_task.await.unwrap();
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn dropping_serve_cancels_an_open_websocket(inference: Arc<Gateway>) {
|
||||
let upstream_listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let upstream = upstream_listener.local_addr().unwrap();
|
||||
let (ready_sender, ready_receiver) = oneshot::channel();
|
||||
let (eof_sender, eof_receiver) = oneshot::channel();
|
||||
let upstream_task = tokio::spawn(async move {
|
||||
let (stream, _) = upstream_listener.accept().await.unwrap();
|
||||
let mut socket = accept_async(stream).await.unwrap();
|
||||
ready_sender.send(()).unwrap();
|
||||
let result = timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.unwrap();
|
||||
eof_sender
|
||||
.send(result.is_none_or(|message| message.is_err()))
|
||||
.unwrap();
|
||||
});
|
||||
|
||||
let (gateway, _shutdown, server) = managed_gateway(inference, upstream).await;
|
||||
let stream = TcpStream::connect(gateway).await.unwrap();
|
||||
let (mut socket, response) = timeout(
|
||||
Duration::from_secs(2),
|
||||
client_async(format!("ws://{gateway}/realtime"), stream),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
|
||||
timeout(Duration::from_secs(2), ready_receiver)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
server.abort();
|
||||
assert!(
|
||||
timeout(Duration::from_secs(2), server)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap_err()
|
||||
.is_cancelled()
|
||||
);
|
||||
let result = timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(result.is_none_or(|message| message.is_err()));
|
||||
assert!(
|
||||
timeout(Duration::from_secs(2), eof_receiver)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
);
|
||||
upstream_task.await.unwrap();
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn unmanaged_axum_server_rejects_websocket_forwarding(inference: Arc<Gateway>) {
|
||||
let upstream = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let upstream_address = upstream.local_addr().unwrap();
|
||||
let config =
|
||||
Config::from_yaml("model_list: []\ngeneral_settings:\n master_key: gateway-key\n")
|
||||
.unwrap();
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let gateway = listener.local_addr().unwrap();
|
||||
let fallback = Fallback::Forward(Forward::new(&format!("http://{upstream_address}")).unwrap());
|
||||
let app = litellm_gateway::router(inference, &config, None, None, fallback);
|
||||
let (shutdown, stopped) = oneshot::channel();
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app)
|
||||
.with_graceful_shutdown(async move {
|
||||
let _ = stopped.await;
|
||||
})
|
||||
.await
|
||||
});
|
||||
|
||||
let stream = TcpStream::connect(gateway).await.unwrap();
|
||||
let request = axum::http::Request::builder()
|
||||
.uri(format!("ws://{gateway}/realtime"))
|
||||
.header("Host", gateway.to_string())
|
||||
.header("Connection", "Upgrade")
|
||||
.header("Upgrade", "websocket")
|
||||
.header("Sec-WebSocket-Key", "dGhlIHNhbXBsZSBub25jZQ==")
|
||||
.header("Sec-WebSocket-Version", "13")
|
||||
.body(())
|
||||
.unwrap();
|
||||
let error = timeout(Duration::from_secs(2), client_async(request, stream))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap_err();
|
||||
let response = match error {
|
||||
tokio_tungstenite::tungstenite::Error::Http(response) => response,
|
||||
error => panic!("expected an HTTP rejection, got {error}"),
|
||||
};
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_ne!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
|
||||
let body = String::from_utf8(response.body().clone().unwrap()).unwrap();
|
||||
assert!(body.contains("requires litellm_gateway::serve"));
|
||||
|
||||
shutdown.send(()).unwrap();
|
||||
timeout(Duration::from_secs(5), server)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue