mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
test(gateway): organize forwarding tests by traffic shape
Split http.rs into exchange, stream, upload and client_identity, rename connection.rs to lifecycle, and add slow_head, idle_stream and download. New cases cover minutes-long waits for headers and idle MCP streams on paused time, binary framed streams, 1xx, redirects with cookies and range requests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
5a5dad4b95
commit
e34c874f91
8 changed files with 566 additions and 138 deletions
68
litellm-rust/crates/gateway-forward/tests/client_identity.rs
Normal file
68
litellm-rust/crates/gateway-forward/tests/client_identity.rs
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
mod support;
|
||||
|
||||
use axum::{body::Body, http::StatusCode, response::Response};
|
||||
use litellm_gateway_forward::PEER_HEADER;
|
||||
use rstest::rstest;
|
||||
use support::{Gateway, recording, within};
|
||||
|
||||
fn ok() -> Response {
|
||||
Response::new(Body::from("ok"))
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn python_sees_forwarding_headers_untouched_and_the_tcp_peer_rust_saw() {
|
||||
let (upstream, mut received) = recording(ok).await;
|
||||
let gateway = Gateway::start(upstream).await;
|
||||
let mut raw = gateway.connect().await;
|
||||
raw.send(
|
||||
"GET /sso/callback HTTP/1.1\r\n\
|
||||
host: public.example\r\n\
|
||||
x-forwarded-for: 192.0.2.1\r\n\
|
||||
x-forwarded-for: 192.0.2.2\r\n\
|
||||
x-forwarded-proto: https\r\n\
|
||||
forwarded: for=192.0.2.1;proto=https\r\n\
|
||||
x-litellm-rust-peer: 10.1.2.3:1\r\n\r\n",
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
within(raw.response_head()).await.unwrap().status,
|
||||
StatusCode::OK
|
||||
);
|
||||
|
||||
let request = received.recv().await.unwrap();
|
||||
let forwarded_for: Vec<_> = request.headers.get_all("x-forwarded-for").iter().collect();
|
||||
assert_eq!(forwarded_for, ["192.0.2.1", "192.0.2.2"]);
|
||||
assert_eq!(request.headers["x-forwarded-proto"], "https");
|
||||
assert_eq!(request.headers["forwarded"], "for=192.0.2.1;proto=https");
|
||||
let peers: Vec<_> = request.headers.get_all(PEER_HEADER).iter().collect();
|
||||
assert_eq!(peers, [raw.local_address().to_string().as_str()]);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn http2_clients_reach_python_with_their_public_host() {
|
||||
let (upstream, mut received) = recording(ok).await;
|
||||
let gateway = Gateway::start(upstream).await;
|
||||
let stream = tokio::net::TcpStream::connect(gateway.address)
|
||||
.await
|
||||
.unwrap();
|
||||
let (mut sender, connection) = hyper::client::conn::http2::handshake(
|
||||
hyper_util::rt::TokioExecutor::new(),
|
||||
hyper_util::rt::TokioIo::new(stream),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::spawn(connection);
|
||||
let request = axum::http::Request::get("http://public.example/v1/models")
|
||||
.body(Body::empty())
|
||||
.unwrap();
|
||||
|
||||
let response = within(sender.send_request(request)).await.unwrap();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
received.recv().await.unwrap().headers["host"],
|
||||
"public.example"
|
||||
);
|
||||
}
|
||||
80
litellm-rust/crates/gateway-forward/tests/download.rs
Normal file
80
litellm-rust/crates/gateway-forward/tests/download.rs
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
mod support;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::{body::Body, extract::Request, http::StatusCode, response::Response};
|
||||
use http_body_util::BodyExt;
|
||||
use rstest::rstest;
|
||||
use support::{Gateway, client, raw_upstream, recording, within};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn streams_a_body_without_a_content_length_as_python_produces_it() {
|
||||
let first: &[u8] = &[0xff, 0xfb, 0x90, 0x64, 0x00, 0x0d, 0x0a];
|
||||
let rest = vec![0xa5; 64 * 1024];
|
||||
let finish = Arc::new(Notify::new());
|
||||
let finished = finish.clone();
|
||||
let tail = rest.clone();
|
||||
let upstream = raw_upstream(move |mut python| {
|
||||
let finished = finished.clone();
|
||||
let tail = tail.clone();
|
||||
async move {
|
||||
python.request_head().await.unwrap();
|
||||
python
|
||||
.send([&b"HTTP/1.1 200 OK\r\ncontent-type: audio/mpeg\r\ntransfer-encoding: chunked\r\n\r\n7\r\n"[..], first, b"\r\n"].concat())
|
||||
.await;
|
||||
finished.notified().await;
|
||||
python
|
||||
.send([format!("{:x}\r\n", tail.len()).as_bytes(), &tail, b"\r\n0\r\n\r\n"].concat())
|
||||
.await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
let gateway = Gateway::start(upstream).await;
|
||||
let mut response = within(
|
||||
client().request(
|
||||
Request::post(gateway.url("/v1/audio/speech"))
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(!response.headers().contains_key("content-length"));
|
||||
let frame = within(response.body_mut().frame()).await.unwrap().unwrap();
|
||||
assert_eq!(frame.into_data().unwrap(), first);
|
||||
finish.notify_one();
|
||||
let body = within(response.into_body().collect())
|
||||
.await
|
||||
.unwrap()
|
||||
.to_bytes();
|
||||
assert_eq!(body, rest);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn range_requests_and_partial_content_pass_unchanged() {
|
||||
let (upstream, mut received) = recording(|| -> Response {
|
||||
Response::builder()
|
||||
.status(StatusCode::PARTIAL_CONTENT)
|
||||
.header("content-range", "bytes 2-5/10")
|
||||
.header("accept-ranges", "bytes")
|
||||
.body(Body::from("2345"))
|
||||
.unwrap()
|
||||
})
|
||||
.await;
|
||||
let gateway = Gateway::start(upstream).await;
|
||||
let mut raw = gateway.connect().await;
|
||||
raw.send("GET /v1/videos/video_1/content HTTP/1.1\r\nhost: x\r\nrange: bytes=2-5\r\nif-range: \"v1\"\r\n\r\n")
|
||||
.await;
|
||||
|
||||
let head = within(raw.response_head()).await.unwrap();
|
||||
assert_eq!(head.status, StatusCode::PARTIAL_CONTENT);
|
||||
assert_eq!(head.header("content-range"), Some(&b"bytes 2-5/10"[..]));
|
||||
assert_eq!(raw.body(head.content_length().unwrap()).await, b"2345");
|
||||
let request = received.recv().await.unwrap();
|
||||
assert_eq!(request.headers["range"], "bytes=2-5");
|
||||
assert_eq!(request.headers["if-range"], "\"v1\"");
|
||||
}
|
||||
|
|
@ -1,19 +1,14 @@
|
|||
mod support;
|
||||
|
||||
use std::convert::Infallible;
|
||||
|
||||
use axum::{
|
||||
Router,
|
||||
body::{Body, Bytes},
|
||||
body::Body,
|
||||
extract::Request,
|
||||
http::{HeaderValue, StatusCode},
|
||||
response::Response,
|
||||
};
|
||||
use http_body_util::BodyExt;
|
||||
use litellm_gateway_forward::PEER_HEADER;
|
||||
use rstest::rstest;
|
||||
use support::{Gateway, client, raw_upstream, recording, serve, within};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
fn ok() -> Response {
|
||||
Response::new(Body::from("ok"))
|
||||
|
|
@ -38,11 +33,6 @@ async fn forwards_the_request_as_the_client_sent_it_minus_hop_headers() {
|
|||
"PATCH /v1/messages?x=a%2Fb&x=c HTTP/1.1\r\n\
|
||||
host: public.example\r\n\
|
||||
authorization: Bearer test\r\n\
|
||||
x-forwarded-for: 192.0.2.1\r\n\
|
||||
x-forwarded-for: 192.0.2.2\r\n\
|
||||
x-forwarded-proto: https\r\n\
|
||||
forwarded: for=192.0.2.1;proto=https\r\n\
|
||||
x-litellm-rust-peer: 10.1.2.3:1\r\n\
|
||||
connection: x-remove, keep-alive\r\n\
|
||||
x-remove: remove\r\n\
|
||||
proxy-authorization: remove\r\n\
|
||||
|
|
@ -62,12 +52,6 @@ async fn forwards_the_request_as_the_client_sent_it_minus_hop_headers() {
|
|||
assert_eq!(request.uri, "/v1/messages?x=a%2Fb&x=c");
|
||||
assert_eq!(request.headers["host"], "public.example");
|
||||
assert_eq!(request.headers["authorization"], "Bearer test");
|
||||
let forwarded_for: Vec<_> = request.headers.get_all("x-forwarded-for").iter().collect();
|
||||
assert_eq!(forwarded_for, ["192.0.2.1", "192.0.2.2"]);
|
||||
assert_eq!(request.headers["x-forwarded-proto"], "https");
|
||||
assert_eq!(request.headers["forwarded"], "for=192.0.2.1;proto=https");
|
||||
let peers: Vec<_> = request.headers.get_all(PEER_HEADER).iter().collect();
|
||||
assert_eq!(peers, [raw.local_address().to_string().as_str()]);
|
||||
assert!(!request.headers.contains_key("x-remove"));
|
||||
assert!(!request.headers.contains_key("proxy-authorization"));
|
||||
assert!(!request.headers.contains_key("te"));
|
||||
|
|
@ -139,34 +123,6 @@ async fn absolute_form_targets_reach_python_as_origin_form() {
|
|||
assert_eq!(request.uri, "/v1/models?x=1");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn http2_clients_reach_python_with_their_public_host() {
|
||||
let (upstream, mut received) = recording(ok).await;
|
||||
let gateway = Gateway::start(upstream).await;
|
||||
let stream = tokio::net::TcpStream::connect(gateway.address)
|
||||
.await
|
||||
.unwrap();
|
||||
let (mut sender, connection) = hyper::client::conn::http2::handshake(
|
||||
hyper_util::rt::TokioExecutor::new(),
|
||||
hyper_util::rt::TokioIo::new(stream),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::spawn(connection);
|
||||
let request = axum::http::Request::get("http://public.example/v1/models")
|
||||
.body(Body::empty())
|
||||
.unwrap();
|
||||
|
||||
let response = within(sender.send_request(request)).await.unwrap();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(
|
||||
received.recv().await.unwrap().headers["host"],
|
||||
"public.example"
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn relays_a_response_with_more_headers_than_hyper_allows_by_default() {
|
||||
|
|
@ -224,58 +180,6 @@ async fn keeps_repeated_response_headers_and_opaque_header_bytes() {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn streams_upload_and_response_incrementally_without_a_body_limit() {
|
||||
let upstream = serve(
|
||||
Router::new().fallback(|request: Request| async { Response::new(request.into_body()) }),
|
||||
)
|
||||
.await;
|
||||
let gateway = Gateway::start(upstream).await;
|
||||
let (sender, receiver) = mpsc::channel::<Result<Bytes, Infallible>>(1);
|
||||
let stream = futures_util::stream::unfold(receiver, |mut receiver| async {
|
||||
receiver.recv().await.map(|chunk| (chunk, receiver))
|
||||
});
|
||||
sender.send(Ok(Bytes::from_static(b"first"))).await.unwrap();
|
||||
let mut response = within(
|
||||
client().request(
|
||||
Request::post(gateway.url("/v1/files"))
|
||||
.body(Body::from_stream(stream))
|
||||
.unwrap(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let frame = within(response.body_mut().frame()).await.unwrap().unwrap();
|
||||
assert_eq!(frame.into_data().unwrap(), "first");
|
||||
let large = Bytes::from(vec![b'x'; 8 * 1024 * 1024]);
|
||||
sender.send(Ok(large.clone())).await.unwrap();
|
||||
drop(sender);
|
||||
let rest = within(response.into_body().collect())
|
||||
.await
|
||||
.unwrap()
|
||||
.to_bytes();
|
||||
assert_eq!(rest, large);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn answers_expect_continue_so_large_uploads_start() {
|
||||
let (upstream, mut received) = recording(ok).await;
|
||||
let gateway = Gateway::start(upstream).await;
|
||||
let mut raw = gateway.connect().await;
|
||||
raw.send("POST /v1/audio/transcriptions HTTP/1.1\r\nhost: x\r\nexpect: 100-continue\r\ncontent-length: 4\r\n\r\n")
|
||||
.await;
|
||||
let interim = within(raw.response_head()).await.unwrap();
|
||||
assert_eq!(interim.status, StatusCode::CONTINUE);
|
||||
raw.send("data").await;
|
||||
assert_eq!(
|
||||
within(raw.response_head()).await.unwrap().status,
|
||||
StatusCode::OK
|
||||
);
|
||||
assert_eq!(received.recv().await.unwrap().body, "data");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn answers_pipelined_requests_in_order() {
|
||||
|
|
@ -318,37 +222,83 @@ async fn bodiless_statuses_keep_the_connection_reusable(#[case] status: StatusCo
|
|||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn relays_a_body_delimited_by_the_upstream_closing() {
|
||||
async fn informational_responses_keep_the_connection_reusable() {
|
||||
let upstream = raw_upstream(|mut python| async move {
|
||||
python.request_head().await.unwrap();
|
||||
python
|
||||
.send("HTTP/1.1 200 OK\r\nconnection: close\r\n\r\nuntil eof")
|
||||
.await;
|
||||
while let Some(head) = python.request_head().await {
|
||||
match head.path.as_str() {
|
||||
"/hints" => {
|
||||
python
|
||||
.send("HTTP/1.1 103 Early Hints\r\nlink: </ui/app.css>; rel=preload\r\n\r\nHTTP/1.1 200 OK\r\ncontent-length: 5\r\n\r\nfinal")
|
||||
.await
|
||||
}
|
||||
_ => {
|
||||
python
|
||||
.send("HTTP/1.1 200 OK\r\ncontent-length: 4\r\n\r\nnext")
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
let gateway = Gateway::start(upstream).await;
|
||||
let response = client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap());
|
||||
let body = within(response).await.unwrap().into_body().collect();
|
||||
assert_eq!(within(body).await.unwrap().to_bytes(), "until eof");
|
||||
let mut raw = gateway.connect().await;
|
||||
raw.send("GET /hints HTTP/1.1\r\nhost: x\r\n\r\nGET /next HTTP/1.1\r\nhost: x\r\n\r\n")
|
||||
.await;
|
||||
let mut hinted = within(raw.response_head()).await.unwrap();
|
||||
while hinted.status.is_informational() {
|
||||
hinted = within(raw.response_head()).await.unwrap();
|
||||
}
|
||||
assert_eq!(hinted.status, StatusCode::OK);
|
||||
assert_eq!(raw.body(hinted.content_length().unwrap()).await, b"final");
|
||||
let next = within(raw.response_head()).await.unwrap();
|
||||
assert_eq!(raw.body(next.content_length().unwrap()).await, b"next");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::found(
|
||||
StatusCode::FOUND,
|
||||
"https://idp.example/authorize?state=a%2Fb&redirect_uri=https%3A%2F%2Fpublic.example%2Fsso%2Fcallback"
|
||||
)]
|
||||
#[case::see_other(StatusCode::SEE_OTHER, "/ui/?login=success")]
|
||||
#[case::temporary(StatusCode::TEMPORARY_REDIRECT, "./mcp/oauth/authorize?client_id=x")]
|
||||
#[tokio::test]
|
||||
async fn an_upstream_dying_mid_stream_fails_the_client_body_instead_of_ending_it() {
|
||||
let upstream = raw_upstream(|mut python| async move {
|
||||
python.request_head().await.unwrap();
|
||||
python
|
||||
.send("HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\n\r\n6\r\ndata: \r\n")
|
||||
.await;
|
||||
async fn redirects_reach_the_client_with_location_and_cookies_unchanged(
|
||||
#[case] status: StatusCode,
|
||||
#[case] location: &'static str,
|
||||
) {
|
||||
let (upstream, _received) = recording(move || {
|
||||
Response::builder()
|
||||
.status(status)
|
||||
.header("location", location)
|
||||
.header(
|
||||
"set-cookie",
|
||||
"token=a%2Fb; Path=/; HttpOnly; Secure; SameSite=Lax",
|
||||
)
|
||||
.header("set-cookie", "state=; Max-Age=0")
|
||||
.body(Body::empty())
|
||||
.unwrap()
|
||||
})
|
||||
.await;
|
||||
let gateway = Gateway::start(upstream).await;
|
||||
let response =
|
||||
within(client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap()))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert!(within(response.into_body().collect()).await.is_err());
|
||||
let mut raw = gateway.connect().await;
|
||||
raw.send("GET /sso/key/generate HTTP/1.1\r\nhost: public.example\r\n\r\n")
|
||||
.await;
|
||||
let head = within(raw.response_head()).await.unwrap();
|
||||
assert_eq!(head.status, status);
|
||||
assert_eq!(head.header("location"), Some(location.as_bytes()));
|
||||
let cookies: Vec<_> = head
|
||||
.headers
|
||||
.iter()
|
||||
.filter(|(name, _)| name == "set-cookie")
|
||||
.map(|(_, value)| value.as_slice())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
cookies,
|
||||
[
|
||||
&b"token=a%2Fb; Path=/; HttpOnly; Secure; SameSite=Lax"[..],
|
||||
b"state=; Max-Age=0"
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
|
|
@ -368,24 +318,3 @@ async fn non_websocket_upgrades_reach_python_as_plain_requests() {
|
|||
assert!(!request.headers.contains_key("connection"));
|
||||
assert!(!request.headers.contains_key("http2-settings"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::get("GET")]
|
||||
#[case::delete("DELETE")]
|
||||
#[case::options("OPTIONS")]
|
||||
#[case::post("POST")]
|
||||
#[tokio::test]
|
||||
async fn forwards_a_chunked_body_on_any_method(#[case] method: &str) {
|
||||
let (upstream, mut received) = recording(ok).await;
|
||||
let gateway = Gateway::start(upstream).await;
|
||||
let mut raw = gateway.connect().await;
|
||||
raw.send(format!(
|
||||
"{method} /v1/search HTTP/1.1\r\nhost: x\r\ntransfer-encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\n\r\n"
|
||||
))
|
||||
.await;
|
||||
assert_eq!(
|
||||
within(raw.response_head()).await.unwrap().status,
|
||||
StatusCode::OK
|
||||
);
|
||||
assert_eq!(received.recv().await.unwrap().body, "body");
|
||||
}
|
||||
82
litellm-rust/crates/gateway-forward/tests/idle_stream.rs
Normal file
82
litellm-rust/crates/gateway-forward/tests/idle_stream.rs
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
mod support;
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::{
|
||||
body::Body,
|
||||
extract::Request,
|
||||
http::StatusCode,
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use http_body_util::BodyExt;
|
||||
use rstest::rstest;
|
||||
use support::{Gateway, client, raw_upstream, recording, within};
|
||||
use tokio::{sync::mpsc, time::Instant};
|
||||
|
||||
/// An MCP client holds a GET stream open for server-initiated messages, which may not come for
|
||||
/// many minutes, so neither hop may close a stream for being idle.
|
||||
#[rstest]
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn an_idle_mcp_stream_stays_open_in_both_directions() {
|
||||
let idle = Duration::from_secs(20 * 60);
|
||||
let (session, mut sessions) = mpsc::unbounded_channel();
|
||||
let upstream = raw_upstream(move |mut python| {
|
||||
let session = session.clone();
|
||||
async move {
|
||||
let head = python.request_head().await.unwrap();
|
||||
session
|
||||
.send(head.header("mcp-session-id").map(<[u8]>::to_vec))
|
||||
.unwrap();
|
||||
python
|
||||
.send("HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\nmcp-session-id: session-1\r\ntransfer-encoding: chunked\r\n\r\n")
|
||||
.await;
|
||||
tokio::time::sleep(idle).await;
|
||||
python.send("c\r\ndata: ping\n\n\r\n").await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
let gateway = Gateway::start(upstream).await;
|
||||
let started = Instant::now();
|
||||
|
||||
let mut response = client()
|
||||
.request(
|
||||
Request::get(gateway.url("/github/mcp"))
|
||||
.header("accept", "text/event-stream")
|
||||
.header("mcp-session-id", "session-1")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.headers()["mcp-session-id"], "session-1");
|
||||
assert_eq!(
|
||||
sessions.recv().await.unwrap().as_deref(),
|
||||
Some(&b"session-1"[..])
|
||||
);
|
||||
let frame = response.body_mut().frame().await.unwrap().unwrap();
|
||||
assert_eq!(frame.into_data().unwrap(), "data: ping\n\n");
|
||||
assert!(started.elapsed() >= idle);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn ends_a_session_with_a_bodiless_delete() {
|
||||
let (upstream, mut received) = recording(|| -> Response {
|
||||
([("mcp-session-id", "session-1")], StatusCode::NO_CONTENT).into_response()
|
||||
})
|
||||
.await;
|
||||
let gateway = Gateway::start(upstream).await;
|
||||
let mut raw = gateway.connect().await;
|
||||
raw.send("DELETE /mcp HTTP/1.1\r\nhost: x\r\nmcp-session-id: session-1\r\n\r\n")
|
||||
.await;
|
||||
|
||||
let head = within(raw.response_head()).await.unwrap();
|
||||
assert_eq!(head.status, StatusCode::NO_CONTENT);
|
||||
assert_eq!(head.header("mcp-session-id"), Some(&b"session-1"[..]));
|
||||
let request = received.recv().await.unwrap();
|
||||
assert_eq!(request.method, "DELETE");
|
||||
assert_eq!(request.headers["mcp-session-id"], "session-1");
|
||||
assert!(!request.headers.contains_key("transfer-encoding"));
|
||||
assert!(request.body.is_empty());
|
||||
}
|
||||
41
litellm-rust/crates/gateway-forward/tests/slow_head.rs
Normal file
41
litellm-rust/crates/gateway-forward/tests/slow_head.rs
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
mod support;
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::{body::Body, extract::Request, http::StatusCode};
|
||||
use http_body_util::BodyExt;
|
||||
use rstest::rstest;
|
||||
use support::{Gateway, client, raw_upstream};
|
||||
use tokio::time::Instant;
|
||||
|
||||
/// Reasoning models with `stream=false`, image and video generation, and model health checks send
|
||||
/// no response headers until the provider finishes, which can take many minutes.
|
||||
#[rstest]
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn waits_for_response_headers_as_long_as_python_takes() {
|
||||
let thinking = Duration::from_secs(30 * 60);
|
||||
let upstream = raw_upstream(move |mut python| async move {
|
||||
python.request_head().await.unwrap();
|
||||
tokio::time::sleep(thinking).await;
|
||||
python
|
||||
.send("HTTP/1.1 200 OK\r\ncontent-length: 5\r\n\r\nimage")
|
||||
.await;
|
||||
})
|
||||
.await;
|
||||
let gateway = Gateway::start(upstream).await;
|
||||
let started = Instant::now();
|
||||
|
||||
let response = client()
|
||||
.request(
|
||||
Request::post(gateway.url("/v1/images/generations"))
|
||||
.body(Body::from("{}"))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert!(started.elapsed() >= thinking);
|
||||
let body = response.into_body().collect().await.unwrap().to_bytes();
|
||||
assert_eq!(body, "image");
|
||||
}
|
||||
136
litellm-rust/crates/gateway-forward/tests/stream.rs
Normal file
136
litellm-rust/crates/gateway-forward/tests/stream.rs
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
mod support;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::{body::Body, extract::Request, http::StatusCode};
|
||||
use http_body_util::BodyExt;
|
||||
use rstest::rstest;
|
||||
use support::{Gateway, client, raw_upstream, within};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
fn chunk(bytes: &[u8]) -> Vec<u8> {
|
||||
[format!("{:x}\r\n", bytes.len()).as_bytes(), bytes, b"\r\n"].concat()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn each_event_reaches_the_client_before_python_produces_the_next() {
|
||||
let produce = Arc::new(Notify::new());
|
||||
let next = produce.clone();
|
||||
let upstream = raw_upstream(move |mut python| {
|
||||
let next = next.clone();
|
||||
async move {
|
||||
python.request_head().await.unwrap();
|
||||
python
|
||||
.send("HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\n\r\n")
|
||||
.await;
|
||||
for event in ["data: 1\n\n", "data: 2\n\n", "data: [DONE]\n\n"] {
|
||||
python.send(chunk(event.as_bytes())).await;
|
||||
next.notified().await;
|
||||
}
|
||||
python.send("0\r\n\r\n").await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
let gateway = Gateway::start(upstream).await;
|
||||
let mut response = within(
|
||||
client().request(
|
||||
Request::post(gateway.url("/v1/chat/completions"))
|
||||
.body(Body::from(r#"{"stream":true}"#))
|
||||
.unwrap(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
for event in ["data: 1\n\n", "data: 2\n\n", "data: [DONE]\n\n"] {
|
||||
let frame = within(response.body_mut().frame()).await.unwrap().unwrap();
|
||||
assert_eq!(frame.into_data().unwrap(), event);
|
||||
produce.notify_one();
|
||||
}
|
||||
assert!(within(response.body_mut().frame()).await.is_none());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn relays_a_binary_framed_stream_unchanged() {
|
||||
let message: Vec<u8> = [
|
||||
&[
|
||||
0x00, 0x00, 0x00, 0x3d, 0x00, 0x00, 0x00, 0x20, 0x8a, 0x0d, 0x0a, 0xff,
|
||||
][..],
|
||||
b"\x0b:event-type\x07\x00\x05chunk\x0d:content-type\x07\x00\x10application/json",
|
||||
b"0\r\n\r\n{\"bytes\":\"eyJ0eXBlIjoiY29udGVudCJ9\"}",
|
||||
&[0xc3, 0x28, 0x00, 0x0d],
|
||||
]
|
||||
.concat();
|
||||
let sent = message.clone();
|
||||
let upstream = raw_upstream(move |mut python| {
|
||||
let sent = sent.clone();
|
||||
async move {
|
||||
python.request_head().await.unwrap();
|
||||
python
|
||||
.send("HTTP/1.1 200 OK\r\ncontent-type: application/vnd.amazon.eventstream\r\ntransfer-encoding: chunked\r\n\r\n")
|
||||
.await;
|
||||
for piece in sent.chunks(7) {
|
||||
python.send(chunk(piece)).await;
|
||||
}
|
||||
python.send("0\r\n\r\n").await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
let gateway = Gateway::start(upstream).await;
|
||||
let response = within(
|
||||
client().request(
|
||||
Request::post(gateway.url("/bedrock/model/m/converse-stream"))
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
response.headers()["content-type"],
|
||||
"application/vnd.amazon.eventstream"
|
||||
);
|
||||
let body = within(response.into_body().collect())
|
||||
.await
|
||||
.unwrap()
|
||||
.to_bytes();
|
||||
assert_eq!(body, message);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn relays_a_body_delimited_by_the_upstream_closing() {
|
||||
let upstream = raw_upstream(|mut python| async move {
|
||||
python.request_head().await.unwrap();
|
||||
python
|
||||
.send("HTTP/1.1 200 OK\r\nconnection: close\r\n\r\nuntil eof")
|
||||
.await;
|
||||
})
|
||||
.await;
|
||||
let gateway = Gateway::start(upstream).await;
|
||||
let response = client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap());
|
||||
let body = within(response).await.unwrap().into_body().collect();
|
||||
assert_eq!(within(body).await.unwrap().to_bytes(), "until eof");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn an_upstream_dying_mid_stream_fails_the_client_body_instead_of_ending_it() {
|
||||
let upstream = raw_upstream(|mut python| async move {
|
||||
python.request_head().await.unwrap();
|
||||
python
|
||||
.send("HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\n\r\n6\r\ndata: \r\n")
|
||||
.await;
|
||||
})
|
||||
.await;
|
||||
let gateway = Gateway::start(upstream).await;
|
||||
let response =
|
||||
within(client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap()))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert!(within(response.into_body().collect()).await.is_err());
|
||||
}
|
||||
92
litellm-rust/crates/gateway-forward/tests/upload.rs
Normal file
92
litellm-rust/crates/gateway-forward/tests/upload.rs
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
mod support;
|
||||
|
||||
use std::convert::Infallible;
|
||||
|
||||
use axum::{
|
||||
Router,
|
||||
body::{Body, Bytes},
|
||||
extract::Request,
|
||||
http::StatusCode,
|
||||
response::Response,
|
||||
};
|
||||
use http_body_util::BodyExt;
|
||||
use rstest::rstest;
|
||||
use support::{Gateway, client, recording, serve, within};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
fn ok() -> Response {
|
||||
Response::new(Body::from("ok"))
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn streams_upload_and_response_incrementally_without_a_body_limit() {
|
||||
let upstream = serve(
|
||||
Router::new().fallback(|request: Request| async { Response::new(request.into_body()) }),
|
||||
)
|
||||
.await;
|
||||
let gateway = Gateway::start(upstream).await;
|
||||
let (sender, receiver) = mpsc::channel::<Result<Bytes, Infallible>>(1);
|
||||
let stream = futures_util::stream::unfold(receiver, |mut receiver| async {
|
||||
receiver.recv().await.map(|chunk| (chunk, receiver))
|
||||
});
|
||||
sender.send(Ok(Bytes::from_static(b"first"))).await.unwrap();
|
||||
let mut response = within(
|
||||
client().request(
|
||||
Request::post(gateway.url("/v1/files"))
|
||||
.body(Body::from_stream(stream))
|
||||
.unwrap(),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let frame = within(response.body_mut().frame()).await.unwrap().unwrap();
|
||||
assert_eq!(frame.into_data().unwrap(), "first");
|
||||
let large = Bytes::from(vec![b'x'; 8 * 1024 * 1024]);
|
||||
sender.send(Ok(large.clone())).await.unwrap();
|
||||
drop(sender);
|
||||
let rest = within(response.into_body().collect())
|
||||
.await
|
||||
.unwrap()
|
||||
.to_bytes();
|
||||
assert_eq!(rest, large);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn answers_expect_continue_so_large_uploads_start() {
|
||||
let (upstream, mut received) = recording(ok).await;
|
||||
let gateway = Gateway::start(upstream).await;
|
||||
let mut raw = gateway.connect().await;
|
||||
raw.send("POST /v1/audio/transcriptions HTTP/1.1\r\nhost: x\r\nexpect: 100-continue\r\ncontent-length: 4\r\n\r\n")
|
||||
.await;
|
||||
let interim = within(raw.response_head()).await.unwrap();
|
||||
assert_eq!(interim.status, StatusCode::CONTINUE);
|
||||
raw.send("data").await;
|
||||
assert_eq!(
|
||||
within(raw.response_head()).await.unwrap().status,
|
||||
StatusCode::OK
|
||||
);
|
||||
assert_eq!(received.recv().await.unwrap().body, "data");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::get("GET")]
|
||||
#[case::delete("DELETE")]
|
||||
#[case::options("OPTIONS")]
|
||||
#[case::post("POST")]
|
||||
#[tokio::test]
|
||||
async fn forwards_a_chunked_body_on_any_method(#[case] method: &str) {
|
||||
let (upstream, mut received) = recording(ok).await;
|
||||
let gateway = Gateway::start(upstream).await;
|
||||
let mut raw = gateway.connect().await;
|
||||
raw.send(format!(
|
||||
"{method} /v1/search HTTP/1.1\r\nhost: x\r\ntransfer-encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\n\r\n"
|
||||
))
|
||||
.await;
|
||||
assert_eq!(
|
||||
within(raw.response_head()).await.unwrap().status,
|
||||
StatusCode::OK
|
||||
);
|
||||
assert_eq!(received.recv().await.unwrap().body, "body");
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue