From e34c874f91dd49474c1282ec4f7ef4ade41e95fa Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 13:16:55 +0900 Subject: [PATCH] test(gateway): organize forwarding tests by traffic shape Split http.rs into exchange, stream, upload and client_identity, rename connection.rs to lifecycle, and add slow_head, idle_stream and download. New cases cover minutes-long waits for headers and idle MCP streams on paused time, binary framed streams, 1xx, redirects with cookies and range requests. Co-Authored-By: Claude Opus 5.5 --- .../gateway-forward/tests/client_identity.rs | 68 ++++++ .../crates/gateway-forward/tests/download.rs | 80 +++++++ .../tests/{http.rs => exchange.rs} | 205 ++++++------------ .../gateway-forward/tests/idle_stream.rs | 82 +++++++ .../tests/{connection.rs => lifecycle.rs} | 0 .../crates/gateway-forward/tests/slow_head.rs | 41 ++++ .../crates/gateway-forward/tests/stream.rs | 136 ++++++++++++ .../crates/gateway-forward/tests/upload.rs | 92 ++++++++ 8 files changed, 566 insertions(+), 138 deletions(-) create mode 100644 litellm-rust/crates/gateway-forward/tests/client_identity.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/download.rs rename litellm-rust/crates/gateway-forward/tests/{http.rs => exchange.rs} (63%) create mode 100644 litellm-rust/crates/gateway-forward/tests/idle_stream.rs rename litellm-rust/crates/gateway-forward/tests/{connection.rs => lifecycle.rs} (100%) create mode 100644 litellm-rust/crates/gateway-forward/tests/slow_head.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/stream.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/upload.rs diff --git a/litellm-rust/crates/gateway-forward/tests/client_identity.rs b/litellm-rust/crates/gateway-forward/tests/client_identity.rs new file mode 100644 index 00000000000..e4d502fd01c --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/client_identity.rs @@ -0,0 +1,68 @@ +mod support; + +use axum::{body::Body, http::StatusCode, response::Response}; +use litellm_gateway_forward::PEER_HEADER; +use rstest::rstest; +use support::{Gateway, recording, within}; + +fn ok() -> Response { + Response::new(Body::from("ok")) +} + +#[rstest] +#[tokio::test] +async fn python_sees_forwarding_headers_untouched_and_the_tcp_peer_rust_saw() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send( + "GET /sso/callback HTTP/1.1\r\n\ + host: public.example\r\n\ + x-forwarded-for: 192.0.2.1\r\n\ + x-forwarded-for: 192.0.2.2\r\n\ + x-forwarded-proto: https\r\n\ + forwarded: for=192.0.2.1;proto=https\r\n\ + x-litellm-rust-peer: 10.1.2.3:1\r\n\r\n", + ) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + + let request = received.recv().await.unwrap(); + let forwarded_for: Vec<_> = request.headers.get_all("x-forwarded-for").iter().collect(); + assert_eq!(forwarded_for, ["192.0.2.1", "192.0.2.2"]); + assert_eq!(request.headers["x-forwarded-proto"], "https"); + assert_eq!(request.headers["forwarded"], "for=192.0.2.1;proto=https"); + let peers: Vec<_> = request.headers.get_all(PEER_HEADER).iter().collect(); + assert_eq!(peers, [raw.local_address().to_string().as_str()]); +} + +#[rstest] +#[tokio::test] +async fn http2_clients_reach_python_with_their_public_host() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let stream = tokio::net::TcpStream::connect(gateway.address) + .await + .unwrap(); + let (mut sender, connection) = hyper::client::conn::http2::handshake( + hyper_util::rt::TokioExecutor::new(), + hyper_util::rt::TokioIo::new(stream), + ) + .await + .unwrap(); + tokio::spawn(connection); + let request = axum::http::Request::get("http://public.example/v1/models") + .body(Body::empty()) + .unwrap(); + + let response = within(sender.send_request(request)).await.unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + received.recv().await.unwrap().headers["host"], + "public.example" + ); +} diff --git a/litellm-rust/crates/gateway-forward/tests/download.rs b/litellm-rust/crates/gateway-forward/tests/download.rs new file mode 100644 index 00000000000..ea22f6f610c --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/download.rs @@ -0,0 +1,80 @@ +mod support; + +use std::sync::Arc; + +use axum::{body::Body, extract::Request, http::StatusCode, response::Response}; +use http_body_util::BodyExt; +use rstest::rstest; +use support::{Gateway, client, raw_upstream, recording, within}; +use tokio::sync::Notify; + +#[rstest] +#[tokio::test] +async fn streams_a_body_without_a_content_length_as_python_produces_it() { + let first: &[u8] = &[0xff, 0xfb, 0x90, 0x64, 0x00, 0x0d, 0x0a]; + let rest = vec![0xa5; 64 * 1024]; + let finish = Arc::new(Notify::new()); + let finished = finish.clone(); + let tail = rest.clone(); + let upstream = raw_upstream(move |mut python| { + let finished = finished.clone(); + let tail = tail.clone(); + async move { + python.request_head().await.unwrap(); + python + .send([&b"HTTP/1.1 200 OK\r\ncontent-type: audio/mpeg\r\ntransfer-encoding: chunked\r\n\r\n7\r\n"[..], first, b"\r\n"].concat()) + .await; + finished.notified().await; + python + .send([format!("{:x}\r\n", tail.len()).as_bytes(), &tail, b"\r\n0\r\n\r\n"].concat()) + .await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut response = within( + client().request( + Request::post(gateway.url("/v1/audio/speech")) + .body(Body::empty()) + .unwrap(), + ), + ) + .await + .unwrap(); + + assert!(!response.headers().contains_key("content-length")); + let frame = within(response.body_mut().frame()).await.unwrap().unwrap(); + assert_eq!(frame.into_data().unwrap(), first); + finish.notify_one(); + let body = within(response.into_body().collect()) + .await + .unwrap() + .to_bytes(); + assert_eq!(body, rest); +} + +#[rstest] +#[tokio::test] +async fn range_requests_and_partial_content_pass_unchanged() { + let (upstream, mut received) = recording(|| -> Response { + Response::builder() + .status(StatusCode::PARTIAL_CONTENT) + .header("content-range", "bytes 2-5/10") + .header("accept-ranges", "bytes") + .body(Body::from("2345")) + .unwrap() + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET /v1/videos/video_1/content HTTP/1.1\r\nhost: x\r\nrange: bytes=2-5\r\nif-range: \"v1\"\r\n\r\n") + .await; + + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::PARTIAL_CONTENT); + assert_eq!(head.header("content-range"), Some(&b"bytes 2-5/10"[..])); + assert_eq!(raw.body(head.content_length().unwrap()).await, b"2345"); + let request = received.recv().await.unwrap(); + assert_eq!(request.headers["range"], "bytes=2-5"); + assert_eq!(request.headers["if-range"], "\"v1\""); +} diff --git a/litellm-rust/crates/gateway-forward/tests/http.rs b/litellm-rust/crates/gateway-forward/tests/exchange.rs similarity index 63% rename from litellm-rust/crates/gateway-forward/tests/http.rs rename to litellm-rust/crates/gateway-forward/tests/exchange.rs index 78de426f521..bc8564016de 100644 --- a/litellm-rust/crates/gateway-forward/tests/http.rs +++ b/litellm-rust/crates/gateway-forward/tests/exchange.rs @@ -1,19 +1,14 @@ mod support; -use std::convert::Infallible; - use axum::{ Router, - body::{Body, Bytes}, + body::Body, extract::Request, http::{HeaderValue, StatusCode}, response::Response, }; -use http_body_util::BodyExt; -use litellm_gateway_forward::PEER_HEADER; use rstest::rstest; use support::{Gateway, client, raw_upstream, recording, serve, within}; -use tokio::sync::mpsc; fn ok() -> Response { Response::new(Body::from("ok")) @@ -38,11 +33,6 @@ async fn forwards_the_request_as_the_client_sent_it_minus_hop_headers() { "PATCH /v1/messages?x=a%2Fb&x=c HTTP/1.1\r\n\ host: public.example\r\n\ authorization: Bearer test\r\n\ - x-forwarded-for: 192.0.2.1\r\n\ - x-forwarded-for: 192.0.2.2\r\n\ - x-forwarded-proto: https\r\n\ - forwarded: for=192.0.2.1;proto=https\r\n\ - x-litellm-rust-peer: 10.1.2.3:1\r\n\ connection: x-remove, keep-alive\r\n\ x-remove: remove\r\n\ proxy-authorization: remove\r\n\ @@ -62,12 +52,6 @@ async fn forwards_the_request_as_the_client_sent_it_minus_hop_headers() { assert_eq!(request.uri, "/v1/messages?x=a%2Fb&x=c"); assert_eq!(request.headers["host"], "public.example"); assert_eq!(request.headers["authorization"], "Bearer test"); - let forwarded_for: Vec<_> = request.headers.get_all("x-forwarded-for").iter().collect(); - assert_eq!(forwarded_for, ["192.0.2.1", "192.0.2.2"]); - assert_eq!(request.headers["x-forwarded-proto"], "https"); - assert_eq!(request.headers["forwarded"], "for=192.0.2.1;proto=https"); - let peers: Vec<_> = request.headers.get_all(PEER_HEADER).iter().collect(); - assert_eq!(peers, [raw.local_address().to_string().as_str()]); assert!(!request.headers.contains_key("x-remove")); assert!(!request.headers.contains_key("proxy-authorization")); assert!(!request.headers.contains_key("te")); @@ -139,34 +123,6 @@ async fn absolute_form_targets_reach_python_as_origin_form() { assert_eq!(request.uri, "/v1/models?x=1"); } -#[rstest] -#[tokio::test] -async fn http2_clients_reach_python_with_their_public_host() { - let (upstream, mut received) = recording(ok).await; - let gateway = Gateway::start(upstream).await; - let stream = tokio::net::TcpStream::connect(gateway.address) - .await - .unwrap(); - let (mut sender, connection) = hyper::client::conn::http2::handshake( - hyper_util::rt::TokioExecutor::new(), - hyper_util::rt::TokioIo::new(stream), - ) - .await - .unwrap(); - tokio::spawn(connection); - let request = axum::http::Request::get("http://public.example/v1/models") - .body(Body::empty()) - .unwrap(); - - let response = within(sender.send_request(request)).await.unwrap(); - - assert_eq!(response.status(), StatusCode::OK); - assert_eq!( - received.recv().await.unwrap().headers["host"], - "public.example" - ); -} - #[rstest] #[tokio::test] async fn relays_a_response_with_more_headers_than_hyper_allows_by_default() { @@ -224,58 +180,6 @@ async fn keeps_repeated_response_headers_and_opaque_header_bytes() { ); } -#[rstest] -#[tokio::test] -async fn streams_upload_and_response_incrementally_without_a_body_limit() { - let upstream = serve( - Router::new().fallback(|request: Request| async { Response::new(request.into_body()) }), - ) - .await; - let gateway = Gateway::start(upstream).await; - let (sender, receiver) = mpsc::channel::>(1); - let stream = futures_util::stream::unfold(receiver, |mut receiver| async { - receiver.recv().await.map(|chunk| (chunk, receiver)) - }); - sender.send(Ok(Bytes::from_static(b"first"))).await.unwrap(); - let mut response = within( - client().request( - Request::post(gateway.url("/v1/files")) - .body(Body::from_stream(stream)) - .unwrap(), - ), - ) - .await - .unwrap(); - let frame = within(response.body_mut().frame()).await.unwrap().unwrap(); - assert_eq!(frame.into_data().unwrap(), "first"); - let large = Bytes::from(vec![b'x'; 8 * 1024 * 1024]); - sender.send(Ok(large.clone())).await.unwrap(); - drop(sender); - let rest = within(response.into_body().collect()) - .await - .unwrap() - .to_bytes(); - assert_eq!(rest, large); -} - -#[rstest] -#[tokio::test] -async fn answers_expect_continue_so_large_uploads_start() { - let (upstream, mut received) = recording(ok).await; - let gateway = Gateway::start(upstream).await; - let mut raw = gateway.connect().await; - raw.send("POST /v1/audio/transcriptions HTTP/1.1\r\nhost: x\r\nexpect: 100-continue\r\ncontent-length: 4\r\n\r\n") - .await; - let interim = within(raw.response_head()).await.unwrap(); - assert_eq!(interim.status, StatusCode::CONTINUE); - raw.send("data").await; - assert_eq!( - within(raw.response_head()).await.unwrap().status, - StatusCode::OK - ); - assert_eq!(received.recv().await.unwrap().body, "data"); -} - #[rstest] #[tokio::test] async fn answers_pipelined_requests_in_order() { @@ -318,37 +222,83 @@ async fn bodiless_statuses_keep_the_connection_reusable(#[case] status: StatusCo #[rstest] #[tokio::test] -async fn relays_a_body_delimited_by_the_upstream_closing() { +async fn informational_responses_keep_the_connection_reusable() { let upstream = raw_upstream(|mut python| async move { - python.request_head().await.unwrap(); - python - .send("HTTP/1.1 200 OK\r\nconnection: close\r\n\r\nuntil eof") - .await; + while let Some(head) = python.request_head().await { + match head.path.as_str() { + "/hints" => { + python + .send("HTTP/1.1 103 Early Hints\r\nlink: ; rel=preload\r\n\r\nHTTP/1.1 200 OK\r\ncontent-length: 5\r\n\r\nfinal") + .await + } + _ => { + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 4\r\n\r\nnext") + .await + } + } + } }) .await; let gateway = Gateway::start(upstream).await; - let response = client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap()); - let body = within(response).await.unwrap().into_body().collect(); - assert_eq!(within(body).await.unwrap().to_bytes(), "until eof"); + let mut raw = gateway.connect().await; + raw.send("GET /hints HTTP/1.1\r\nhost: x\r\n\r\nGET /next HTTP/1.1\r\nhost: x\r\n\r\n") + .await; + let mut hinted = within(raw.response_head()).await.unwrap(); + while hinted.status.is_informational() { + hinted = within(raw.response_head()).await.unwrap(); + } + assert_eq!(hinted.status, StatusCode::OK); + assert_eq!(raw.body(hinted.content_length().unwrap()).await, b"final"); + let next = within(raw.response_head()).await.unwrap(); + assert_eq!(raw.body(next.content_length().unwrap()).await, b"next"); } #[rstest] +#[case::found( + StatusCode::FOUND, + "https://idp.example/authorize?state=a%2Fb&redirect_uri=https%3A%2F%2Fpublic.example%2Fsso%2Fcallback" +)] +#[case::see_other(StatusCode::SEE_OTHER, "/ui/?login=success")] +#[case::temporary(StatusCode::TEMPORARY_REDIRECT, "./mcp/oauth/authorize?client_id=x")] #[tokio::test] -async fn an_upstream_dying_mid_stream_fails_the_client_body_instead_of_ending_it() { - let upstream = raw_upstream(|mut python| async move { - python.request_head().await.unwrap(); - python - .send("HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\n\r\n6\r\ndata: \r\n") - .await; +async fn redirects_reach_the_client_with_location_and_cookies_unchanged( + #[case] status: StatusCode, + #[case] location: &'static str, +) { + let (upstream, _received) = recording(move || { + Response::builder() + .status(status) + .header("location", location) + .header( + "set-cookie", + "token=a%2Fb; Path=/; HttpOnly; Secure; SameSite=Lax", + ) + .header("set-cookie", "state=; Max-Age=0") + .body(Body::empty()) + .unwrap() }) .await; let gateway = Gateway::start(upstream).await; - let response = - within(client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap())) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::OK); - assert!(within(response.into_body().collect()).await.is_err()); + let mut raw = gateway.connect().await; + raw.send("GET /sso/key/generate HTTP/1.1\r\nhost: public.example\r\n\r\n") + .await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, status); + assert_eq!(head.header("location"), Some(location.as_bytes())); + let cookies: Vec<_> = head + .headers + .iter() + .filter(|(name, _)| name == "set-cookie") + .map(|(_, value)| value.as_slice()) + .collect(); + assert_eq!( + cookies, + [ + &b"token=a%2Fb; Path=/; HttpOnly; Secure; SameSite=Lax"[..], + b"state=; Max-Age=0" + ] + ); } #[rstest] @@ -368,24 +318,3 @@ async fn non_websocket_upgrades_reach_python_as_plain_requests() { assert!(!request.headers.contains_key("connection")); assert!(!request.headers.contains_key("http2-settings")); } - -#[rstest] -#[case::get("GET")] -#[case::delete("DELETE")] -#[case::options("OPTIONS")] -#[case::post("POST")] -#[tokio::test] -async fn forwards_a_chunked_body_on_any_method(#[case] method: &str) { - let (upstream, mut received) = recording(ok).await; - let gateway = Gateway::start(upstream).await; - let mut raw = gateway.connect().await; - raw.send(format!( - "{method} /v1/search HTTP/1.1\r\nhost: x\r\ntransfer-encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\n\r\n" - )) - .await; - assert_eq!( - within(raw.response_head()).await.unwrap().status, - StatusCode::OK - ); - assert_eq!(received.recv().await.unwrap().body, "body"); -} diff --git a/litellm-rust/crates/gateway-forward/tests/idle_stream.rs b/litellm-rust/crates/gateway-forward/tests/idle_stream.rs new file mode 100644 index 00000000000..d0ce27fa6a9 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/idle_stream.rs @@ -0,0 +1,82 @@ +mod support; + +use std::time::Duration; + +use axum::{ + body::Body, + extract::Request, + http::StatusCode, + response::{IntoResponse, Response}, +}; +use http_body_util::BodyExt; +use rstest::rstest; +use support::{Gateway, client, raw_upstream, recording, within}; +use tokio::{sync::mpsc, time::Instant}; + +/// An MCP client holds a GET stream open for server-initiated messages, which may not come for +/// many minutes, so neither hop may close a stream for being idle. +#[rstest] +#[tokio::test(start_paused = true)] +async fn an_idle_mcp_stream_stays_open_in_both_directions() { + let idle = Duration::from_secs(20 * 60); + let (session, mut sessions) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python| { + let session = session.clone(); + async move { + let head = python.request_head().await.unwrap(); + session + .send(head.header("mcp-session-id").map(<[u8]>::to_vec)) + .unwrap(); + python + .send("HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\nmcp-session-id: session-1\r\ntransfer-encoding: chunked\r\n\r\n") + .await; + tokio::time::sleep(idle).await; + python.send("c\r\ndata: ping\n\n\r\n").await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let started = Instant::now(); + + let mut response = client() + .request( + Request::get(gateway.url("/github/mcp")) + .header("accept", "text/event-stream") + .header("mcp-session-id", "session-1") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.headers()["mcp-session-id"], "session-1"); + assert_eq!( + sessions.recv().await.unwrap().as_deref(), + Some(&b"session-1"[..]) + ); + let frame = response.body_mut().frame().await.unwrap().unwrap(); + assert_eq!(frame.into_data().unwrap(), "data: ping\n\n"); + assert!(started.elapsed() >= idle); +} + +#[rstest] +#[tokio::test] +async fn ends_a_session_with_a_bodiless_delete() { + let (upstream, mut received) = recording(|| -> Response { + ([("mcp-session-id", "session-1")], StatusCode::NO_CONTENT).into_response() + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("DELETE /mcp HTTP/1.1\r\nhost: x\r\nmcp-session-id: session-1\r\n\r\n") + .await; + + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::NO_CONTENT); + assert_eq!(head.header("mcp-session-id"), Some(&b"session-1"[..])); + let request = received.recv().await.unwrap(); + assert_eq!(request.method, "DELETE"); + assert_eq!(request.headers["mcp-session-id"], "session-1"); + assert!(!request.headers.contains_key("transfer-encoding")); + assert!(request.body.is_empty()); +} diff --git a/litellm-rust/crates/gateway-forward/tests/connection.rs b/litellm-rust/crates/gateway-forward/tests/lifecycle.rs similarity index 100% rename from litellm-rust/crates/gateway-forward/tests/connection.rs rename to litellm-rust/crates/gateway-forward/tests/lifecycle.rs diff --git a/litellm-rust/crates/gateway-forward/tests/slow_head.rs b/litellm-rust/crates/gateway-forward/tests/slow_head.rs new file mode 100644 index 00000000000..764733aa524 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/slow_head.rs @@ -0,0 +1,41 @@ +mod support; + +use std::time::Duration; + +use axum::{body::Body, extract::Request, http::StatusCode}; +use http_body_util::BodyExt; +use rstest::rstest; +use support::{Gateway, client, raw_upstream}; +use tokio::time::Instant; + +/// Reasoning models with `stream=false`, image and video generation, and model health checks send +/// no response headers until the provider finishes, which can take many minutes. +#[rstest] +#[tokio::test(start_paused = true)] +async fn waits_for_response_headers_as_long_as_python_takes() { + let thinking = Duration::from_secs(30 * 60); + let upstream = raw_upstream(move |mut python| async move { + python.request_head().await.unwrap(); + tokio::time::sleep(thinking).await; + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 5\r\n\r\nimage") + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let started = Instant::now(); + + let response = client() + .request( + Request::post(gateway.url("/v1/images/generations")) + .body(Body::from("{}")) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + assert!(started.elapsed() >= thinking); + let body = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(body, "image"); +} diff --git a/litellm-rust/crates/gateway-forward/tests/stream.rs b/litellm-rust/crates/gateway-forward/tests/stream.rs new file mode 100644 index 00000000000..b533a8544a6 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/stream.rs @@ -0,0 +1,136 @@ +mod support; + +use std::sync::Arc; + +use axum::{body::Body, extract::Request, http::StatusCode}; +use http_body_util::BodyExt; +use rstest::rstest; +use support::{Gateway, client, raw_upstream, within}; +use tokio::sync::Notify; + +fn chunk(bytes: &[u8]) -> Vec { + [format!("{:x}\r\n", bytes.len()).as_bytes(), bytes, b"\r\n"].concat() +} + +#[rstest] +#[tokio::test] +async fn each_event_reaches_the_client_before_python_produces_the_next() { + let produce = Arc::new(Notify::new()); + let next = produce.clone(); + let upstream = raw_upstream(move |mut python| { + let next = next.clone(); + async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\n\r\n") + .await; + for event in ["data: 1\n\n", "data: 2\n\n", "data: [DONE]\n\n"] { + python.send(chunk(event.as_bytes())).await; + next.notified().await; + } + python.send("0\r\n\r\n").await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut response = within( + client().request( + Request::post(gateway.url("/v1/chat/completions")) + .body(Body::from(r#"{"stream":true}"#)) + .unwrap(), + ), + ) + .await + .unwrap(); + + for event in ["data: 1\n\n", "data: 2\n\n", "data: [DONE]\n\n"] { + let frame = within(response.body_mut().frame()).await.unwrap().unwrap(); + assert_eq!(frame.into_data().unwrap(), event); + produce.notify_one(); + } + assert!(within(response.body_mut().frame()).await.is_none()); +} + +#[rstest] +#[tokio::test] +async fn relays_a_binary_framed_stream_unchanged() { + let message: Vec = [ + &[ + 0x00, 0x00, 0x00, 0x3d, 0x00, 0x00, 0x00, 0x20, 0x8a, 0x0d, 0x0a, 0xff, + ][..], + b"\x0b:event-type\x07\x00\x05chunk\x0d:content-type\x07\x00\x10application/json", + b"0\r\n\r\n{\"bytes\":\"eyJ0eXBlIjoiY29udGVudCJ9\"}", + &[0xc3, 0x28, 0x00, 0x0d], + ] + .concat(); + let sent = message.clone(); + let upstream = raw_upstream(move |mut python| { + let sent = sent.clone(); + async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ncontent-type: application/vnd.amazon.eventstream\r\ntransfer-encoding: chunked\r\n\r\n") + .await; + for piece in sent.chunks(7) { + python.send(chunk(piece)).await; + } + python.send("0\r\n\r\n").await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let response = within( + client().request( + Request::post(gateway.url("/bedrock/model/m/converse-stream")) + .body(Body::empty()) + .unwrap(), + ), + ) + .await + .unwrap(); + + assert_eq!( + response.headers()["content-type"], + "application/vnd.amazon.eventstream" + ); + let body = within(response.into_body().collect()) + .await + .unwrap() + .to_bytes(); + assert_eq!(body, message); +} + +#[rstest] +#[tokio::test] +async fn relays_a_body_delimited_by_the_upstream_closing() { + let upstream = raw_upstream(|mut python| async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\nconnection: close\r\n\r\nuntil eof") + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let response = client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap()); + let body = within(response).await.unwrap().into_body().collect(); + assert_eq!(within(body).await.unwrap().to_bytes(), "until eof"); +} + +#[rstest] +#[tokio::test] +async fn an_upstream_dying_mid_stream_fails_the_client_body_instead_of_ending_it() { + let upstream = raw_upstream(|mut python| async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\n\r\n6\r\ndata: \r\n") + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let response = + within(client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap())) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert!(within(response.into_body().collect()).await.is_err()); +} diff --git a/litellm-rust/crates/gateway-forward/tests/upload.rs b/litellm-rust/crates/gateway-forward/tests/upload.rs new file mode 100644 index 00000000000..a0a20a4ce41 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/upload.rs @@ -0,0 +1,92 @@ +mod support; + +use std::convert::Infallible; + +use axum::{ + Router, + body::{Body, Bytes}, + extract::Request, + http::StatusCode, + response::Response, +}; +use http_body_util::BodyExt; +use rstest::rstest; +use support::{Gateway, client, recording, serve, within}; +use tokio::sync::mpsc; + +fn ok() -> Response { + Response::new(Body::from("ok")) +} + +#[rstest] +#[tokio::test] +async fn streams_upload_and_response_incrementally_without_a_body_limit() { + let upstream = serve( + Router::new().fallback(|request: Request| async { Response::new(request.into_body()) }), + ) + .await; + let gateway = Gateway::start(upstream).await; + let (sender, receiver) = mpsc::channel::>(1); + let stream = futures_util::stream::unfold(receiver, |mut receiver| async { + receiver.recv().await.map(|chunk| (chunk, receiver)) + }); + sender.send(Ok(Bytes::from_static(b"first"))).await.unwrap(); + let mut response = within( + client().request( + Request::post(gateway.url("/v1/files")) + .body(Body::from_stream(stream)) + .unwrap(), + ), + ) + .await + .unwrap(); + let frame = within(response.body_mut().frame()).await.unwrap().unwrap(); + assert_eq!(frame.into_data().unwrap(), "first"); + let large = Bytes::from(vec![b'x'; 8 * 1024 * 1024]); + sender.send(Ok(large.clone())).await.unwrap(); + drop(sender); + let rest = within(response.into_body().collect()) + .await + .unwrap() + .to_bytes(); + assert_eq!(rest, large); +} + +#[rstest] +#[tokio::test] +async fn answers_expect_continue_so_large_uploads_start() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("POST /v1/audio/transcriptions HTTP/1.1\r\nhost: x\r\nexpect: 100-continue\r\ncontent-length: 4\r\n\r\n") + .await; + let interim = within(raw.response_head()).await.unwrap(); + assert_eq!(interim.status, StatusCode::CONTINUE); + raw.send("data").await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(received.recv().await.unwrap().body, "data"); +} + +#[rstest] +#[case::get("GET")] +#[case::delete("DELETE")] +#[case::options("OPTIONS")] +#[case::post("POST")] +#[tokio::test] +async fn forwards_a_chunked_body_on_any_method(#[case] method: &str) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "{method} /v1/search HTTP/1.1\r\nhost: x\r\ntransfer-encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\n\r\n" + )) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(received.recv().await.unwrap().body, "body"); +}