mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge aabe3dda67 into a80d97d2d6
This commit is contained in:
commit
fc0310dcee
5 changed files with 1018 additions and 6 deletions
|
|
@ -1,5 +1,9 @@
|
|||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
#[error(
|
||||
"LITELLM_RUST_GATEWAY=true requires LITELLM_PYTHON_UPSTREAM to be a valid http:// origin"
|
||||
)]
|
||||
PythonUpstreamConfig,
|
||||
#[error(transparent)]
|
||||
Http(#[from] litellm_http::Error),
|
||||
#[error("environment_variables values must be strings, numbers or booleans")]
|
||||
|
|
|
|||
|
|
@ -116,6 +116,11 @@ pub async fn build_mcp(
|
|||
))
|
||||
}
|
||||
|
||||
pub fn forward_only(forward: Forward) -> Router {
|
||||
dispatch::forward_unmatched(Router::new(), Fallback::Forward(forward))
|
||||
.layer(axum::middleware::from_fn(log_transport_request))
|
||||
}
|
||||
|
||||
pub fn router(
|
||||
inference: Arc<Gateway>,
|
||||
config: &Config,
|
||||
|
|
@ -164,8 +169,16 @@ pub async fn serve(
|
|||
}
|
||||
|
||||
async fn log_request(request: Request, next: Next) -> Response {
|
||||
let request_id = Uuid::new_v4().to_string();
|
||||
let log_body_chunks = tracing::enabled!(tracing::Level::DEBUG);
|
||||
log_request_with_body(request, next, log_body_chunks).await
|
||||
}
|
||||
|
||||
async fn log_transport_request(request: Request, next: Next) -> Response {
|
||||
log_request_with_body(request, next, false).await
|
||||
}
|
||||
|
||||
async fn log_request_with_body(request: Request, next: Next, log_body_chunks: bool) -> Response {
|
||||
let request_id = Uuid::new_v4().to_string();
|
||||
let method = request.method().clone();
|
||||
let path = request.uri().path().to_owned();
|
||||
let started = Instant::now();
|
||||
|
|
|
|||
|
|
@ -45,6 +45,19 @@ async fn main() -> Result<(), Box<dyn Error>> {
|
|||
let settings: Settings = settings::from_iter(std::env::vars_os())?;
|
||||
let level = settings.log_level();
|
||||
Logger::new(StderrSink { level }).install_global()?;
|
||||
if settings.litellm_rust_gateway {
|
||||
let forward = settings.forwarder()?;
|
||||
let listener =
|
||||
tokio::net::TcpListener::bind((settings.host.as_str(), settings.port)).await?;
|
||||
tracing::info!(address = %listener.local_addr()?, log_level = %level, "forward-only gateway listening");
|
||||
litellm_gateway::serve(
|
||||
listener,
|
||||
litellm_gateway::forward_only(forward),
|
||||
shutdown_signal(),
|
||||
)
|
||||
.await?;
|
||||
return Ok(());
|
||||
}
|
||||
let config = Config::load(settings.litellm_config)?;
|
||||
let inference = litellm_gateway::build_inference(&config)?;
|
||||
let fallback = match settings.litellm_python_upstream.as_deref() {
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ pub(super) struct Settings {
|
|||
pub port: u16,
|
||||
pub litellm_config: String,
|
||||
pub litellm_python_upstream: Option<String>,
|
||||
pub litellm_rust_gateway: bool,
|
||||
rust_log: Option<String>,
|
||||
}
|
||||
|
||||
|
|
@ -21,12 +22,38 @@ impl Default for Settings {
|
|||
port: 4000,
|
||||
litellm_config: "config.yaml".into(),
|
||||
litellm_python_upstream: None,
|
||||
litellm_rust_gateway: false,
|
||||
rust_log: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Settings {
|
||||
pub fn forwarder(&self) -> Result<litellm_gateway::Forward, litellm_gateway::Error> {
|
||||
let invalid = || litellm_gateway::Error::PythonUpstreamConfig;
|
||||
let upstream = self
|
||||
.litellm_python_upstream
|
||||
.as_deref()
|
||||
.ok_or_else(invalid)?;
|
||||
if upstream.contains('#') {
|
||||
return Err(invalid());
|
||||
}
|
||||
let uri: axum::http::Uri = upstream.parse().map_err(|_| invalid())?;
|
||||
let authority = uri.authority().ok_or_else(invalid)?;
|
||||
let port = authority.as_str().strip_prefix(authority.host());
|
||||
if authority.host().is_empty()
|
||||
|| !port.is_some_and(|suffix| {
|
||||
suffix.is_empty()
|
||||
|| suffix
|
||||
.strip_prefix(':')
|
||||
.is_some_and(|port| port.parse::<u16>().is_ok_and(|port| port != 0))
|
||||
})
|
||||
{
|
||||
return Err(invalid());
|
||||
}
|
||||
litellm_gateway::Forward::new(upstream).map_err(|_| invalid())
|
||||
}
|
||||
|
||||
pub fn log_level(&self) -> Level {
|
||||
self.rust_log
|
||||
.as_deref()
|
||||
|
|
@ -55,11 +82,22 @@ fn default_secure_cookies() -> bool {
|
|||
pub(super) fn from_iter<T: DeserializeOwned>(
|
||||
variables: impl IntoIterator<Item = (OsString, OsString)>,
|
||||
) -> envy::Result<T> {
|
||||
envy::from_iter(
|
||||
variables
|
||||
.into_iter()
|
||||
.filter_map(|(key, value)| Some((key.into_string().ok()?, value.into_string().ok()?))),
|
||||
)
|
||||
let variables = variables
|
||||
.into_iter()
|
||||
.map(|(key, value)| {
|
||||
if key
|
||||
.to_str()
|
||||
.is_some_and(|key| key.eq_ignore_ascii_case("LITELLM_RUST_GATEWAY"))
|
||||
&& !matches!(value.to_str(), Some("true" | "false"))
|
||||
{
|
||||
return Err(envy::Error::Custom(
|
||||
"LITELLM_RUST_GATEWAY must be true or false".into(),
|
||||
));
|
||||
}
|
||||
Ok(key.into_string().ok().zip(value.into_string().ok()))
|
||||
})
|
||||
.collect::<envy::Result<Vec<_>>>()?;
|
||||
envy::from_iter(variables.into_iter().flatten())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -84,9 +122,76 @@ mod tests {
|
|||
assert_eq!(settings.port, 4000);
|
||||
assert_eq!(settings.litellm_config, "config.yaml");
|
||||
assert_eq!(settings.litellm_python_upstream, None);
|
||||
assert!(!settings.litellm_rust_gateway);
|
||||
assert_eq!(settings.log_level(), Level::INFO);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::enabled("true", true)]
|
||||
#[case::disabled("false", false)]
|
||||
fn forward_only_flag(#[case] value: &str, #[case] expected: bool) {
|
||||
let settings: Settings = parse(&[("LITELLM_RUST_GATEWAY", value)]).unwrap();
|
||||
assert_eq!(settings.litellm_rust_gateway, expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::empty("")]
|
||||
#[case::uppercase("TRUE")]
|
||||
#[case::numeric("1")]
|
||||
#[case::padded(" true ")]
|
||||
fn invalid_forward_only_flag(#[case] value: &str) {
|
||||
assert!(parse::<Settings>(&[("LITELLM_RUST_GATEWAY", value)]).is_err());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[rstest]
|
||||
fn non_unicode_forward_only_flag_fails() {
|
||||
use std::os::unix::ffi::OsStringExt;
|
||||
|
||||
assert!(
|
||||
from_iter::<Settings>([(
|
||||
OsString::from("LITELLM_RUST_GATEWAY"),
|
||||
OsString::from_vec(vec![0xff]),
|
||||
)])
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::hostname("http://localhost")]
|
||||
#[case::explicit_port("http://127.0.0.1:4002")]
|
||||
#[case::ipv6("http://[::1]:4002")]
|
||||
fn valid_python_upstream_origins_build_forwarder(#[case] upstream: &str) {
|
||||
let settings: Settings = parse(&[("LITELLM_PYTHON_UPSTREAM", upstream)]).unwrap();
|
||||
|
||||
assert!(settings.forwarder().is_ok());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::empty("")]
|
||||
#[case::https("https://localhost:4002")]
|
||||
#[case::path("http://localhost:4002/python")]
|
||||
#[case::query("http://localhost:4002?token=secret")]
|
||||
#[case::fragment("http://localhost/#token=secret")]
|
||||
#[case::userinfo("http://user:secret@localhost:4002")]
|
||||
#[case::malformed("http://[::1")]
|
||||
#[case::named_port("http://localhost:http")]
|
||||
#[case::zero_port("http://localhost:0")]
|
||||
#[case::overflow_port("http://localhost:65536")]
|
||||
fn invalid_python_upstream_origins_reject_forwarder(#[case] upstream: &str) {
|
||||
let settings: Settings = parse(&[("LITELLM_PYTHON_UPSTREAM", upstream)]).unwrap();
|
||||
|
||||
let error = settings
|
||||
.forwarder()
|
||||
.err()
|
||||
.expect("invalid origin should fail");
|
||||
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"LITELLM_RUST_GATEWAY=true requires LITELLM_PYTHON_UPSTREAM to be a valid http:// origin"
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn environment_overrides_startup_settings() {
|
||||
let settings: Settings = parse(&[
|
||||
|
|
|
|||
877
litellm-rust/crates/gateway/tests/startup.rs
Normal file
877
litellm-rust/crates/gateway/tests/startup.rs
Normal file
|
|
@ -0,0 +1,877 @@
|
|||
use std::{
|
||||
fs,
|
||||
io::Write,
|
||||
net::SocketAddr,
|
||||
process::{Child, Command, Stdio},
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use axum::{
|
||||
Router,
|
||||
body::{Body, Bytes, to_bytes},
|
||||
extract::{Request, State},
|
||||
http::{
|
||||
HeaderMap, HeaderValue, Method, StatusCode, Uri,
|
||||
header::{AUTHORIZATION, CONTENT_TYPE, WWW_AUTHENTICATE},
|
||||
},
|
||||
response::Response,
|
||||
};
|
||||
use futures_util::{SinkExt, StreamExt, stream};
|
||||
use http_body_util::BodyExt;
|
||||
use litellm_gateway_inference::MAX_BODY_BYTES;
|
||||
use litellm_http::Client;
|
||||
use rstest::{fixture, rstest};
|
||||
use serde_json::{Value, json};
|
||||
use tempfile::NamedTempFile;
|
||||
use tokio::{
|
||||
io::AsyncReadExt,
|
||||
net::{TcpListener, TcpStream},
|
||||
sync::{mpsc, oneshot},
|
||||
task::JoinHandle,
|
||||
time::{sleep, timeout},
|
||||
};
|
||||
use tokio_tungstenite::{accept_async, client_async, tungstenite::Message};
|
||||
|
||||
const REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const STARTUP_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
static PROCESS_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
|
||||
|
||||
struct GatewayProcess {
|
||||
child: Child,
|
||||
address: SocketAddr,
|
||||
logs: NamedTempFile,
|
||||
_config: Option<NamedTempFile>,
|
||||
_lock: tokio::sync::MutexGuard<'static, ()>,
|
||||
}
|
||||
|
||||
impl GatewayProcess {
|
||||
async fn spawn(
|
||||
flag: Option<&str>,
|
||||
upstream: Option<&str>,
|
||||
extra: &[(&str, &str)],
|
||||
config: Option<&str>,
|
||||
) -> Self {
|
||||
let lock = PROCESS_LOCK.lock().await;
|
||||
let reserved = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = reserved.local_addr().unwrap();
|
||||
drop(reserved);
|
||||
|
||||
let config = config.map(|contents| {
|
||||
let mut file = NamedTempFile::new().unwrap();
|
||||
file.write_all(contents.as_bytes()).unwrap();
|
||||
file
|
||||
});
|
||||
let logs = NamedTempFile::new().unwrap();
|
||||
let mut command = Command::new(env!("CARGO_BIN_EXE_litellm-gateway"));
|
||||
command
|
||||
.env_clear()
|
||||
.env("HOST", address.ip().to_string())
|
||||
.env("PORT", address.port().to_string())
|
||||
.env("RUST_LOG", "debug")
|
||||
.envs(extra.iter().copied())
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::from(logs.as_file().try_clone().unwrap()));
|
||||
if let Some(flag) = flag {
|
||||
command.env("LITELLM_RUST_GATEWAY", flag);
|
||||
}
|
||||
if let Some(upstream) = upstream {
|
||||
command.env("LITELLM_PYTHON_UPSTREAM", upstream);
|
||||
}
|
||||
if let Some(config) = config.as_ref() {
|
||||
command.env("LITELLM_CONFIG", config.path());
|
||||
}
|
||||
let child = command.spawn().unwrap();
|
||||
|
||||
Self {
|
||||
child,
|
||||
address,
|
||||
logs,
|
||||
_config: config,
|
||||
_lock: lock,
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_until_ready(&mut self) {
|
||||
timeout(STARTUP_TIMEOUT, async {
|
||||
loop {
|
||||
if let Some(status) = self.child.try_wait().unwrap() {
|
||||
panic!(
|
||||
"gateway exited before listening with {status}: {}",
|
||||
self.read_logs()
|
||||
);
|
||||
}
|
||||
if TcpStream::connect(self.address).await.is_ok() {
|
||||
break;
|
||||
}
|
||||
sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("gateway should listen within the startup timeout");
|
||||
}
|
||||
|
||||
fn url(&self, path: &str) -> String {
|
||||
format!("http://{}{path}", self.address)
|
||||
}
|
||||
|
||||
async fn wait_for_exit(mut self) -> String {
|
||||
let status = timeout(STARTUP_TIMEOUT, async {
|
||||
loop {
|
||||
if let Some(status) = self.child.try_wait().unwrap() {
|
||||
break status;
|
||||
}
|
||||
sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("invalid startup should exit within the timeout");
|
||||
assert!(
|
||||
!status.success(),
|
||||
"invalid startup should exit unsuccessfully"
|
||||
);
|
||||
self.read_logs()
|
||||
}
|
||||
|
||||
fn stop(mut self) -> String {
|
||||
let _ = self.child.kill();
|
||||
let _ = self.child.wait();
|
||||
self.read_logs()
|
||||
}
|
||||
|
||||
fn read_logs(&self) -> String {
|
||||
fs::read_to_string(self.logs.path()).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for GatewayProcess {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.child.kill();
|
||||
let _ = self.child.wait();
|
||||
}
|
||||
}
|
||||
|
||||
struct ServerHandle {
|
||||
address: SocketAddr,
|
||||
shutdown: Option<oneshot::Sender<()>>,
|
||||
task: JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl ServerHandle {
|
||||
async fn start(app: Router) -> Self {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let (shutdown, stopped) = oneshot::channel();
|
||||
let task = tokio::spawn(async move {
|
||||
axum::serve(listener, app)
|
||||
.with_graceful_shutdown(async move {
|
||||
let _ = stopped.await;
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
Self {
|
||||
address,
|
||||
shutdown: Some(shutdown),
|
||||
task,
|
||||
}
|
||||
}
|
||||
|
||||
fn origin(&self) -> String {
|
||||
format!("http://{}", self.address)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ServerHandle {
|
||||
fn drop(&mut self) {
|
||||
if let Some(shutdown) = self.shutdown.take() {
|
||||
let _ = shutdown.send(());
|
||||
}
|
||||
self.task.abort();
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ObservedRequest {
|
||||
method: Method,
|
||||
uri: Uri,
|
||||
headers: HeaderMap,
|
||||
body: Bytes,
|
||||
}
|
||||
|
||||
struct RecordingUpstream {
|
||||
server: ServerHandle,
|
||||
received: mpsc::UnboundedReceiver<ObservedRequest>,
|
||||
}
|
||||
|
||||
impl RecordingUpstream {
|
||||
async fn echo() -> Self {
|
||||
let (sender, received) = mpsc::unbounded_channel();
|
||||
let app = Router::new().fallback(echo_request).with_state(sender);
|
||||
Self {
|
||||
server: ServerHandle::start(app).await,
|
||||
received,
|
||||
}
|
||||
}
|
||||
|
||||
async fn delegated_auth() -> Self {
|
||||
let (sender, received) = mpsc::unbounded_channel();
|
||||
let app = Router::new()
|
||||
.fallback(delegated_auth_request)
|
||||
.with_state(sender);
|
||||
Self {
|
||||
server: ServerHandle::start(app).await,
|
||||
received,
|
||||
}
|
||||
}
|
||||
|
||||
fn origin(&self) -> String {
|
||||
self.server.origin()
|
||||
}
|
||||
|
||||
async fn next(&mut self) -> ObservedRequest {
|
||||
timeout(REQUEST_TIMEOUT, self.received.recv())
|
||||
.await
|
||||
.expect("upstream should receive the request")
|
||||
.expect("upstream request channel should remain open")
|
||||
}
|
||||
}
|
||||
|
||||
#[fixture]
|
||||
async fn echo_upstream() -> RecordingUpstream {
|
||||
RecordingUpstream::echo().await
|
||||
}
|
||||
|
||||
async fn read_request(
|
||||
sender: &mpsc::UnboundedSender<ObservedRequest>,
|
||||
request: Request,
|
||||
) -> (HeaderMap, Bytes) {
|
||||
let (parts, body) = request.into_parts();
|
||||
let body = to_bytes(body, MAX_BODY_BYTES + 1024 * 1024).await.unwrap();
|
||||
sender
|
||||
.send(ObservedRequest {
|
||||
method: parts.method,
|
||||
uri: parts.uri,
|
||||
headers: parts.headers.clone(),
|
||||
body: body.clone(),
|
||||
})
|
||||
.unwrap();
|
||||
(parts.headers, body)
|
||||
}
|
||||
|
||||
fn copy_header(headers: &HeaderMap, response: &mut Response, name: &'static str) {
|
||||
if let Some(value) = headers.get(name) {
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(name, HeaderValue::from_bytes(value.as_bytes()).unwrap());
|
||||
}
|
||||
}
|
||||
|
||||
async fn echo_request(
|
||||
State(sender): State<mpsc::UnboundedSender<ObservedRequest>>,
|
||||
request: Request,
|
||||
) -> Response {
|
||||
let (headers, body) = read_request(&sender, request).await;
|
||||
let mut response = Response::new(Body::from(body));
|
||||
*response.status_mut() = StatusCode::ACCEPTED;
|
||||
response.headers_mut().insert(
|
||||
"x-upstream-secret",
|
||||
HeaderValue::from_static("response-secret"),
|
||||
);
|
||||
copy_header(&headers, &mut response, CONTENT_TYPE.as_str());
|
||||
copy_header(&headers, &mut response, "mcp-session-id");
|
||||
copy_header(&headers, &mut response, "last-event-id");
|
||||
copy_header(&headers, &mut response, "mcp-protocol-version");
|
||||
response
|
||||
}
|
||||
|
||||
async fn delegated_auth_request(
|
||||
State(sender): State<mpsc::UnboundedSender<ObservedRequest>>,
|
||||
request: Request,
|
||||
) -> Response {
|
||||
let (headers, _) = read_request(&sender, request).await;
|
||||
let allowed = headers
|
||||
.get(AUTHORIZATION)
|
||||
.is_some_and(|value| value == "Bearer delegated-allow");
|
||||
let (status, body, decision) = if allowed {
|
||||
(StatusCode::CREATED, "python-only model response", "allowed")
|
||||
} else {
|
||||
(StatusCode::FORBIDDEN, "delegated auth denied", "rejected")
|
||||
};
|
||||
let mut response = Response::new(Body::from(body));
|
||||
*response.status_mut() = status;
|
||||
response
|
||||
.headers_mut()
|
||||
.insert("x-auth-decision", HeaderValue::from_static(decision));
|
||||
if !allowed {
|
||||
response.headers_mut().insert(
|
||||
WWW_AUTHENTICATE,
|
||||
HeaderValue::from_static("Bearer realm=python"),
|
||||
);
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
async fn forward_gateway(upstream: &str, extra: &[(&str, &str)]) -> GatewayProcess {
|
||||
let mut gateway = GatewayProcess::spawn(Some("true"), Some(upstream), extra, None).await;
|
||||
gateway.wait_until_ready().await;
|
||||
gateway
|
||||
}
|
||||
|
||||
fn client() -> Client {
|
||||
Client::for_test()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::chat_completions("/v1/chat/completions", "PATCH")]
|
||||
#[case::messages("/v1/messages", "PUT")]
|
||||
#[case::responses("/v1/responses", "DELETE")]
|
||||
#[case::liveliness("/health/liveliness", "POST")]
|
||||
#[case::readiness("/health/readiness", "PATCH")]
|
||||
#[case::health("/health", "OPTIONS")]
|
||||
#[case::ui("/ui", "POST")]
|
||||
#[case::mcp("/mcp", "PUT")]
|
||||
#[tokio::test]
|
||||
async fn forwards_every_path_method_and_body_to_python(
|
||||
#[future] echo_upstream: RecordingUpstream,
|
||||
#[case] path: &str,
|
||||
#[case] method: &str,
|
||||
) {
|
||||
let mut upstream = echo_upstream.await;
|
||||
let gateway = forward_gateway(&upstream.origin(), &[]).await;
|
||||
let body = format!("arbitrary body for {path}");
|
||||
let method = Method::from_bytes(method.as_bytes()).unwrap();
|
||||
|
||||
let response = client()
|
||||
.request(method.clone(), gateway.url(path))
|
||||
.header("x-client-value", "preserved")
|
||||
.body(body.clone())
|
||||
.timeout(REQUEST_TIMEOUT)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::ACCEPTED);
|
||||
assert_eq!(response.bytes().await.unwrap(), body.as_bytes());
|
||||
let observed = upstream.next().await;
|
||||
assert_eq!(observed.method, method);
|
||||
assert_eq!(observed.uri.path(), path);
|
||||
assert_eq!(observed.headers["x-client-value"], "preserved");
|
||||
assert_eq!(
|
||||
observed.headers["x-litellm-gateway-declined"],
|
||||
"route not native"
|
||||
);
|
||||
assert_eq!(observed.body, body.as_bytes());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::missing_config(None, &[("LITELLM_CONFIG", "/missing/gateway-config.yaml")])]
|
||||
#[case::invalid_config_and_ui(
|
||||
Some("model_list: ["),
|
||||
&[
|
||||
("LITELLM_UI_PATH", "/missing/dashboard"),
|
||||
("LITELLM_UI_SECURE_COOKIES", "not-a-bool"),
|
||||
],
|
||||
)]
|
||||
#[tokio::test]
|
||||
async fn forward_only_starts_before_config_and_ui_loading(
|
||||
#[future] echo_upstream: RecordingUpstream,
|
||||
#[case] config: Option<&str>,
|
||||
#[case] extra: &[(&str, &str)],
|
||||
) {
|
||||
let mut upstream = echo_upstream.await;
|
||||
let mut gateway =
|
||||
GatewayProcess::spawn(Some("true"), Some(&upstream.origin()), extra, config).await;
|
||||
gateway.wait_until_ready().await;
|
||||
|
||||
let response = client()
|
||||
.post(gateway.url("/v1/chat/completions"))
|
||||
.body("python owns this request")
|
||||
.timeout(REQUEST_TIMEOUT)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::ACCEPTED);
|
||||
assert_eq!(response.bytes().await.unwrap(), "python owns this request");
|
||||
assert_eq!(upstream.next().await.uri.path(), "/v1/chat/completions");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::allowed("Bearer delegated-allow", StatusCode::CREATED)]
|
||||
#[case::rejected("Bearer delegated-deny", StatusCode::FORBIDDEN)]
|
||||
#[tokio::test]
|
||||
async fn python_only_models_and_delegated_auth_match_direct_python(
|
||||
#[case] authorization: &str,
|
||||
#[case] expected: StatusCode,
|
||||
) {
|
||||
let mut upstream = RecordingUpstream::delegated_auth().await;
|
||||
let gateway = forward_gateway(&upstream.origin(), &[]).await;
|
||||
let body = json!({"model": "python-only", "messages": [{"role": "user", "content": "hello"}]})
|
||||
.to_string();
|
||||
let direct = client()
|
||||
.post(format!("{}/v1/chat/completions", upstream.origin()))
|
||||
.header(AUTHORIZATION, authorization)
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.body(body.clone())
|
||||
.timeout(REQUEST_TIMEOUT)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let direct_status = direct.status();
|
||||
let direct_decision = direct.headers()["x-auth-decision"].clone();
|
||||
let direct_challenge = direct.headers().get(WWW_AUTHENTICATE).cloned();
|
||||
let direct_body = direct.bytes().await.unwrap();
|
||||
let forwarded = client()
|
||||
.post(gateway.url("/v1/chat/completions"))
|
||||
.header(AUTHORIZATION, authorization)
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.body(body.clone())
|
||||
.timeout(REQUEST_TIMEOUT)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(direct_status, expected);
|
||||
assert_eq!(forwarded.status(), direct_status);
|
||||
assert_eq!(forwarded.headers()["x-auth-decision"], direct_decision);
|
||||
assert_eq!(
|
||||
forwarded.headers().get(WWW_AUTHENTICATE),
|
||||
direct_challenge.as_ref()
|
||||
);
|
||||
assert_eq!(forwarded.bytes().await.unwrap(), direct_body);
|
||||
let _direct_request = upstream.next().await;
|
||||
let forwarded_request = upstream.next().await;
|
||||
assert_eq!(forwarded_request.headers[AUTHORIZATION], authorization);
|
||||
assert_eq!(forwarded_request.body, body.as_bytes());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn opaque_and_mcp_transport_data_round_trip_unchanged(
|
||||
#[future] echo_upstream: RecordingUpstream,
|
||||
) {
|
||||
let mut upstream = echo_upstream.await;
|
||||
let gateway = forward_gateway(&upstream.origin(), &[]).await;
|
||||
let body = Bytes::from_static(b"v=0\r\n\0\xffbinary");
|
||||
|
||||
let response = client()
|
||||
.post(gateway.url("/mcp?transport=opaque"))
|
||||
.header(CONTENT_TYPE, "application/sdp")
|
||||
.header("mcp-session-id", "session-123")
|
||||
.header("last-event-id", "event-456")
|
||||
.header("mcp-protocol-version", "2025-06-18")
|
||||
.body(body.clone())
|
||||
.timeout(REQUEST_TIMEOUT)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::ACCEPTED);
|
||||
assert_eq!(response.headers()[CONTENT_TYPE], "application/sdp");
|
||||
assert_eq!(response.headers()["mcp-session-id"], "session-123");
|
||||
assert_eq!(response.headers()["last-event-id"], "event-456");
|
||||
assert_eq!(response.headers()["mcp-protocol-version"], "2025-06-18");
|
||||
assert_eq!(response.bytes().await.unwrap(), body);
|
||||
let observed = upstream.next().await;
|
||||
assert_eq!(
|
||||
observed.uri.path_and_query().unwrap(),
|
||||
"/mcp?transport=opaque"
|
||||
);
|
||||
assert_eq!(observed.headers[CONTENT_TYPE], "application/sdp");
|
||||
assert_eq!(observed.headers["mcp-session-id"], "session-123");
|
||||
assert_eq!(observed.headers["last-event-id"], "event-456");
|
||||
assert_eq!(observed.headers["mcp-protocol-version"], "2025-06-18");
|
||||
assert_eq!(observed.body, body);
|
||||
}
|
||||
|
||||
struct StreamingState {
|
||||
first_chunk: tokio::sync::Mutex<Option<oneshot::Sender<()>>>,
|
||||
}
|
||||
|
||||
async fn count_streamed_body(
|
||||
State(state): State<Arc<StreamingState>>,
|
||||
request: Request,
|
||||
) -> Response {
|
||||
let mut body = request.into_body();
|
||||
let mut size = 0;
|
||||
while let Some(frame) = body.frame().await {
|
||||
let frame = frame.unwrap();
|
||||
if let Some(data) = frame.data_ref() {
|
||||
size += data.len();
|
||||
if let Some(first_chunk) = state.first_chunk.lock().await.take() {
|
||||
let _ = first_chunk.send(());
|
||||
}
|
||||
}
|
||||
}
|
||||
Response::new(Body::from(size.to_string()))
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn streams_uploads_larger_than_the_native_body_limit_before_eof() {
|
||||
let (first_chunk, first_chunk_received) = oneshot::channel();
|
||||
let upstream = ServerHandle::start(Router::new().fallback(count_streamed_body).with_state(
|
||||
Arc::new(StreamingState {
|
||||
first_chunk: tokio::sync::Mutex::new(Some(first_chunk)),
|
||||
}),
|
||||
))
|
||||
.await;
|
||||
let gateway = forward_gateway(&upstream.origin(), &[]).await;
|
||||
let (release, released) = oneshot::channel::<()>();
|
||||
let first =
|
||||
stream::once(async { Ok::<_, std::io::Error>(Bytes::from_static(b"streamed-before-eof")) });
|
||||
let remainder = stream::once(async move {
|
||||
let _ = released.await;
|
||||
Ok(Bytes::from(vec![b'x'; MAX_BODY_BYTES]))
|
||||
});
|
||||
let request = tokio::spawn(
|
||||
client()
|
||||
.post(gateway.url("/large-python-upload"))
|
||||
.body(reqwest::Body::wrap_stream(first.chain(remainder)))
|
||||
.timeout(Duration::from_secs(30))
|
||||
.send(),
|
||||
);
|
||||
|
||||
timeout(REQUEST_TIMEOUT, first_chunk_received)
|
||||
.await
|
||||
.expect("upstream should receive the first chunk before request EOF")
|
||||
.unwrap();
|
||||
release.send(()).unwrap();
|
||||
let response = timeout(Duration::from_secs(30), request)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let received = response.text().await.unwrap().parse::<usize>().unwrap();
|
||||
assert!(received > MAX_BODY_BYTES);
|
||||
}
|
||||
|
||||
struct PartialDisconnect {
|
||||
address: SocketAddr,
|
||||
body_seen: Option<oneshot::Receiver<()>>,
|
||||
task: Option<JoinHandle<usize>>,
|
||||
}
|
||||
|
||||
impl PartialDisconnect {
|
||||
async fn start() -> Self {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let (body_seen, seen) = oneshot::channel();
|
||||
let task = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
let mut received = Vec::new();
|
||||
let mut chunk = [0; 4096];
|
||||
loop {
|
||||
let size = socket.read(&mut chunk).await.unwrap();
|
||||
if size == 0 {
|
||||
break;
|
||||
}
|
||||
received.extend_from_slice(&chunk[..size]);
|
||||
let body_started = received
|
||||
.windows(4)
|
||||
.position(|window| window == b"\r\n\r\n")
|
||||
.is_some_and(|headers| received.len() > headers + 4);
|
||||
if body_started {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let _ = body_seen.send(());
|
||||
drop(socket);
|
||||
match timeout(Duration::from_secs(2), listener.accept()).await {
|
||||
Ok(Ok((socket, _))) => {
|
||||
drop(socket);
|
||||
2
|
||||
}
|
||||
_ => 1,
|
||||
}
|
||||
});
|
||||
Self {
|
||||
address,
|
||||
body_seen: Some(seen),
|
||||
task: Some(task),
|
||||
}
|
||||
}
|
||||
|
||||
fn origin(&self) -> String {
|
||||
format!("http://{}", self.address)
|
||||
}
|
||||
|
||||
async fn body_seen(&mut self) {
|
||||
timeout(
|
||||
REQUEST_TIMEOUT,
|
||||
self.body_seen.take().expect("body receiver should exist"),
|
||||
)
|
||||
.await
|
||||
.expect("upstream should consume part of the request")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn finish(mut self) -> usize {
|
||||
timeout(REQUEST_TIMEOUT, self.task.take().unwrap())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PartialDisconnect {
|
||||
fn drop(&mut self) {
|
||||
if let Some(task) = &self.task {
|
||||
task.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn does_not_retry_after_upstream_consumes_a_partial_body_and_disconnects() {
|
||||
let mut upstream = PartialDisconnect::start().await;
|
||||
let gateway = forward_gateway(&upstream.origin(), &[]).await;
|
||||
let (release, released) = oneshot::channel::<()>();
|
||||
let first =
|
||||
stream::once(async { Ok::<_, std::io::Error>(Bytes::from_static(b"first request bytes")) });
|
||||
let remainder = stream::once(async move {
|
||||
let _ = released.await;
|
||||
Ok(Bytes::from_static(b"remaining request bytes"))
|
||||
});
|
||||
let request = tokio::spawn(
|
||||
client()
|
||||
.post(gateway.url("/partial-disconnect"))
|
||||
.body(reqwest::Body::wrap_stream(first.chain(remainder)))
|
||||
.timeout(REQUEST_TIMEOUT)
|
||||
.send(),
|
||||
);
|
||||
|
||||
upstream.body_seen().await;
|
||||
release.send(()).unwrap();
|
||||
let response = timeout(REQUEST_TIMEOUT, request)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::BAD_GATEWAY);
|
||||
assert_eq!(upstream.finish().await, 1);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn unreachable_upstream_returns_bad_gateway() {
|
||||
let unreachable = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let origin = format!("http://{}", unreachable.local_addr().unwrap());
|
||||
let gateway = forward_gateway(&origin, &[]).await;
|
||||
drop(unreachable);
|
||||
|
||||
let response = client()
|
||||
.post(gateway.url("/python-only"))
|
||||
.body("request")
|
||||
.timeout(REQUEST_TIMEOUT)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::BAD_GATEWAY);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::invalid_flag(
|
||||
Some("gateway-flag-secret"),
|
||||
Some("http://127.0.0.1:4002"),
|
||||
"gateway-flag-secret"
|
||||
)]
|
||||
#[case::missing_upstream(Some("true"), None, "configuration-secret")]
|
||||
#[case::empty_upstream(Some("true"), Some(""), "configuration-secret")]
|
||||
#[case::https(
|
||||
Some("true"),
|
||||
Some("https://upstream-secret.invalid"),
|
||||
"upstream-secret"
|
||||
)]
|
||||
#[case::path(
|
||||
Some("true"),
|
||||
Some("http://127.0.0.1/upstream-secret"),
|
||||
"upstream-secret"
|
||||
)]
|
||||
#[case::query(
|
||||
Some("true"),
|
||||
Some("http://127.0.0.1?token=upstream-secret"),
|
||||
"upstream-secret"
|
||||
)]
|
||||
#[case::fragment(
|
||||
Some("true"),
|
||||
Some("http://127.0.0.1/#upstream-secret"),
|
||||
"upstream-secret"
|
||||
)]
|
||||
#[case::userinfo(
|
||||
Some("true"),
|
||||
Some("http://upstream-secret@127.0.0.1"),
|
||||
"upstream-secret"
|
||||
)]
|
||||
#[case::malformed(Some("true"), Some("http://[upstream-secret"), "upstream-secret")]
|
||||
#[case::malformed_port(
|
||||
Some("true"),
|
||||
Some("http://127.0.0.1:upstream-secret"),
|
||||
"upstream-secret"
|
||||
)]
|
||||
#[case::zero_port(Some("true"), Some("http://127.0.0.1:0"), ":0")]
|
||||
#[case::overflow_port(Some("true"), Some("http://127.0.0.1:65536"), "65536")]
|
||||
#[tokio::test]
|
||||
async fn invalid_startup_inputs_fail_closed_without_echoing_values(
|
||||
#[case] flag: Option<&str>,
|
||||
#[case] upstream: Option<&str>,
|
||||
#[case] forbidden: &str,
|
||||
) {
|
||||
let process = GatewayProcess::spawn(flag, upstream, &[], None).await;
|
||||
|
||||
let logs = process.wait_for_exit().await;
|
||||
|
||||
assert!(!logs.contains(forbidden));
|
||||
}
|
||||
|
||||
const NATIVE_CONFIG: &str = "model_list: []\ngeneral_settings:\n master_key: gateway-key\n";
|
||||
|
||||
#[rstest]
|
||||
#[case::unset(None)]
|
||||
#[case::explicit_false(Some("false"))]
|
||||
#[tokio::test]
|
||||
async fn disabled_forward_only_flag_keeps_native_auth_health_and_dispatch(
|
||||
#[case] flag: Option<&str>,
|
||||
) {
|
||||
let mut gateway = GatewayProcess::spawn(flag, None, &[], Some(NATIVE_CONFIG)).await;
|
||||
gateway.wait_until_ready().await;
|
||||
|
||||
let auth = client()
|
||||
.post(gateway.url("/v1/chat/completions"))
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.body(r#"{"model":"python-only","messages":[]}"#)
|
||||
.timeout(REQUEST_TIMEOUT)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let health = client()
|
||||
.get(gateway.url("/health"))
|
||||
.timeout(REQUEST_TIMEOUT)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(auth.status(), StatusCode::UNAUTHORIZED);
|
||||
assert_eq!(health.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn python_upstream_without_enable_flag_keeps_native_gateway(
|
||||
#[future] echo_upstream: RecordingUpstream,
|
||||
) {
|
||||
let mut upstream = echo_upstream.await;
|
||||
let mut gateway =
|
||||
GatewayProcess::spawn(None, Some(&upstream.origin()), &[], Some(NATIVE_CONFIG)).await;
|
||||
gateway.wait_until_ready().await;
|
||||
|
||||
let native = client()
|
||||
.post(gateway.url("/v1/chat/completions"))
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.body(r#"{"model":"python-only","messages":[]}"#)
|
||||
.timeout(REQUEST_TIMEOUT)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(native.status(), StatusCode::UNAUTHORIZED);
|
||||
assert!(upstream.received.try_recv().is_err());
|
||||
|
||||
let fallback = client()
|
||||
.post(gateway.url("/python-only"))
|
||||
.body("fallback request")
|
||||
.timeout(REQUEST_TIMEOUT)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(fallback.status(), StatusCode::ACCEPTED);
|
||||
assert_eq!(upstream.next().await.uri.path(), "/python-only");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn executable_forward_only_mode_inherits_websocket_lifecycle() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let origin = format!("http://{}", listener.local_addr().unwrap());
|
||||
let upstream = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut socket = accept_async(stream).await.unwrap();
|
||||
let message = socket.next().await.unwrap().unwrap();
|
||||
socket.send(message).await.unwrap();
|
||||
});
|
||||
let gateway = forward_gateway(&origin, &[]).await;
|
||||
let stream = timeout(REQUEST_TIMEOUT, TcpStream::connect(gateway.address))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let (mut socket, response) = timeout(
|
||||
REQUEST_TIMEOUT,
|
||||
client_async(
|
||||
format!("ws://{}/realtime?model=python-only", gateway.address),
|
||||
stream,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS);
|
||||
socket.send(Message::text("through gateway")).await.unwrap();
|
||||
assert_eq!(
|
||||
timeout(REQUEST_TIMEOUT, socket.next())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
Message::text("through gateway")
|
||||
);
|
||||
drop(socket);
|
||||
timeout(REQUEST_TIMEOUT, upstream).await.unwrap().unwrap();
|
||||
}
|
||||
|
||||
fn outcome_record(logs: &str) -> Value {
|
||||
logs.lines()
|
||||
.filter_map(|line| serde_json::from_str::<Value>(line).ok())
|
||||
.find(|record| record["message"] == "response headers")
|
||||
.expect("forward-only request should emit one outcome record")
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn debug_logs_only_request_outcome_metadata(#[future] echo_upstream: RecordingUpstream) {
|
||||
let upstream = echo_upstream.await;
|
||||
let origin = upstream.origin();
|
||||
let gateway = forward_gateway(&origin, &[]).await;
|
||||
|
||||
let response = client()
|
||||
.post(gateway.url("/v1/messages?token=query-secret"))
|
||||
.header(AUTHORIZATION, "Bearer header-secret")
|
||||
.body("request-body-secret")
|
||||
.timeout(REQUEST_TIMEOUT)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::ACCEPTED);
|
||||
assert_eq!(response.headers()["x-upstream-secret"], "response-secret");
|
||||
assert_eq!(response.bytes().await.unwrap(), "request-body-secret");
|
||||
let logs = gateway.stop();
|
||||
let record = outcome_record(&logs);
|
||||
|
||||
assert_eq!(record["fields"]["status"], 202);
|
||||
assert_eq!(record["fields"]["method"], "POST");
|
||||
assert_eq!(record["fields"]["path"], "/v1/messages");
|
||||
assert!(record["fields"]["request_id"].as_str().is_some());
|
||||
assert!(!logs.contains("header-secret"));
|
||||
assert!(!logs.contains("query-secret"));
|
||||
assert!(!logs.contains("request-body-secret"));
|
||||
assert!(!logs.contains("response-secret"));
|
||||
assert!(!logs.contains(&origin));
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue