From c2833d5fd74c9a911f867834eb8bd3312e2e8ce6 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 3 Nov 2026 12:24:20 +0900 Subject: [PATCH] fix(gateway): disable Nagle on both forwarding hops uvicorn sets TCP_NODELAY on every connection through asyncio, so token-by-token SSE and realtime audio deltas went out at once. Without it the forwarder can hold each small write for an ACK on real networks Co-Authored-By: Claude Opus 5.5 --- litellm-rust/crates/gateway-forward/src/forward.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/litellm-rust/crates/gateway-forward/src/forward.rs b/litellm-rust/crates/gateway-forward/src/forward.rs index 5941d173a6f..bc12fb53271 100644 --- a/litellm-rust/crates/gateway-forward/src/forward.rs +++ b/litellm-rust/crates/gateway-forward/src/forward.rs @@ -9,6 +9,7 @@ use axum::{ extract::{ConnectInfo, DefaultBodyLimit, Request, State}, http::{HeaderValue, StatusCode, Uri, Version, header}, response::Response, + serve::ListenerExt, }; use hyper_util::{ client::legacy::{Client, connect::HttpConnector}, @@ -45,10 +46,12 @@ impl Forward { { return Err(Error::Upstream); } + let mut connector = HttpConnector::new(); + connector.set_nodelay(true); let client = Client::builder(TokioExecutor::new()) .pool_timer(TokioTimer::new()) .pool_idle_timeout(upstream_keepalive / 2) - .build_http(); + .build(connector); Ok(Self { upstream, client, @@ -72,6 +75,11 @@ impl Forward { let upgrades = self.upgrades.clone(); let _close_on_drop = upgrades.close_on_drop(); let draining = upgrades.clone(); + let listener = listener.tap_io(|stream| { + if let Err(error) = stream.set_nodelay(true) { + tracing::warn!(%error, "could not disable Nagle on a client connection"); + } + }); let result = axum::serve( listener, self.router()