mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(gateway): disable Nagle on both forwarding hops
uvicorn sets TCP_NODELAY on every connection through asyncio, so token-by-token SSE and realtime audio deltas went out at once. Without it the forwarder can hold each small write for an ACK on real networks Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
d27528b10d
commit
c2833d5fd7
1 changed files with 9 additions and 1 deletions
|
|
@ -9,6 +9,7 @@ use axum::{
|
|||
extract::{ConnectInfo, DefaultBodyLimit, Request, State},
|
||||
http::{HeaderValue, StatusCode, Uri, Version, header},
|
||||
response::Response,
|
||||
serve::ListenerExt,
|
||||
};
|
||||
use hyper_util::{
|
||||
client::legacy::{Client, connect::HttpConnector},
|
||||
|
|
@ -45,10 +46,12 @@ impl Forward {
|
|||
{
|
||||
return Err(Error::Upstream);
|
||||
}
|
||||
let mut connector = HttpConnector::new();
|
||||
connector.set_nodelay(true);
|
||||
let client = Client::builder(TokioExecutor::new())
|
||||
.pool_timer(TokioTimer::new())
|
||||
.pool_idle_timeout(upstream_keepalive / 2)
|
||||
.build_http();
|
||||
.build(connector);
|
||||
Ok(Self {
|
||||
upstream,
|
||||
client,
|
||||
|
|
@ -72,6 +75,11 @@ impl Forward {
|
|||
let upgrades = self.upgrades.clone();
|
||||
let _close_on_drop = upgrades.close_on_drop();
|
||||
let draining = upgrades.clone();
|
||||
let listener = listener.tap_io(|stream| {
|
||||
if let Err(error) = stream.set_nodelay(true) {
|
||||
tracing::warn!(%error, "could not disable Nagle on a client connection");
|
||||
}
|
||||
});
|
||||
let result = axum::serve(
|
||||
listener,
|
||||
self.router()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue