From 40be7fa1a8f884c311d44054733f5319f61a7f87 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Fri, 6 Nov 2026 04:03:34 +0900 Subject: [PATCH 01/40] feat(gateway): forward HTTP requests to the Python proxy (#49604) * feat(gateway): forward HTTP requests to the Python proxy Co-Authored-By: Claude Opus 5.5 * fix(gateway): preserve required forwarding headers and separate upgrades * test(gateway): clarify hash compatibility and relocate forwarding docs * test(gateway): define URL fragment forwarding behavior --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../.agents/skills/rust-gateway/SKILL.md | 2 +- litellm-rust/Cargo.lock | 59 +- litellm-rust/Cargo.toml | 1 + litellm-rust/crates/gateway-forward/AGENTS.md | 37 + .../crates/gateway-forward/Cargo.toml | 30 + .../crates/gateway-forward/FORWARDING.md | 31 + .../crates/gateway-forward/src/connections.rs | 58 ++ .../crates/gateway-forward/src/connector.rs | 122 +++ .../crates/gateway-forward/src/error.rs | 91 ++ .../crates/gateway-forward/src/forward.rs | 169 ++++ .../crates/gateway-forward/src/lib.rs | 29 + .../crates/gateway-forward/src/rewrite.rs | 161 ++++ .../crates/gateway-forward/src/server.rs | 116 +++ .../crates/gateway-forward/src/settings.rs | 113 +++ .../crates/gateway-forward/src/websocket.rs | 86 ++ .../gateway-forward/tests/client_identity.rs | 223 +++++ .../crates/gateway-forward/tests/download.rs | 192 ++++ .../crates/gateway-forward/tests/exchange.rs | 855 ++++++++++++++++++ .../crates/gateway-forward/tests/framing.rs | 259 ++++++ .../gateway-forward/tests/idle_stream.rs | 102 +++ .../crates/gateway-forward/tests/lifecycle.rs | 473 ++++++++++ .../crates/gateway-forward/tests/roundtrip.rs | 360 ++++++++ .../crates/gateway-forward/tests/settings.rs | 110 +++ .../crates/gateway-forward/tests/slow_head.rs | 41 + .../crates/gateway-forward/tests/stream.rs | 313 +++++++ .../gateway-forward/tests/support/mod.rs | 311 +++++++ .../crates/gateway-forward/tests/upload.rs | 339 +++++++ .../crates/gateway-forward/tests/upstream.rs | 21 + .../crates/gateway-forward/tests/websocket.rs | 719 +++++++++++++++ litellm-rust/crates/gateway/Cargo.toml | 1 + litellm-rust/crates/gateway/src/main.rs | 26 +- litellm-rust/crates/gateway/src/settings.rs | 22 +- 32 files changed, 5448 insertions(+), 24 deletions(-) create mode 100644 litellm-rust/crates/gateway-forward/AGENTS.md create mode 100644 litellm-rust/crates/gateway-forward/Cargo.toml create mode 100644 litellm-rust/crates/gateway-forward/FORWARDING.md create mode 100644 litellm-rust/crates/gateway-forward/src/connections.rs create mode 100644 litellm-rust/crates/gateway-forward/src/connector.rs create mode 100644 litellm-rust/crates/gateway-forward/src/error.rs create mode 100644 litellm-rust/crates/gateway-forward/src/forward.rs create mode 100644 litellm-rust/crates/gateway-forward/src/lib.rs create mode 100644 litellm-rust/crates/gateway-forward/src/rewrite.rs create mode 100644 litellm-rust/crates/gateway-forward/src/server.rs create mode 100644 litellm-rust/crates/gateway-forward/src/settings.rs create mode 100644 litellm-rust/crates/gateway-forward/src/websocket.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/client_identity.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/download.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/exchange.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/framing.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/idle_stream.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/lifecycle.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/roundtrip.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/settings.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/slow_head.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/stream.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/support/mod.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/upload.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/upstream.rs create mode 100644 litellm-rust/crates/gateway-forward/tests/websocket.rs diff --git a/litellm-rust/.agents/skills/rust-gateway/SKILL.md b/litellm-rust/.agents/skills/rust-gateway/SKILL.md index 4fee0b54a68..923c9bf5ab6 100644 --- a/litellm-rust/.agents/skills/rust-gateway/SKILL.md +++ b/litellm-rust/.agents/skills/rust-gateway/SKILL.md @@ -7,7 +7,7 @@ metadata: # Rust gateway -`crates/gateway` mounts endpoint routers and serves the listener. `crates/gateway-inference`, `gateway-mcp` and `gateway-ui` own their endpoints, `gateway-auth` is the inbound authentication middleware and `gateway-management` holds management logic that has no router yet. A crate's `AGENTS.md`, where it has one, says what belongs in it. Read the reference for the part you are touching +`crates/gateway` mounts endpoint routers and serves the listener. `crates/gateway-inference`, `gateway-mcp` and `gateway-ui` own their endpoints, `gateway-auth` is the inbound authentication middleware and `gateway-management` holds management logic that has no router yet. `gateway-forward` is the reverse proxy that relays every request to the Python proxy and has no routes of its own. A crate's `AGENTS.md`, where it has one, says what belongs in it. Read the reference for the part you are touching - [surfaces](references/surfaces.md) - Each API surface mirrors the envelope the Python proxy returns there, so the shapes differ on purpose diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 23ebfc7d3cf..107b55e89cb 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -547,7 +547,7 @@ dependencies = [ "http 1.4.2", "http-body 0.4.6", "hyper 0.14.32", - "hyper 1.10.1", + "hyper 1.12.0", "hyper-rustls 0.24.2", "hyper-rustls 0.27.9", "hyper-util", @@ -736,7 +736,7 @@ dependencies = [ "http 1.4.2", "http-body 1.1.0", "http-body-util", - "hyper 1.10.1", + "hyper 1.12.0", "hyper-util", "itoa", "matchit", @@ -1000,7 +1000,7 @@ dependencies = [ "home", "http 1.4.2", "http-body-util", - "hyper 1.10.1", + "hyper 1.12.0", "hyper-named-pipe", "hyper-rustls 0.27.9", "hyper-util", @@ -2255,7 +2255,7 @@ dependencies = [ "chrono", "http 1.4.2", "http-body-util", - "hyper 1.10.1", + "hyper 1.12.0", "hyper-rustls 0.27.9", "hyper-util", "ring", @@ -2391,7 +2391,7 @@ dependencies = [ "http 1.4.2", "http-body 1.1.0", "http-body-util", - "hyper 1.10.1", + "hyper 1.12.0", "lazy_static", "opentelemetry", "opentelemetry-semantic-conventions", @@ -2833,9 +2833,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.10.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" dependencies = [ "atomic-waker", "bytes", @@ -2860,7 +2860,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fab3637d6b04a8037af8a266fdf6cf92ea957e8c53981a2bf6136572531025bf" dependencies = [ "hex", - "hyper 1.10.1", + "hyper 1.12.0", "hyper-util", "pin-project-lite", "tokio", @@ -2889,7 +2889,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ "http 1.4.2", - "hyper 1.10.1", + "hyper 1.12.0", "hyper-util", "rustls 0.23.42", "rustls-native-certs", @@ -2904,7 +2904,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.10.1", + "hyper 1.12.0", "hyper-util", "pin-project-lite", "tokio", @@ -2923,7 +2923,7 @@ dependencies = [ "futures-util", "http 1.4.2", "http-body 1.1.0", - "hyper 1.10.1", + "hyper 1.12.0", "ipnet", "libc", "percent-encoding", @@ -2942,7 +2942,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper 1.10.1", + "hyper 1.12.0", "hyper-util", "pin-project-lite", "tokio", @@ -3769,6 +3769,7 @@ dependencies = [ "litellm-config", "litellm-error", "litellm-gateway-auth", + "litellm-gateway-forward", "litellm-gateway-inference", "litellm-gateway-mcp", "litellm-gateway-ui", @@ -3815,6 +3816,29 @@ dependencies = [ "veil", ] +[[package]] +name = "litellm-gateway-forward" +version = "0.1.0" +dependencies = [ + "axum", + "bytes", + "envy", + "futures-util", + "http-body-util", + "httparse", + "hyper 1.12.0", + "hyper-util", + "proptest", + "rstest", + "serde", + "thiserror 2.0.19", + "tokio", + "tokio-tungstenite", + "tokio-util", + "tower-service", + "tracing", +] + [[package]] name = "litellm-gateway-inference" version = "0.1.0" @@ -6031,7 +6055,7 @@ dependencies = [ "http 1.4.2", "http-body 1.1.0", "http-body-util", - "hyper 1.10.1", + "hyper 1.12.0", "hyper-rustls 0.27.9", "hyper-util", "js-sys", @@ -7512,6 +7536,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", "pin-project-lite", "tokio", ] @@ -7591,7 +7616,7 @@ dependencies = [ "http 1.4.2", "http-body 1.1.0", "http-body-util", - "hyper 1.10.1", + "hyper 1.12.0", "hyper-timeout", "hyper-util", "percent-encoding", @@ -8167,9 +8192,9 @@ dependencies = [ [[package]] name = "want" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" dependencies = [ "try-lock", ] @@ -8551,7 +8576,7 @@ dependencies = [ "futures", "http 1.4.2", "http-body-util", - "hyper 1.10.1", + "hyper 1.12.0", "hyper-util", "log", "once_cell", diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 71914a0bffe..7121df06bc0 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -26,6 +26,7 @@ litellm-inference-ocr = { path = "crates/inference-ocr" } litellm-inference-testing = { path = "crates/inference-testing" } litellm-gateway-mcp = { path = "crates/gateway-mcp" } litellm-gateway = { path = "crates/gateway" } +litellm-gateway-forward = { path = "crates/gateway-forward" } litellm-gateway-inference = { path = "crates/gateway-inference" } litellm-gateway-auth = { path = "crates/gateway-auth" } litellm-gateway-management = { path = "crates/gateway-management" } diff --git a/litellm-rust/crates/gateway-forward/AGENTS.md b/litellm-rust/crates/gateway-forward/AGENTS.md new file mode 100644 index 00000000000..2135980266f --- /dev/null +++ b/litellm-rust/crates/gateway-forward/AGENTS.md @@ -0,0 +1,37 @@ +# Forwarding gateway + +`Forward` is a reverse proxy in front of the Python proxy on loopback. Python must see each request as if the client had connected to it directly, because every Python route (inference, MCP, pass-through, UI, WebSocket) is served behind it without Rust knowing which one it is + +- Serve HTTP/1.1 only, as uvicorn does. The supervisor never puts Rust in front of a Python server that speaks HTTP/2 +- Never branch on a route or path. A behavior one endpoint needs is a protocol behavior every endpoint gets +- Forward the method, request target, headers, and body byte for byte within the [URL contract](FORWARDING.md#url-fragments-and-literal-hashes). An unescaped `#` starts a fragment and is omitted with everything after it, while `%23` is preserved. Only drop hop-by-hop headers (RFC 9110 section 7.6.1) and set `x-litellm-rust-peer`, which replaces any client copy +- Stream both bodies. Never buffer, cap, or time out a body, since uploads and provider streams have no upper bound +- A client disconnect closes the upstream connection at every stage, because that is how Python learns to stop a provider stream +- An upstream that dies mid-response fails the client body. Never end it cleanly, or SSE clients treat a truncated stream as complete +- Never invent a Host. Python builds absolute URLs from it, so a missing one must never become the private upstream address +- A request Python could not have received unchanged (HTTP/1.1 without Host, a transfer coding other than chunked, a target without a path) gets the status uvicorn would have given, from Rust +- Pooled upstream connections are retired well before Python's keep-alive timeout, so a request never lands on a connection Python is closing +- Idle client connections close after Python's keep-alive timeout, as uvicorn closed them, so a client that vanishes cannot hold a socket forever. Only the wait for the next request is timed, never an exchange in progress +- WebSocket is the one protocol switch Python serves. Upgrades are relayed as raw bytes after the 101 and drained on shutdown + +## Layout + +Every forwarding decision has one home. `rewrite.rs` owns what Python and the client receive, as pure functions over requests and responses. `Rejection` in `error.rs` is the only place that picks a status for a request Rust answers itself. `settings.rs` parses the environment into typed values, so an `Upstream` is a loopback origin by construction. `server.rs` accepts client connections and owns their timeouts, since `axum::serve` exposes none, and `connections.rs` tracks every connection and relay for shutdown. `forward.rs` and `websocket.rs` only move requests, bodies and upgraded connections and never touch a header themselves. `connector.rs` keeps hyper reading Python's answer after Python stops reading the request + +## Environment + +`Settings` reads only these variables, and only when `LITELLM_FORWARD_UPSTREAM` is set. The Python supervisor in `litellm/proxy/rust_forward/` sets every one of them from the values Python actually runs with, so all of them are required and a missing or invalid one fails startup naming it. The crate never reads Python's own settings (`HOST`, `PORT`, `KEEPALIVE_TIMEOUT`, `GRACEFUL_SHUTDOWN_TIMEOUT`) and keeps no copy of Python's defaults, since Python can get those values from CLI flags the environment never shows + +| Variable | Meaning | +|---|---| +| `LITELLM_FORWARD_UPSTREAM` | Python's loopback origin, e.g. `http://127.0.0.1:53211` | +| `LITELLM_FORWARD_HOST` | public bind host | +| `LITELLM_FORWARD_PORT` | public bind port | +| `LITELLM_FORWARD_UPSTREAM_KEEPALIVE` | seconds uvicorn keeps an idle connection open. Pooled connections are retired at half of it, and idle client connections are closed after it | +| `LITELLM_FORWARD_SHUTDOWN_DRAIN` | seconds to drain on shutdown, already including Python's graceful timeout and teardown headroom | + +## Tests + +Test through real sockets with `tests/support`, never `oneshot`. `Gateway` runs the forwarder on a listener like production, `recording` is a well-behaved upstream that captures what Python would receive, and `raw_upstream` and `Raw` drive bytes by hand for framing, timing and disconnects a real server would not produce. Waits measured in minutes use paused tokio time instead of real sleeps + +The forwarder sees HTTP/1.1 messages, not routes, so the tests are split into one file per traffic shape uvicorn can emit: `exchange` (plain request and response), `framing` (requests whose framing a parser could read two ways), `slow_head` (minutes before response headers), `stream` (incremental response bodies), `idle_stream` (long-lived streams that go quiet), `upload`, `download`, `websocket`, `client_identity`, `lifecycle` (disconnects, pooling, idle timeouts, shutdown) and `roundtrip` (the forwarding invariant for generated messages). A new route never gets its own test. Add its behavior to the shape it already has, and when it needs a wire behavior none of the files covers, add a new shape file for it diff --git a/litellm-rust/crates/gateway-forward/Cargo.toml b/litellm-rust/crates/gateway-forward/Cargo.toml new file mode 100644 index 00000000000..2fef19dca34 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "litellm-gateway-forward" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +axum.workspace = true +envy = "0.4.2" +hyper = { version = "1.12.0", features = ["http1", "server"] } +hyper-util = { workspace = true, features = ["client-legacy", "http1", "tokio"] } +serde.workspace = true +thiserror.workspace = true +tokio = { workspace = true, features = ["io-util"] } +tokio-util = { version = "0.7", features = ["rt"] } +tower-service = "0.3.3" +tracing.workspace = true + +[dev-dependencies] +bytes.workspace = true +futures-util.workspace = true +http-body-util = "0.1" +httparse = "1" +hyper = { version = "1.12.0", features = ["client"] } +proptest.workspace = true +rstest.workspace = true +tokio = { workspace = true, features = ["io-util", "sync", "test-util"] } +tokio-tungstenite.workspace = true diff --git a/litellm-rust/crates/gateway-forward/FORWARDING.md b/litellm-rust/crates/gateway-forward/FORWARDING.md new file mode 100644 index 00000000000..532f341aba5 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/FORWARDING.md @@ -0,0 +1,31 @@ +# Python forwarding compatibility + +The opt-in forwarding gateway serves HTTP/1.1 and WebSockets in front of Python. Python still owns every route and policy. Its protocol contract and environment settings are in [AGENTS.md](AGENTS.md) + +## URL fragments and literal hashes + +The gateway follows Hyper's `http::Uri` parsing. If a raw client sends an unescaped `#` in the request target, only the path and query before the first `#` reach Python. For example, `/v1/files?purpose=batch#retry&limit=1` becomes `/v1/files?purpose=batch`, and `/gemini/v1beta/files/a#b?key=k` becomes `/gemini/v1beta/files/a` + +This is accepted behavior because `#` starts a URL fragment, which is separated before dereferencing ([RFC 3986 section 3.5](https://www.rfc-editor.org/rfc/rfc3986.html#section-3.5)). HTTP origin-form contains only the path and optional query, not a fragment ([RFC 9112 section 3.2.1](https://www.rfc-editor.org/rfc/rfc9112.html#section-3.2.1)). Browsers and URL-based clients omit the fragment before sending the request, so UI hash routing is unaffected + +A literal hash in path or query data must be encoded as `%23`, which the gateway forwards unchanged. Direct Python with h11 also accepts unescaped hashes as data, but that permissive raw-target behavior is outside this URL contract. Active socket tests cover both [fragment omission](tests/exchange.rs) and [encoded-hash preservation](tests/exchange.rs) + +## Known differences from uvicorn with h11 + +These cases remain executable as ignored socket tests. They are not passing parity checks + +| Input | Direct Python | Through Rust | Regression test | +|---|---|---|---| +| Header after a removed hop-by-hop field | Keeps the remaining header order | `HeaderMap::remove` can move a later header | [dropping_a_hop_by_hop_header_keeps_the_others_in_order](tests/exchange.rs) | +| Repeated header names interleaved with other names | Keeps the interleaving | Groups values by name, keeping their order within that name | [interleaved_repeated_headers_reach_python_interleaved](tests/exchange.rs) | +| Obsolete folded header, such as `x-a: b\r\n c` | Unfolds to `b c` | Rejects with 400 | [a_folded_header_reaches_python_unfolded](tests/exchange.rs) | +| Identical Content-Length list, such as `7, 7` | Accepts length 7 | Rejects with 400 | [a_content_length_list_of_one_value_reaches_python](tests/framing.rs) | +| HTTP/1.0 request with chunked transfer coding | Decodes the body | Rejects with 400 | [an_http10_chunked_upload_reaches_python](tests/framing.rs) | + +Clients can avoid the parser rejections by sending unfolded headers, one Content-Length value, and HTTP/1.1 for chunked uploads. Integrations must not depend on header ordering across different field names + +Run `cargo test -p litellm-gateway-forward --test exchange --test framing -- --ignored` from this directory to reproduce the differences + +## Connection header handling + +The gateway strips standard hop-by-hop fields and fields named by Connection, except routing, representation, credential, forwarding, and WebSocket handshake headers. A client cannot nominate those end-to-end fields for removal. The private `x-litellm-rust-peer` always replaces any client-supplied copy diff --git a/litellm-rust/crates/gateway-forward/src/connections.rs b/litellm-rust/crates/gateway-forward/src/connections.rs new file mode 100644 index 00000000000..10a4867fa51 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/connections.rs @@ -0,0 +1,58 @@ +use tokio_util::{ + sync::{CancellationToken, DropGuard}, + task::TaskTracker, +}; + +/// Every client connection and WebSocket relay the gateway runs, so shutdown can wait for them +/// and then close whatever outlives the drain. +#[derive(Clone, Default)] +pub(crate) struct Connections { + tasks: TaskTracker, + draining: CancellationToken, + closed: CancellationToken, +} + +impl Connections { + /// Runs `work` until it finishes or the gateway closes every connection. + pub fn spawn(&self, work: impl Future + Send + 'static) { + let closed = self.closed.clone(); + self.tasks.spawn(async move { + tokio::select! { + () = closed.cancelled() => {} + () = work => {} + } + }); + } + + pub fn begin_shutdown(&self) { + self.draining.cancel(); + } + + pub async fn draining(&self) { + self.draining.cancelled().await; + } + + pub fn is_draining(&self) -> bool { + self.draining.is_cancelled() + } + + pub async fn until_closed(&self, future: impl Future) -> Option { + tokio::select! { + () = self.closed.cancelled() => None, + output = future => Some(output), + } + } + + pub fn close(&self) { + self.closed.cancel(); + } + + pub fn close_on_drop(&self) -> DropGuard { + self.closed.clone().drop_guard() + } + + pub async fn wait(&self) { + self.tasks.close(); + self.tasks.wait().await; + } +} diff --git a/litellm-rust/crates/gateway-forward/src/connector.rs b/litellm-rust/crates/gateway-forward/src/connector.rs new file mode 100644 index 00000000000..5eb998db0c0 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/connector.rs @@ -0,0 +1,122 @@ +use std::{ + io::{self, IoSlice}, + pin::Pin, + task::{Context, Poll, ready}, +}; + +use axum::http::Uri; +use hyper::rt::{Read, ReadBufCursor, Write}; +use hyper_util::{ + client::legacy::connect::{Connected, Connection, HttpConnector}, + rt::TokioIo, +}; +use tokio::net::TcpStream; + +/// Connects to Python like `HttpConnector`, but keeps hyper reading Python's answer after Python +/// stops reading the request. uvicorn rejects an upload (413, 401) as soon as it reads the head +/// and closes with the body unread, so writing the rest fails, and hyper would fail the request +/// and drop the answer it already received. Like curl and httpcore, the forwarder discards the +/// rest of the request instead and reads on. A connection that never answered still fails, on the +/// read, and no request is ever sent twice. +#[derive(Clone)] +pub(crate) struct Connector(pub HttpConnector); + +impl tower_service::Service for Connector { + type Response = AnswerAfterRefusal; + type Error = >::Error; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, context: &mut Context<'_>) -> Poll> { + self.0.poll_ready(context) + } + + fn call(&mut self, uri: Uri) -> Self::Future { + let connecting = self.0.call(uri); + Box::pin(async move { + Ok(AnswerAfterRefusal { + stream: connecting.await?, + refused: false, + }) + }) + } +} + +pub(crate) struct AnswerAfterRefusal { + stream: TokioIo, + refused: bool, +} + +impl AnswerAfterRefusal { + fn written(&mut self, written: io::Result, length: usize) -> io::Result { + match written { + Err(error) + if matches!( + error.kind(), + io::ErrorKind::BrokenPipe | io::ErrorKind::ConnectionReset + ) => + { + self.refused = true; + Ok(length) + } + written => written, + } + } +} + +impl Read for AnswerAfterRefusal { + fn poll_read( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffer: ReadBufCursor<'_>, + ) -> Poll> { + Pin::new(&mut self.stream).poll_read(context, buffer) + } +} + +impl Write for AnswerAfterRefusal { + fn poll_write( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffer: &[u8], + ) -> Poll> { + if self.refused { + return Poll::Ready(Ok(buffer.len())); + } + let written = ready!(Pin::new(&mut self.stream).poll_write(context, buffer)); + Poll::Ready(self.written(written, buffer.len())) + } + + fn poll_write_vectored( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffers: &[IoSlice<'_>], + ) -> Poll> { + let length = buffers.iter().map(|buffer| buffer.len()).sum(); + if self.refused { + return Poll::Ready(Ok(length)); + } + let written = ready!(Pin::new(&mut self.stream).poll_write_vectored(context, buffers)); + Poll::Ready(self.written(written, length)) + } + + fn is_write_vectored(&self) -> bool { + self.stream.is_write_vectored() + } + + fn poll_flush(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { + if self.refused { + return Poll::Ready(Ok(())); + } + Pin::new(&mut self.stream).poll_flush(context) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.stream).poll_shutdown(context) + } +} + +impl Connection for AnswerAfterRefusal { + fn connected(&self) -> Connected { + self.stream.connected() + } +} diff --git a/litellm-rust/crates/gateway-forward/src/error.rs b/litellm-rust/crates/gateway-forward/src/error.rs new file mode 100644 index 00000000000..7dbdde68a85 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/error.rs @@ -0,0 +1,91 @@ +use axum::{ + http::StatusCode, + response::{IntoResponse, Response}, +}; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("invalid forwarding upstream URI")] + UpstreamUri(#[source] axum::http::uri::InvalidUri), + #[error( + "LITELLM_FORWARD_UPSTREAM must be a loopback IPv4 http origin without a path, query or credentials" + )] + Upstream, + #[error("{0} must be set in forward mode")] + MissingSetting(String), + #[error("invalid forwarding setting: {0}")] + InvalidSetting(String), + #[error("forwarding gateway I/O failed: {0}")] + Io(#[from] std::io::Error), +} + +/// Why a request got an answer from Rust instead of from Python. +#[derive(Debug, thiserror::Error)] +pub(crate) enum Rejection { + #[error("HTTP/1.1 request without a Host header")] + MissingHost, + #[error("request target has no path to forward")] + NoPath, + #[error("request uses a transfer coding other than chunked")] + UnsupportedTransferCoding, + #[error("client request body failed: {0}")] + ClientBody(#[source] hyper_util::client::legacy::Error), + #[error("python proxy unreachable: {0}")] + Unreachable(#[source] hyper_util::client::legacy::Error), + #[error("python proxy switched protocols without being asked to")] + UnrequestedUpgrade, + #[error("python proxy websocket upgrade failed: {0}")] + UpgradeFailed(#[source] hyper::Error), + #[error("forwarding gateway is shutting down")] + ShuttingDown, + #[error("could not build the request for the python proxy")] + Internal, +} + +impl Rejection { + /// A client body that ends early or breaks its framing fails the upstream request too, but + /// the fault is the client's, not Python's. + pub fn from_upstream(error: hyper_util::client::legacy::Error) -> Self { + let client_body = std::error::Error::source(&error) + .and_then(|source| source.downcast_ref::()) + .is_some_and(hyper::Error::is_user); + match client_body { + true => Self::ClientBody(error), + false => Self::Unreachable(error), + } + } + + fn status(&self) -> StatusCode { + match self { + Self::MissingHost + | Self::NoPath + | Self::UnsupportedTransferCoding + | Self::ClientBody(_) => StatusCode::BAD_REQUEST, + Self::Unreachable(_) | Self::UnrequestedUpgrade | Self::UpgradeFailed(_) => { + StatusCode::BAD_GATEWAY + } + Self::ShuttingDown => StatusCode::SERVICE_UNAVAILABLE, + Self::Internal => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn is_gateway_fault(&self) -> bool { + matches!( + self, + Self::Unreachable(_) + | Self::UnrequestedUpgrade + | Self::UpgradeFailed(_) + | Self::Internal + ) + } +} + +impl IntoResponse for Rejection { + fn into_response(self) -> Response { + match self.is_gateway_fault() { + true => tracing::warn!(error = %self, "request not forwarded"), + false => tracing::debug!(error = %self, "request not forwarded"), + } + self.status().into_response() + } +} diff --git a/litellm-rust/crates/gateway-forward/src/forward.rs b/litellm-rust/crates/gateway-forward/src/forward.rs new file mode 100644 index 00000000000..69fb63ada7d --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/forward.rs @@ -0,0 +1,169 @@ +use std::{ + pin::Pin, + task::{Context, Poll}, + time::Duration, +}; + +use axum::{ + body::Body, + extract::Request, + http::{StatusCode, Uri}, + response::{IntoResponse, Response}, +}; +use hyper::body::{Frame, Incoming, SizeHint}; +use hyper_util::{ + client::legacy::{Client, connect::HttpConnector}, + rt::{TokioExecutor, TokioTimer}, +}; +use tokio::net::TcpListener; + +use crate::{ + Rejection, Upstream, connections::Connections, connector::Connector, rewrite, server, + server::ClientSocket, websocket, +}; + +// h11 counts no headers and only caps a head at 16 KiB, so it accepts as many headers as fit in +// that many shortest lines (`a:\r\n`). Python copies every provider header into its response +// too, so hyper's default of 100 would turn ordinary traffic into 431s and 502s. +pub(crate) const MAX_HEADERS: usize = 16 * 1024 / b"a:\r\n".len(); + +#[derive(Clone)] +pub struct Forward { + upstream: Uri, + client: Client, + connections: Connections, + keepalive: Duration, +} + +impl Forward { + /// `upstream_keepalive` is how long the upstream keeps an idle connection open. Pooled + /// connections are dropped well before that, so a request never lands on one it is closing, + /// and idle client connections are closed after it, as uvicorn did. + pub fn new(upstream: &Upstream, upstream_keepalive: Duration) -> Self { + let mut connector = HttpConnector::new(); + connector.set_nodelay(true); + let client = Client::builder(TokioExecutor::new()) + .http1_max_headers(MAX_HEADERS) + .pool_timer(TokioTimer::new()) + .pool_idle_timeout(upstream_keepalive / 2) + .set_host(false) + .build(Connector(connector)); + Self { + upstream: upstream.uri().clone(), + client, + connections: Connections::default(), + keepalive: upstream_keepalive, + } + } + + /// Serves until `shutdown` resolves, then lets open requests and WebSockets finish for up to + /// `drain` before closing whatever is left. Python shuts down alongside and ends long streams + /// itself, so `drain` only backstops a Python that never does. + pub async fn serve( + self, + listener: TcpListener, + shutdown: impl Future, + drain: Duration, + ) { + server::serve(self, listener, shutdown, drain).await; + } + + pub(crate) fn connections(&self) -> &Connections { + &self.connections + } + + pub(crate) fn keepalive(&self) -> Duration { + self.keepalive + } + + pub(crate) async fn handle(&self, client: ClientSocket, request: Request) -> Response { + self.forward(client, request) + .await + .unwrap_or_else(IntoResponse::into_response) + } + + async fn forward( + &self, + client: ClientSocket, + mut request: Request, + ) -> Result { + if let Some(downstream) = websocket::take_handshake(&self.connections, &mut request)? { + return self.forward_websocket(client, request, downstream).await; + } + let request = rewrite::upstream_request(request, &self.upstream, client, false)?; + let response = self + .client + .request(request) + .await + .map_err(Rejection::from_upstream)?; + http_response(response) + } + + async fn forward_websocket( + &self, + client: ClientSocket, + request: Request, + downstream: hyper::upgrade::OnUpgrade, + ) -> Result { + let request = rewrite::upstream_request(request, &self.upstream, client, true)?; + let response = self + .connections + .until_closed(self.client.request(request)) + .await + .ok_or(Rejection::ShuttingDown)? + .map_err(Rejection::from_upstream)?; + if response.status() == StatusCode::SWITCHING_PROTOCOLS { + return websocket::switch(&self.connections, downstream, response).await; + } + http_response(response) + } +} + +fn http_response(response: Response) -> Result { + if response.status() == StatusCode::SWITCHING_PROTOCOLS { + return Err(Rejection::UnrequestedUpgrade); + } + Ok( + rewrite::downstream_response(response) + .map(|body| Body::new(Relayed { body, failed: None })), + ) +} + +/// Python's response body on its way to the client. +struct Relayed { + body: Incoming, + failed: Option, +} + +impl hyper::body::Body for Relayed { + type Data = ::Data; + type Error = hyper::Error; + + // hyper aborts a client connection whose response body fails without flushing the head it + // buffered, so a head Python sent right before dying would never arrive and SDKs would retry + // a request Python already ran. Holding the error for one poll lets hyper flush the head. + fn poll_frame( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + ) -> Poll, Self::Error>>> { + if let Some(error) = self.failed.take() { + return Poll::Ready(Some(Err(error))); + } + match Pin::new(&mut self.body).poll_frame(context) { + Poll::Ready(Some(Err(error))) => { + self.failed = Some(error); + context.waker().wake_by_ref(); + Poll::Pending + } + frame => frame, + } + } + + fn is_end_stream(&self) -> bool { + self.failed.is_none() && self.body.is_end_stream() + } + + fn size_hint(&self) -> SizeHint { + self.body.size_hint() + } +} diff --git a/litellm-rust/crates/gateway-forward/src/lib.rs b/litellm-rust/crates/gateway-forward/src/lib.rs new file mode 100644 index 00000000000..4ac17b50544 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/lib.rs @@ -0,0 +1,29 @@ +mod connections; +mod connector; +mod error; +mod forward; +mod rewrite; +mod server; +mod settings; +mod websocket; + +pub use error::Error; +pub(crate) use error::Rejection; +pub use forward::Forward; +pub use rewrite::PEER_HEADER; +pub use settings::{Settings, Upstream}; + +/// Binds the public listener and forwards to Python until `shutdown` resolves, then drains for +/// `settings.shutdown_drain`. +pub async fn run( + settings: Settings, + shutdown: impl Future + Send + 'static, +) -> Result<(), Error> { + let forward = Forward::new(&settings.upstream, settings.upstream_keepalive); + let listener = tokio::net::TcpListener::bind((settings.host.as_str(), settings.port)).await?; + tracing::info!(address = %listener.local_addr()?, "forwarding gateway listening"); + forward + .serve(listener, shutdown, settings.shutdown_drain) + .await; + Ok(()) +} diff --git a/litellm-rust/crates/gateway-forward/src/rewrite.rs b/litellm-rust/crates/gateway-forward/src/rewrite.rs new file mode 100644 index 00000000000..fa22bf19037 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/rewrite.rs @@ -0,0 +1,161 @@ +use axum::{ + body::HttpBody, + extract::Request, + http::{HeaderMap, HeaderValue, Response, Uri, Version, header}, +}; + +use crate::{Rejection, server::ClientSocket}; + +pub const PEER_HEADER: &str = "x-litellm-rust-peer"; + +/// The request Python receives: the client's method, target, headers and body unchanged, sent to +/// `upstream` over HTTP/1.1 without the client connection's own headers, plus the client's TCP +/// peer in `PEER_HEADER`, which replaces any copy the client sent. +pub(crate) fn upstream_request( + mut request: Request, + upstream: &Uri, + client: ClientSocket, + websocket: bool, +) -> Result { + if !only_chunked(request.headers()) { + return Err(Rejection::UnsupportedTransferCoding); + } + let path_and_query = request + .uri() + .path_and_query() + .cloned() + .ok_or(Rejection::NoPath)?; + if let Some(host) = public_host(&request, client)? { + request.headers_mut().insert(header::HOST, host); + } + let mut uri = upstream.clone().into_parts(); + uri.path_and_query = Some(path_and_query); + *request.uri_mut() = Uri::from_parts(uri).map_err(|_| Rejection::Internal)?; + *request.version_mut() = Version::HTTP_11; + let chunked = request.body().size_hint().exact().is_none(); + let headers = request.headers_mut(); + strip_hop_by_hop(headers); + if chunked { + // hyper sends no body for a GET with an unknown length unless told it is chunked. + headers.insert( + header::TRANSFER_ENCODING, + HeaderValue::from_static("chunked"), + ); + } + if websocket { + restore_websocket_handshake(headers); + } + headers.insert( + PEER_HEADER, + HeaderValue::from_str(&client.peer.to_string()).map_err(|_| Rejection::Internal)?, + ); + Ok(request) +} + +// Python builds absolute URLs from Host and, without a usable one, from the address it listens +// on, which behind Rust is the private loopback port. A client that sent no usable Host gets the +// public address it connected to, which is what Python used before. h11 rejects HTTP/1.1 without +// Host. +fn public_host(request: &Request, client: ClientSocket) -> Result, Rejection> { + let host = request.headers().get(header::HOST); + if host.is_some_and(|host| !host.is_empty()) { + return Ok(None); + } + if host.is_none() && request.version() == Version::HTTP_11 { + return Err(Rejection::MissingHost); + } + HeaderValue::from_str(&client.local.to_string()) + .map(Some) + .map_err(|_| Rejection::Internal) +} + +// hyper only decodes chunked framing, so anything else would reach Python still coded but framed +// as a plain body. uvicorn rejects it with the 400 it sends for every h11 parse error. +fn only_chunked(headers: &HeaderMap) -> bool { + headers + .get_all(header::TRANSFER_ENCODING) + .iter() + .flat_map(|value| value.as_bytes().split(|byte| *byte == b',')) + .all(|coding| coding.trim_ascii().eq_ignore_ascii_case(b"chunked")) +} + +/// The response the client receives: Python's, without the upstream connection's own headers. +pub(crate) fn downstream_response(mut response: Response) -> Response { + strip_hop_by_hop(response.headers_mut()); + response +} + +/// Python's 101 as the client receives it, still naming the protocol both sides switch to. +pub(crate) fn switched_response(mut response: Response) -> Response { + strip_hop_by_hop(response.headers_mut()); + restore_websocket_handshake(response.headers_mut()); + response +} + +pub(crate) fn is_websocket_handshake(headers: &HeaderMap) -> bool { + headers + .get(header::UPGRADE) + .is_some_and(|value| value.as_bytes().eq_ignore_ascii_case(b"websocket")) + && headers + .get_all(header::CONNECTION) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .any(|token| token.trim().eq_ignore_ascii_case("upgrade")) +} + +const HOP_BY_HOP: [&str; 9] = [ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "transfer-encoding", + "upgrade", + "proxy-connection", +]; + +// Per-connection headers (RFC 9110 section 7.6.1) describe the client's connection to Rust, not +// Rust's connection to Python, so a proxy must drop them instead of forwarding them. +fn strip_hop_by_hop(headers: &mut HeaderMap) { + let named: Vec<_> = headers + .get_all(header::CONNECTION) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .filter_map(|name| header::HeaderName::from_bytes(name.trim().as_bytes()).ok()) + .filter(|name| !must_forward(name)) + .collect(); + for name in named { + headers.remove(name); + } + for name in HOP_BY_HOP { + headers.remove(name); + } +} + +fn must_forward(name: &header::HeaderName) -> bool { + matches!( + name.as_str(), + "host" + | "authorization" + | "cookie" + | "set-cookie" + | "www-authenticate" + | "authentication-info" + | "api-key" + | "x-api-key" + | "litellm-api-key" + | "forwarded" + | PEER_HEADER + ) || name.as_str().starts_with("content-") + || name.as_str().starts_with("x-forwarded-") + || name.as_str().starts_with("sec-websocket-") +} + +// Stripping drops the handshake's Connection and Upgrade, so each hop gets them back. +fn restore_websocket_handshake(headers: &mut HeaderMap) { + headers.insert(header::CONNECTION, HeaderValue::from_static("upgrade")); + headers.insert(header::UPGRADE, HeaderValue::from_static("websocket")); +} diff --git a/litellm-rust/crates/gateway-forward/src/server.rs b/litellm-rust/crates/gateway-forward/src/server.rs new file mode 100644 index 00000000000..0388642446a --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/server.rs @@ -0,0 +1,116 @@ +use std::{convert::Infallible, io, net::SocketAddr, pin::pin, time::Duration}; + +use axum::{body::Body, extract::Request}; +use hyper::{body::Incoming, server::conn::http1, service::service_fn}; +use hyper_util::rt::{TokioIo, TokioTimer}; +use tokio::net::TcpListener; + +use crate::{Forward, forward::MAX_HEADERS}; + +/// The client's TCP connection to the public listener. +#[derive(Clone, Copy, Debug)] +pub(crate) struct ClientSocket { + pub peer: SocketAddr, + pub local: SocketAddr, +} + +/// Serves each public connection with hyper directly, because `axum::serve` cannot configure +/// the connection timeouts uvicorn applied to the same clients. +pub(crate) async fn serve( + forward: Forward, + listener: TcpListener, + shutdown: impl Future, + drain: Duration, +) { + let connections = forward.connections().clone(); + let _close_on_drop = connections.close_on_drop(); + let keepalive = forward.keepalive(); + let builder = builder(keepalive); + let mut shutdown = pin!(shutdown); + loop { + let (stream, peer) = tokio::select! { + () = &mut shutdown => break, + accepted = listener.accept() => match accepted { + Ok(accepted) => accepted, + Err(error) => { + back_off(error).await; + continue; + } + }, + }; + let local = match stream.local_addr() { + Ok(local) => local, + Err(error) => { + tracing::debug!(%error, "client connection closed before it was served"); + continue; + } + }; + if let Err(error) = stream.set_nodelay(true) { + tracing::warn!(%error, "could not disable Nagle on a client connection"); + } + let socket = ClientSocket { peer, local }; + let forward = forward.clone(); + let service = service_fn(move |request: Request| { + let forward = forward.clone(); + async move { Ok::<_, Infallible>(forward.handle(socket, request.map(Body::new)).await) } + }); + let connection = builder + .serve_connection(TokioIo::new(stream), service) + .with_upgrades(); + let draining = connections.clone(); + connections.spawn(async move { + let mut connection = pin!(connection); + let ended = tokio::select! { + ended = connection.as_mut() => ended, + () = draining.draining() => { + connection.as_mut().graceful_shutdown(); + connection.await + } + }; + if let Err(error) = ended { + tracing::debug!(%error, "client connection ended with an error"); + } + }); + } + drop(listener); + connections.begin_shutdown(); + if tokio::time::timeout(drain, connections.wait()) + .await + .is_err() + { + tracing::warn!( + ?drain, + "closing connections still open after the shutdown drain" + ); + } + connections.close(); + connections.wait().await; +} + +// uvicorn closes a client connection that sits idle for `--keepalive_timeout` between requests. +// Without the same limit, a client that vanishes without closing (NAT expiry, a crashed host) +// would hold its socket for the life of the process until the gateway runs out of descriptors. +// Operators already tune that timeout against their load balancer, so the public side keeps +// Python's value instead of introducing its own. hyper only runs the header timer while a +// connection waits for its next request, so it never cuts a long stream, upload or WebSocket. +fn builder(keepalive: Duration) -> http1::Builder { + let mut builder = http1::Builder::new(); + builder + .max_headers(MAX_HEADERS) + .timer(TokioTimer::new()) + .header_read_timeout(keepalive); + builder +} + +async fn back_off(error: io::Error) { + if matches!( + error.kind(), + io::ErrorKind::ConnectionRefused + | io::ErrorKind::ConnectionAborted + | io::ErrorKind::ConnectionReset + ) { + return; + } + tracing::error!(%error, "could not accept a client connection"); + tokio::time::sleep(Duration::from_secs(1)).await; +} diff --git a/litellm-rust/crates/gateway-forward/src/settings.rs b/litellm-rust/crates/gateway-forward/src/settings.rs new file mode 100644 index 00000000000..de01a96c519 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/settings.rs @@ -0,0 +1,113 @@ +use std::{ffi::OsString, net::SocketAddr, str::FromStr, time::Duration}; + +use axum::http::Uri; +use serde::Deserialize; + +use crate::Error; + +const UPSTREAM: &str = "LITELLM_FORWARD_UPSTREAM"; + +/// The Python proxy's private origin. Only a loopback IPv4 address parses, so nothing can point +/// the forwarder at another host. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Upstream(Uri); + +impl Upstream { + pub(crate) fn uri(&self) -> &Uri { + &self.0 + } +} + +impl FromStr for Upstream { + type Err = Error; + + fn from_str(value: &str) -> Result { + let uri: Uri = value.parse().map_err(Error::UpstreamUri)?; + let address = uri + .authority() + .filter(|_| uri.scheme_str() == Some("http")) + .filter(|authority| !authority.as_str().contains('@')) + .filter(|_| uri.path() == "/" && uri.query().is_none()) + .and_then(|authority| authority.as_str().parse::().ok()); + match address { + Some(SocketAddr::V4(address)) if address.ip().is_loopback() => Ok(Self(uri)), + _ => Err(Error::Upstream), + } + } +} + +#[derive(Deserialize)] +struct Environment { + litellm_forward_upstream: String, + litellm_forward_host: String, + litellm_forward_port: u16, + litellm_forward_upstream_keepalive: f64, + litellm_forward_shutdown_drain: f64, +} + +/// Everything forward mode runs with. The Python supervisor resolves each value from the settings +/// Python actually runs with, so none of them is guessed or defaulted here. +/// Setting `LITELLM_FORWARD_UPSTREAM` requires every other `LITELLM_FORWARD_*` setting below. +#[derive(Debug, Clone, PartialEq)] +pub struct Settings { + pub upstream: Upstream, + pub host: String, + pub port: u16, + pub upstream_keepalive: Duration, + pub shutdown_drain: Duration, +} + +impl Settings { + /// `None` when `LITELLM_FORWARD_UPSTREAM` is unset, which means the gateway is not forwarding. + pub fn from_env() -> Result, Error> { + Self::from_vars(std::env::vars_os()) + } + + pub fn from_vars( + variables: impl IntoIterator, + ) -> Result, Error> { + let variables: Vec<(String, String)> = variables + .into_iter() + .filter_map(|(key, value)| Some((key.into_string().ok()?, value.into_string().ok()?))) + .collect(); + if !variables.iter().any(|(key, _)| key == UPSTREAM) { + return Ok(None); + } + let environment: Environment = envy::from_iter(variables).map_err(|error| match error { + envy::Error::MissingValue(field) => Error::MissingSetting(field.to_uppercase()), + envy::Error::Custom(message) => Error::InvalidSetting(message), + })?; + Ok(Some(Self { + upstream: environment.litellm_forward_upstream.parse()?, + host: environment.litellm_forward_host, + port: environment.litellm_forward_port, + upstream_keepalive: positive_seconds( + "LITELLM_FORWARD_UPSTREAM_KEEPALIVE", + environment.litellm_forward_upstream_keepalive, + )?, + shutdown_drain: seconds( + "LITELLM_FORWARD_SHUTDOWN_DRAIN", + environment.litellm_forward_shutdown_drain, + )?, + })) + } +} + +fn seconds(variable: &str, value: f64) -> Result { + Duration::try_from_secs_f64(value).map_err(|_| { + Error::InvalidSetting(format!( + "{variable} must be a non-negative number of seconds, got {value}" + )) + }) +} + +/// The keep-alive also bounds the wait for a client's first request, so zero would close every +/// client connection before it could send one. +fn positive_seconds(variable: &str, value: f64) -> Result { + match seconds(variable, value)? { + Duration::ZERO => Err(Error::InvalidSetting(format!( + "{variable} must be a positive number of seconds, got {value}" + ))), + duration => Ok(duration), + } +} diff --git a/litellm-rust/crates/gateway-forward/src/websocket.rs b/litellm-rust/crates/gateway-forward/src/websocket.rs new file mode 100644 index 00000000000..84d89c75607 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/websocket.rs @@ -0,0 +1,86 @@ +use axum::{ + body::Body, + extract::Request, + http::{Method, Version}, + response::Response, +}; +use hyper::{ + body::Incoming, + upgrade::{OnUpgrade, Upgraded}, +}; +use hyper_util::rt::TokioIo; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +use crate::{Rejection, connections::Connections, rewrite}; + +pub(crate) fn take_handshake( + connections: &Connections, + request: &mut Request, +) -> Result, Rejection> { + if request.method() != Method::GET + || request.version() != Version::HTTP_11 + || !rewrite::is_websocket_handshake(request.headers()) + { + return Ok(None); + } + if connections.is_draining() { + return Err(Rejection::ShuttingDown); + } + Ok(Some(hyper::upgrade::on(request))) +} + +pub(crate) async fn switch( + connections: &Connections, + downstream: OnUpgrade, + mut response: Response, +) -> Result { + if !rewrite::is_websocket_handshake(response.headers()) { + return Err(Rejection::UnrequestedUpgrade); + } + let upstream = connections + .until_closed(hyper::upgrade::on(&mut response)) + .await + .ok_or(Rejection::ShuttingDown)? + .map_err(Rejection::UpgradeFailed)?; + connections.spawn(async move { + match downstream.await { + Ok(downstream) => relay(downstream, upstream).await, + Err(_) => tracing::debug!("websocket upgrade did not complete"), + } + }); + Ok(rewrite::switched_response(response).map(|_| Body::empty())) +} + +// uvicorn never half-closes a WebSocket, so Python's EOF means the socket is gone and the relay +// ends. Only a client that is gone ends it from the other side: the client's EOF just half-closes +// Python's side, and a write Python refuses means it closed with its last frames still in flight +// to the client. +async fn relay(downstream: Upgraded, upstream: Upgraded) { + let (mut client_read, mut client_write) = tokio::io::split(TokioIo::new(downstream)); + let (mut python_read, mut python_write) = tokio::io::split(TokioIo::new(upstream)); + let to_python = async { + let mut buffer = vec![0; 8192]; + loop { + let read = match client_read.read(&mut buffer).await { + Ok(0) => break, + Ok(read) => read, + Err(_) => return, + }; + if python_write.write_all(&buffer[..read]).await.is_err() { + return std::future::pending().await; + } + } + python_write.shutdown().await.ok(); + std::future::pending().await + }; + let to_client = async { + tokio::io::copy(&mut python_read, &mut client_write) + .await + .ok(); + client_write.shutdown().await.ok(); + }; + tokio::select! { + () = to_python => tracing::debug!("websocket client disconnected"), + () = to_client => {} + } +} diff --git a/litellm-rust/crates/gateway-forward/tests/client_identity.rs b/litellm-rust/crates/gateway-forward/tests/client_identity.rs new file mode 100644 index 00000000000..c484c89dc1d --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/client_identity.rs @@ -0,0 +1,223 @@ +mod support; + +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; + +use axum::{body::Body, http::StatusCode, response::Response}; +use litellm_gateway_forward::{Forward, PEER_HEADER}; +use rstest::rstest; +use support::{DRAIN, Gateway, Raw, UVICORN_KEEPALIVE, raw_upstream, recording, within}; +use tokio::net::{TcpListener, TcpStream}; + +fn ok() -> Response { + Response::new(Body::from("ok")) +} + +#[rstest] +#[case::ordinary("")] +#[case::connection_nominated( + "connection: X-Forwarded-For, X-Forwarded-Proto, Forwarded, X-Litellm-Rust-Peer\r\n" +)] +#[tokio::test] +async fn python_sees_forwarding_headers_untouched_and_the_tcp_peer_rust_saw( + #[case] connection: &str, +) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "GET /sso/callback HTTP/1.1\r\n\ + host: public.example\r\n\ + x-forwarded-for: 192.0.2.1\r\n\ + x-forwarded-for: 192.0.2.2\r\n\ + x-forwarded-proto: https\r\n\ + forwarded: for=192.0.2.1;proto=https\r\n\ + x-litellm-rust-peer: 10.1.2.3:1\r\n{connection}\r\n", + )) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + + let request = received.recv().await.unwrap(); + let forwarded_for: Vec<_> = request.headers.get_all("x-forwarded-for").iter().collect(); + assert_eq!(forwarded_for, ["192.0.2.1", "192.0.2.2"]); + assert_eq!(request.headers["x-forwarded-proto"], "https"); + assert_eq!(request.headers["forwarded"], "for=192.0.2.1;proto=https"); + let peers: Vec<_> = request.headers.get_all(PEER_HEADER).iter().collect(); + assert_eq!(peers, [raw.local_address().to_string().as_str()]); +} + +/// Python builds redirects and OAuth metadata from Host, and without one from the address it +/// listens on, which behind Rust is its private loopback port. So a request without a usable Host +/// carries the public address the client connected to, except HTTP/1.1, which h11 rejects. +#[rstest] +#[case::http11_without_host("HTTP/1.1", "", false)] +#[case::http10_without_host("HTTP/1.0", "", true)] +#[case::http11_with_an_empty_host("HTTP/1.1", "host:\r\n", true)] +#[tokio::test] +async fn a_request_without_a_usable_host_never_names_the_private_upstream( + #[case] version: &str, + #[case] host: &str, + #[case] forwarded: bool, +) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + + raw.send(format!("GET /sso/key/generate {version}\r\n{host}\r\n")) + .await; + + let status = within(raw.response_head()).await.unwrap().status; + let hosts = received.try_recv().ok().map(|request| { + request + .headers + .get_all("host") + .iter() + .map(|value| value.to_str().unwrap().to_owned()) + .collect::>() + }); + let expected = if forwarded { + (StatusCode::OK, Some(vec![gateway.address.to_string()])) + } else { + (StatusCode::BAD_REQUEST, None) + }; + assert_eq!((status, hosts), expected); +} + +#[rstest] +#[case::uppercase_with_port("PUBLIC.Example:4000")] +#[case::ipv6_literal("[::1]:4000")] +#[case::ipv4_with_port("192.0.2.10:8443")] +#[tokio::test] +async fn python_sees_the_host_header_byte_for_byte(#[case] host: &str) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + + raw.send(format!("GET /ui HTTP/1.1\r\nHost: {host}\r\n\r\n")) + .await; + + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + let request = received.recv().await.unwrap(); + let hosts: Vec<_> = request.headers.get_all("host").iter().collect(); + assert_eq!(hosts, [host]); +} + +/// Python's IP allowlists and rate limits read the peer exactly as uvicorn would have seen it on +/// the public socket, so a dual-stack listener reports IPv4 clients in their mapped form. +#[rstest] +#[case::ipv6("[::1]:0", "[::1]", "[::1]")] +#[case::ipv4_on_a_dual_stack_listener("[::]:0", "127.0.0.1", "[::ffff:127.0.0.1]")] +#[tokio::test] +async fn ipv6_listeners_report_the_peer_bracketed_with_its_port( + #[case] bind: &str, + #[case] connect: &str, + #[case] peer_ip: &str, +) { + let (upstream, mut received) = recording(ok).await; + let forward = Forward::new( + &format!("http://{upstream}").parse().unwrap(), + UVICORN_KEEPALIVE, + ); + let public = TcpListener::bind(bind).await.unwrap(); + let port = public.local_addr().unwrap().port(); + tokio::spawn(forward.serve(public, std::future::pending(), DRAIN)); + let mut raw = Raw::new( + TcpStream::connect(format!("{connect}:{port}")) + .await + .unwrap(), + ); + + raw.send("GET / HTTP/1.1\r\nhost: public.example\r\n\r\n") + .await; + + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!( + received.recv().await.unwrap().headers[PEER_HEADER], + format!("{peer_ip}:{}", raw.local_address().port()).as_str() + ); +} + +/// Without the peer header Python would take Rust's loopback hop as the client, which allowlists +/// often trust, so no spelling, repetition or Connection token may remove or replace it. +#[rstest] +#[case::named_as_hop_by_hop( + "connection: x-litellm-rust-peer\r\nx-litellm-rust-peer: 127.0.0.1:1\r\n" +)] +#[case::mixed_case_copies( + "X-LiteLLM-Rust-Peer: 127.0.0.1:1\r\nx-litellm-rust-peer: 192.0.2.7:2\r\n" +)] +#[case::comma_joined("x-litellm-rust-peer: 127.0.0.1:1, 192.0.2.7:2\r\n")] +#[tokio::test] +async fn a_client_can_neither_strip_nor_spoof_the_peer(#[case] spoof: &str) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + + raw.send(format!( + "GET /key/info HTTP/1.1\r\nhost: public.example\r\n{spoof}\r\n" + )) + .await; + + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + let request = received.recv().await.unwrap(); + let peers: Vec<_> = request.headers.get_all(PEER_HEADER).iter().collect(); + assert_eq!(peers, [raw.local_address().to_string().as_str()]); +} + +/// Pooled upstream connections carry requests from many clients in turn, so the peer belongs to +/// each request and never to the connection it travels on. +#[rstest] +#[tokio::test] +async fn each_request_carries_its_own_peer_over_a_shared_upstream_connection() { + let connections = Arc::new(AtomicUsize::new(0)); + let accepted = connections.clone(); + let upstream = raw_upstream(move |mut python| { + accepted.fetch_add(1, Ordering::SeqCst); + async move { + while let Some(head) = python.request_head().await { + let peer = head.header(PEER_HEADER).unwrap().to_vec(); + python + .send( + [ + format!("HTTP/1.1 200 OK\r\ncontent-length: {}\r\n\r\n", peer.len()) + .as_bytes(), + &peer, + ] + .concat(), + ) + .await; + } + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut clients = [ + gateway.connect().await, + gateway.connect().await, + gateway.connect().await, + ]; + + for index in [0, 1, 2, 2, 1, 0] { + let raw = &mut clients[index]; + raw.send("GET /v1/models HTTP/1.1\r\nhost: public.example\r\n\r\n") + .await; + let head = within(raw.response_head()).await.unwrap(); + let peer = raw.body(head.content_length().unwrap()).await; + assert_eq!(peer, raw.local_address().to_string().into_bytes()); + } + assert_eq!(connections.load(Ordering::SeqCst), 1); +} diff --git a/litellm-rust/crates/gateway-forward/tests/download.rs b/litellm-rust/crates/gateway-forward/tests/download.rs new file mode 100644 index 00000000000..9707d2f53b7 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/download.rs @@ -0,0 +1,192 @@ +mod support; + +use std::{ + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + time::Duration, +}; + +use axum::{body::Body, extract::Request, http::StatusCode, response::Response}; +use http_body_util::BodyExt; +use rstest::rstest; +use support::{Gateway, client, raw_upstream, recording, within}; +use tokio::sync::{Notify, mpsc}; + +#[rstest] +#[tokio::test] +async fn streams_a_body_without_a_content_length_as_python_produces_it() { + let first: &[u8] = &[0xff, 0xfb, 0x90, 0x64, 0x00, 0x0d, 0x0a]; + let rest = vec![0xa5; 64 * 1024]; + let finish = Arc::new(Notify::new()); + let finished = finish.clone(); + let tail = rest.clone(); + let upstream = raw_upstream(move |mut python| { + let finished = finished.clone(); + let tail = tail.clone(); + async move { + python.request_head().await.unwrap(); + python + .send([&b"HTTP/1.1 200 OK\r\ncontent-type: audio/mpeg\r\ntransfer-encoding: chunked\r\n\r\n7\r\n"[..], first, b"\r\n"].concat()) + .await; + finished.notified().await; + python + .send([format!("{:x}\r\n", tail.len()).as_bytes(), &tail, b"\r\n0\r\n\r\n"].concat()) + .await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut response = within( + client().request( + Request::post(gateway.url("/v1/audio/speech")) + .body(Body::empty()) + .unwrap(), + ), + ) + .await + .unwrap(); + + assert!(!response.headers().contains_key("content-length")); + let frame = within(response.body_mut().frame()).await.unwrap().unwrap(); + assert_eq!(frame.into_data().unwrap(), first); + finish.notify_one(); + let body = within(response.into_body().collect()) + .await + .unwrap() + .to_bytes(); + assert_eq!(body, rest); +} + +#[rstest] +#[tokio::test] +async fn range_requests_and_partial_content_pass_unchanged() { + let (upstream, mut received) = recording(|| -> Response { + Response::builder() + .status(StatusCode::PARTIAL_CONTENT) + .header("content-range", "bytes 2-5/10") + .header("accept-ranges", "bytes") + .body(Body::from("2345")) + .unwrap() + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET /v1/videos/video_1/content HTTP/1.1\r\nhost: x\r\nrange: bytes=2-5\r\nif-range: \"v1\"\r\n\r\n") + .await; + + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::PARTIAL_CONTENT); + assert_eq!(head.header("content-range"), Some(&b"bytes 2-5/10"[..])); + assert_eq!(raw.body(head.content_length().unwrap()).await, b"2345"); + let request = received.recv().await.unwrap(); + assert_eq!(request.headers["range"], "bytes=2-5"); + assert_eq!(request.headers["if-range"], "\"v1\""); +} + +#[rstest] +#[tokio::test] +async fn a_large_download_is_paced_by_the_client_and_keeps_its_content_length() { + const LENGTH: usize = 128 * 1024 * 1024; + let written = Arc::new(AtomicUsize::new(0)); + let writing = written.clone(); + let upstream = raw_upstream(move |mut python| { + let writing = writing.clone(); + async move { + python.request_head().await.unwrap(); + python + .send(format!("HTTP/1.1 200 OK\r\ncontent-type: application/octet-stream\r\ncontent-length: {LENGTH}\r\n\r\n")) + .await; + let chunk = [0xa5; 64 * 1024]; + while writing.load(Ordering::SeqCst) < LENGTH { + python.send(chunk).await; + writing.fetch_add(chunk.len(), Ordering::SeqCst); + } + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET /v1/files/file-1/content HTTP/1.1\r\nhost: x\r\n\r\n") + .await; + + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.content_length(), Some(LENGTH)); + assert_eq!(head.header("transfer-encoding"), None); + tokio::time::sleep(Duration::from_secs(1)).await; + assert!( + written.load(Ordering::SeqCst) < LENGTH / 4, + "the forwarder buffered a download the client was not reading" + ); + let mut remaining = LENGTH; + while remaining > 0 { + remaining -= within(raw.body(remaining.min(1 << 20))).await.len(); + } +} + +/// The UI revalidates its static assets, and a 304 must carry exactly the validators Python sent, +/// with no length invented for a body it never has. +#[rstest] +#[tokio::test] +async fn a_conditional_request_gets_python_304_with_its_validators_only() { + let (event, mut events) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python| { + let event = event.clone(); + async move { + while let Some(head) = python.request_head().await { + event + .send(head.header("if-none-match").map(<[u8]>::to_vec)) + .unwrap(); + python + .send("HTTP/1.1 304 Not Modified\r\netag: \"5f1-abc\"\r\ncache-control: no-cache\r\nvary: Accept-Encoding\r\ndate: Tue, 03 Nov 2026 10:00:00 GMT\r\n\r\n") + .await; + } + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + + for _ in 0..2 { + raw.send("GET /ui/_next/static/chunks/main.js HTTP/1.1\r\nhost: x\r\nif-none-match: \"5f1-abc\"\r\n\r\n") + .await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::NOT_MODIFIED); + let mut names: Vec<_> = head.headers.iter().map(|(name, _)| name.as_str()).collect(); + names.sort_unstable(); + assert_eq!(names, ["cache-control", "date", "etag", "vary"]); + assert_eq!(head.header("etag"), Some(&b"\"5f1-abc\""[..])); + assert_eq!( + head.header("date"), + Some(&b"Tue, 03 Nov 2026 10:00:00 GMT"[..]) + ); + assert_eq!( + within(events.recv()).await.unwrap().as_deref(), + Some(&b"\"5f1-abc\""[..]) + ); + } +} + +/// A client knows a download was cut short only because fewer bytes than the Content-Length +/// arrived before the connection closed, so the length must survive and the connection must end. +#[rstest] +#[tokio::test] +async fn a_download_python_cuts_short_closes_the_client_connection_before_its_length() { + let upstream = raw_upstream(|mut python| async move { + python.request_head().await.unwrap(); + python + .send([&b"HTTP/1.1 200 OK\r\ncontent-type: application/octet-stream\r\ncontent-length: 1000\r\n\r\n"[..], &[0xa5; 500]].concat()) + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET /v1/files/file-1/content HTTP/1.1\r\nhost: x\r\n\r\n") + .await; + + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::OK); + assert_eq!(head.content_length(), Some(1000)); + assert_eq!(within(raw.until_closed()).await, [0xa5; 500]); +} diff --git a/litellm-rust/crates/gateway-forward/tests/exchange.rs b/litellm-rust/crates/gateway-forward/tests/exchange.rs new file mode 100644 index 00000000000..8fed7b18453 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/exchange.rs @@ -0,0 +1,855 @@ +mod support; + +use std::{ + convert::Infallible, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, +}; + +use axum::{ + Router, + body::{Body, Bytes}, + extract::Request, + http::{HeaderValue, StatusCode}, + response::Response, +}; +use litellm_gateway_forward::PEER_HEADER; +use rstest::rstest; +use support::{Gateway, Head, client, raw_upstream, recording, serve, within}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + sync::mpsc, +}; + +fn ok() -> Response { + Response::new(Body::from("ok")) +} + +/// An upstream that reports each bodiless request head exactly as it arrived on the wire. +async fn heads() -> (std::net::SocketAddr, mpsc::UnboundedReceiver) { + let (sender, receiver) = mpsc::unbounded_channel(); + let address = raw_upstream(move |mut python| { + let sender = sender.clone(); + async move { + while let Some(head) = python.request_head().await { + sender.send(head).unwrap(); + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\nok") + .await; + } + } + }) + .await; + (address, receiver) +} + +#[rstest] +#[tokio::test] +async fn forwards_the_request_as_the_client_sent_it_minus_hop_headers() { + let (upstream, mut received) = recording(|| { + Response::builder() + .status(StatusCode::UNAUTHORIZED) + .header("connection", "x-upstream-hop") + .header("x-upstream-hop", "remove") + .header("x-response", "keep") + .body(Body::from("python error")) + .unwrap() + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send( + "PATCH /v1/messages?x=a%2Fb&x=c HTTP/1.1\r\n\ + host: public.example\r\n\ + authorization: Bearer test\r\n\ + connection: x-remove, keep-alive\r\n\ + x-remove: remove\r\n\ + proxy-authorization: remove\r\n\ + te: trailers\r\n\ + content-length: 7\r\n\r\npayload", + ) + .await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::UNAUTHORIZED); + assert_eq!(head.header("x-response"), Some(&b"keep"[..])); + assert_eq!(head.header("x-upstream-hop"), None); + let body = raw.body(head.content_length().unwrap()).await; + assert_eq!(body, b"python error"); + + let request = received.recv().await.unwrap(); + assert_eq!(request.method, "PATCH"); + assert_eq!(request.uri, "/v1/messages?x=a%2Fb&x=c"); + assert_eq!(request.headers["host"], "public.example"); + assert_eq!(request.headers["authorization"], "Bearer test"); + assert!(!request.headers.contains_key("x-remove")); + assert!(!request.headers.contains_key("proxy-authorization")); + assert!(!request.headers.contains_key("te")); + assert!(!request.headers.contains_key("connection")); + assert_eq!(request.body, "payload"); +} + +#[rstest] +#[case::host("HoSt", "public.example")] +#[case::length("Content-Length", "7")] +#[case::authorization("AUTHORIZATION", "Bearer test")] +#[case::cookie("Cookie", "session=test")] +#[case::api_key("X-Api-Key", "test")] +#[case::azure_key("Api-Key", "test")] +#[case::litellm_key("Litellm-Api-Key", "test")] +#[case::forwarded("Forwarded", "for=192.0.2.1;proto=https")] +#[case::forwarded_for("X-Forwarded-For", "192.0.2.1")] +#[case::forwarded_proto("X-Forwarded-Proto", "https")] +#[case::forwarded_host("X-Forwarded-Host", "public.example")] +#[case::content_type("Content-Type", "application/octet-stream")] +#[case::content_encoding("Content-Encoding", "identity")] +#[tokio::test] +async fn connection_cannot_remove_end_to_end_headers(#[case] name: &str, #[case] value: &str) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + let host = if name.eq_ignore_ascii_case("host") { + "" + } else { + "host: public.example\r\n" + }; + let length = if name.eq_ignore_ascii_case("content-length") { + "" + } else { + "content-length: 7\r\n" + }; + raw.send(format!( + "POST / HTTP/1.1\r\n{host}{length}{name}: {value}\r\nconnection: x-remove\r\nconnection: {name}, keep-alive\r\nx-remove: remove\r\n\r\npayload" + )).await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + let request = within(received.recv()).await.unwrap(); + assert_eq!(request.headers[name], value); + assert_eq!(request.body, "payload"); + assert!(!request.headers.contains_key("connection")); + assert!(!request.headers.contains_key("x-remove")); +} + +#[rstest] +#[tokio::test] +async fn connection_cannot_remove_response_credentials_or_representation_metadata() { + let (upstream, _received) = recording(|| { + Response::builder() + .status(StatusCode::UNAUTHORIZED) + .header("connection", "Content-Length, Set-Cookie, WWW-Authenticate, Authentication-Info, Content-Type, Content-Encoding, x-remove") + .header("content-length", "2") + .header("set-cookie", "session=; Max-Age=0") + .header("www-authenticate", "Bearer") + .header("authentication-info", "nextnonce=test") + .header("content-type", "text/plain") + .header("content-encoding", "identity") + .header("x-remove", "remove") + .body(Body::from("ok")) + .unwrap() + }).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET / HTTP/1.1\r\nhost: x\r\n\r\n").await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::UNAUTHORIZED); + assert_eq!(head.content_length(), Some(2)); + assert_eq!(head.header("set-cookie"), Some(&b"session=; Max-Age=0"[..])); + assert_eq!(head.header("www-authenticate"), Some(&b"Bearer"[..])); + assert_eq!( + head.header("authentication-info"), + Some(&b"nextnonce=test"[..]) + ); + assert_eq!(head.header("content-type"), Some(&b"text/plain"[..])); + assert_eq!(head.header("content-encoding"), Some(&b"identity"[..])); + assert_eq!(head.header("x-remove"), None); + assert_eq!(raw.body(2).await, b"ok"); +} + +#[rstest] +#[case::get("GET")] +#[case::post("POST")] +#[case::put("PUT")] +#[case::delete("DELETE")] +#[case::options("OPTIONS")] +#[case::extension("PURGE")] +#[case::webdav("PROPFIND")] +#[case::lowercase("get")] +#[case::punctuated("M-SEARCH")] +#[tokio::test] +async fn forwards_every_method(#[case] method: &str) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let response = client() + .request( + Request::builder() + .method(method) + .uri(gateway.url("/mcp")) + .body(Body::from("body")) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let request = received.recv().await.unwrap(); + assert_eq!(request.method, method); + assert_eq!(request.body, "body"); +} + +#[rstest] +#[case::double_slash("//v1//models")] +#[case::encoded_slash("/v1/files/a%2Fb/content")] +#[case::encoded_hash_in_query("/v1/files?purpose=batch%23retry&limit=1")] +#[case::encoded_hash_in_path("/v1/files/report%231.pdf/content")] +#[case::encoded_hash_before_query("/gemini/v1beta/files/a%23b?key=k")] +#[case::encoded_query("/v1/models?q=%E2%9C%93&empty=&flag")] +#[case::utf8_bytes_escaped("/mcp/%F0%9F%A6%80")] +#[case::lowercase_encoded_slash("/v1/files/a%2fb/content")] +#[case::dot_segments("/openai/./v1/../v1/models")] +#[case::encoded_dot_segments("/openai/%2e%2e/v1/%2E/models")] +#[case::matrix_params("/langfuse/api;v=1/public;x=y/traces")] +#[case::plus_and_encoded_space("/gemini/v1beta/models?q=a+b&r=a%20b&s=%2B")] +#[case::empty_query("/v1/models?")] +#[case::question_marks_in_query("/v1/models??a=b?c")] +#[case::colons_in_segments( + "/v1/projects/p/locations/l/publishers/google/models/m:streamGenerateContent?alt=sse" +)] +#[case::sub_delims("/~u/@:!$&'()*+,=/x")] +#[case::raw_bytes_browsers_leave_unescaped("/a{b}|c^d\\e\"f?g={}|^`")] +#[case::invalid_percent_escapes("/a%zz%00b?%")] +#[tokio::test] +async fn forwards_the_request_target_byte_for_byte(#[case] target: &str) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!("GET {target} HTTP/1.1\r\nhost: x\r\n\r\n")) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(received.recv().await.unwrap().uri, target); +} + +#[rstest] +#[tokio::test] +async fn forwards_a_query_string_as_long_as_uvicorn_accepts() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + let target = format!("/vertex_ai/v1/models?key=%2F{}&end", "a%20b".repeat(3000)); + raw.send(format!("GET {target} HTTP/1.1\r\nhost: x\r\n\r\n")) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(received.recv().await.unwrap().uri, target.as_str()); +} + +#[rstest] +#[case::in_query("/v1/files?purpose=batch#retry&limit=1", "/v1/files?purpose=batch")] +#[case::in_path("/v1/files/report#1.pdf/content", "/v1/files/report")] +#[case::before_query("/gemini/v1beta/files/a#b?key=k", "/gemini/v1beta/files/a")] +#[tokio::test] +async fn forwards_only_the_path_and_query_before_a_fragment( + #[case] target: &str, + #[case] expected: &str, +) { + let (upstream, mut received) = heads().await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!("GET {target} HTTP/1.1\r\nhost: x\r\n\r\n")) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(within(received.recv()).await.unwrap().path, expected); +} + +#[rstest] +#[tokio::test] +async fn absolute_form_targets_reach_python_as_origin_form() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send( + "GET http://elsewhere.example/v1/models?x=1 HTTP/1.1\r\nhost: elsewhere.example\r\n\r\n", + ) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + let request = received.recv().await.unwrap(); + assert_eq!(request.uri, "/v1/models?x=1"); +} + +#[rstest] +#[tokio::test] +async fn relays_a_response_with_more_headers_than_hyper_allows_by_default() { + let upstream = serve(axum::Router::new().fallback(|| async { + let headers = (0..150).map(|index| (format!("llm_provider-x-{index}"), "v")); + let mut response = ok(); + for (name, value) in headers { + response.headers_mut().insert( + axum::http::HeaderName::from_bytes(name.as_bytes()).unwrap(), + HeaderValue::from_static(value), + ); + } + response + })) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + + raw.send("GET /v1/chat/completions HTTP/1.1\r\nhost: public.example\r\n\r\n") + .await; + + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::OK); + assert_eq!(head.header("llm_provider-x-149"), Some(&b"v"[..])); +} + +/// uvicorn's h11 parser has no header count limit, only a 16 KiB cap on the whole head. +#[rstest] +#[tokio::test] +async fn forwards_a_request_with_as_many_headers_as_uvicorn_accepts() { + let (sender, mut received) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |python| { + let sender = sender.clone(); + async move { + let mut stream = python.into_stream(); + let mut head = Vec::new(); + while !head.ends_with(b"\r\n\r\n") { + head.push(stream.read_u8().await.unwrap()); + } + sender.send(head).unwrap(); + stream + .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 0\r\n\r\n") + .await + .unwrap(); + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + let headers: String = (0..1000).map(|index| format!("x-{index}: v\r\n")).collect(); + raw.send(format!( + "GET /v1/chat/completions HTTP/1.1\r\nhost: x\r\n{headers}\r\n" + )) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + let head = String::from_utf8(within(received.recv()).await.unwrap()).unwrap(); + assert!(head.contains(&headers)); +} + +async fn header_fields_python_receives(fields: &str) -> Vec<(String, Vec)> { + let (upstream, mut received) = heads().await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "GET /anthropic/v1/messages HTTP/1.1\r\nhost: x\r\n{fields}\r\n" + )) + .await; + within(raw.response_head()).await.unwrap(); + within(received.recv()) + .await + .unwrap() + .headers + .into_iter() + .filter(|(name, _)| name.starts_with("x-") && name != PEER_HEADER) + .collect() +} + +fn fields(expected: &[(&str, &str)]) -> Vec<(String, Vec)> { + expected + .iter() + .map(|(name, value)| ((*name).to_owned(), value.as_bytes().to_vec())) + .collect() +} + +/// Python reads `scope["headers"]` as the list the client sent, in that order. +#[rstest] +#[ignore = "divergence: HeaderMap::remove can move the headers after a dropped hop-by-hop one"] +#[tokio::test] +async fn dropping_a_hop_by_hop_header_keeps_the_others_in_order() { + let received = header_fields_python_receives( + "x-a: 1\r\nconnection: keep-alive\r\nx-b: 2\r\nx-c: 3\r\nx-d: 4\r\n", + ) + .await; + assert_eq!( + received, + fields(&[("x-a", "1"), ("x-b", "2"), ("x-c", "3"), ("x-d", "4")]) + ); +} + +/// uvicorn speaks only HTTP/1.1, so an HTTP/2 prior-knowledge client never reaches Python through +/// Rust either. hyper closes the connection where h11 answers 400, which no HTTP/2 client reads. +#[rstest] +#[tokio::test] +async fn an_http2_client_never_reaches_python() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n\x00\x00\x00\x04\x00\x00\x00\x00\x00") + .await; + + assert!(within(raw.until_closed()).await.is_empty()); + assert!(received.try_recv().is_err()); +} + +#[rstest] +#[ignore = "divergence: HeaderMap groups every value of a name at its first position"] +#[tokio::test] +async fn interleaved_repeated_headers_reach_python_interleaved() { + let received = header_fields_python_receives("x-a: 1\r\nx-b: 2\r\nx-a: 3\r\n").await; + assert_eq!( + received, + fields(&[("x-a", "1"), ("x-b", "2"), ("x-a", "3")]) + ); +} + +/// h11 trims optional whitespace around a value and keeps everything inside it, including empty +/// values and names using every token character. +#[rstest] +#[tokio::test] +async fn python_sees_header_values_as_h11_would_parse_them() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET / HTTP/1.1\r\nhost: x\r\nx-empty:\r\nx-blank: \r\nx-inner: a \t b \t\r\nX-Token!#$%&'*+.^_`|~: v\r\nx-empty: second\r\n\r\n") + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + let request = received.recv().await.unwrap(); + let values = |name: &str| -> Vec<&[u8]> { + request + .headers + .get_all(name) + .iter() + .map(HeaderValue::as_bytes) + .collect() + }; + assert_eq!(values("x-empty"), [&b""[..], b"second"]); + assert_eq!(values("x-blank"), [&b""[..]]); + assert_eq!(values("x-inner"), [&b"a \t b"[..]]); + assert_eq!(values("x-token!#$%&'*+.^_`|~"), [&b"v"[..]]); +} + +/// RFC 9112 section 5.2 lets a server either reject obs-fold or unfold it, and h11 unfolds. +#[rstest] +#[ignore = "divergence: hyper rejects obs-fold with a 400 where uvicorn unfolds it to one space"] +#[tokio::test] +async fn a_folded_header_reaches_python_unfolded() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET / HTTP/1.1\r\nhost: x\r\nx-a: b\r\n c\r\n\r\n") + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(received.recv().await.unwrap().headers["x-a"], "b c"); +} + +#[rstest] +#[tokio::test] +async fn keeps_repeated_response_headers_and_opaque_header_bytes() { + let (upstream, mut received) = recording(|| { + Response::builder() + .header("set-cookie", "a=1; Path=/") + .header("set-cookie", "b=2; HttpOnly") + .header("x-opaque", HeaderValue::from_bytes(b"caf\xe9").unwrap()) + .body(Body::empty()) + .unwrap() + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(b"GET / HTTP/1.1\r\nhost: x\r\nx-opaque: caf\xe9\r\n\r\n") + .await; + let head = within(raw.response_head()).await.unwrap(); + let cookies: Vec<_> = head + .headers + .iter() + .filter(|(name, _)| name == "set-cookie") + .map(|(_, value)| value.as_slice()) + .collect(); + assert_eq!(cookies, [&b"a=1; Path=/"[..], b"b=2; HttpOnly"]); + assert_eq!(head.header("x-opaque"), Some(&b"caf\xe9"[..])); + assert_eq!( + received.recv().await.unwrap().headers["x-opaque"].as_bytes(), + b"caf\xe9" + ); +} + +/// An HTTP/1.0 client cannot decode chunked framing, so a streamed answer reaches it delimited by +/// the connection closing, as uvicorn sends it. +#[rstest] +#[tokio::test] +async fn an_http10_client_gets_a_streamed_answer_unframed_and_python_gets_its_body() { + let (upstream, mut received) = recording(|| { + Response::new(Body::from_stream(futures_util::stream::iter( + ["str", "eamed"].map(|piece| Ok::<_, Infallible>(Bytes::from_static(piece.as_bytes()))), + ))) + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("POST /v1/completions HTTP/1.0\r\nhost: x\r\ncontent-length: 7\r\n\r\npayload") + .await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::OK); + assert_eq!(head.header("transfer-encoding"), None); + assert_eq!(within(raw.until_closed()).await, b"streamed"); + let request = received.recv().await.unwrap(); + assert_eq!(request.method, "POST"); + assert_eq!(request.body, "payload"); +} + +#[rstest] +#[tokio::test] +async fn answers_pipelined_requests_in_order() { + let upstream = serve( + Router::new().fallback(|request: Request| async move { request.uri().path().to_owned() }), + ) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET /first HTTP/1.1\r\nhost: x\r\n\r\nHEAD /second HTTP/1.1\r\nhost: x\r\n\r\nGET /third HTTP/1.1\r\nhost: x\r\n\r\n") + .await; + let first = within(raw.response_head()).await.unwrap(); + assert_eq!(raw.body(first.content_length().unwrap()).await, b"/first"); + let second = within(raw.response_head()).await.unwrap(); + assert_eq!(second.content_length(), Some(7)); + let third = within(raw.response_head()).await.unwrap(); + assert_eq!(raw.body(third.content_length().unwrap()).await, b"/third"); +} + +#[rstest] +#[case::no_content(StatusCode::NO_CONTENT)] +#[case::not_modified(StatusCode::NOT_MODIFIED)] +#[tokio::test] +async fn bodiless_statuses_keep_the_connection_reusable(#[case] status: StatusCode) { + let upstream = serve(Router::new().fallback(move |request: Request| async move { + match request.uri().path() { + "/next" => (StatusCode::OK, "next"), + _ => (status, ""), + } + })) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET /empty HTTP/1.1\r\nhost: x\r\n\r\nGET /next HTTP/1.1\r\nhost: x\r\n\r\n") + .await; + assert_eq!(within(raw.response_head()).await.unwrap().status, status); + let next = within(raw.response_head()).await.unwrap(); + assert_eq!(raw.body(next.content_length().unwrap()).await, b"next"); +} + +#[rstest] +#[tokio::test] +async fn informational_responses_keep_the_connection_reusable() { + let upstream = raw_upstream(|mut python| async move { + while let Some(head) = python.request_head().await { + match head.path.as_str() { + "/hints" => { + python + .send("HTTP/1.1 103 Early Hints\r\nlink: ; rel=preload\r\n\r\nHTTP/1.1 200 OK\r\ncontent-length: 5\r\n\r\nfinal") + .await + } + _ => { + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 4\r\n\r\nnext") + .await + } + } + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET /hints HTTP/1.1\r\nhost: x\r\n\r\nGET /next HTTP/1.1\r\nhost: x\r\n\r\n") + .await; + let mut hinted = within(raw.response_head()).await.unwrap(); + while hinted.status.is_informational() { + hinted = within(raw.response_head()).await.unwrap(); + } + assert_eq!(hinted.status, StatusCode::OK); + assert_eq!(raw.body(hinted.content_length().unwrap()).await, b"final"); + let next = within(raw.response_head()).await.unwrap(); + assert_eq!(raw.body(next.content_length().unwrap()).await, b"next"); +} + +#[rstest] +#[case::found( + StatusCode::FOUND, + "https://idp.example/authorize?state=a%2Fb&redirect_uri=https%3A%2F%2Fpublic.example%2Fsso%2Fcallback" +)] +#[case::see_other(StatusCode::SEE_OTHER, "/ui/?login=success")] +#[case::temporary(StatusCode::TEMPORARY_REDIRECT, "./mcp/oauth/authorize?client_id=x")] +#[tokio::test] +async fn redirects_reach_the_client_with_location_and_cookies_unchanged( + #[case] status: StatusCode, + #[case] location: &'static str, +) { + let (upstream, _received) = recording(move || { + Response::builder() + .status(status) + .header("location", location) + .header( + "set-cookie", + "token=a%2Fb; Path=/; HttpOnly; Secure; SameSite=Lax", + ) + .header("set-cookie", "state=; Max-Age=0") + .header( + "set-cookie", + "session=x; Expires=Wed, 04 Nov 2026 10:00:00 GMT; Path=/", + ) + .body(Body::empty()) + .unwrap() + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET /sso/key/generate HTTP/1.1\r\nhost: public.example\r\n\r\n") + .await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, status); + assert_eq!(head.header("location"), Some(location.as_bytes())); + let cookies: Vec<_> = head + .headers + .iter() + .filter(|(name, _)| name == "set-cookie") + .map(|(_, value)| value.as_slice()) + .collect(); + assert_eq!( + cookies, + [ + &b"token=a%2Fb; Path=/; HttpOnly; Secure; SameSite=Lax"[..], + b"state=; Max-Age=0", + b"session=x; Expires=Wed, 04 Nov 2026 10:00:00 GMT; Path=/", + ] + ); +} + +#[rstest] +#[tokio::test] +async fn non_websocket_upgrades_reach_python_as_plain_requests() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET / HTTP/1.1\r\nhost: x\r\nconnection: upgrade, http2-settings\r\nupgrade: h2c\r\nhttp2-settings: AAMAAABkAAQAoAAAAAIAAAAA\r\n\r\n") + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + let request = received.recv().await.unwrap(); + assert!(!request.headers.contains_key("upgrade")); + assert!(!request.headers.contains_key("connection")); + assert!(!request.headers.contains_key("http2-settings")); +} + +#[rstest] +#[tokio::test] +async fn forwards_an_asterisk_form_options_request() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("OPTIONS * HTTP/1.1\r\nhost: x\r\n\r\n").await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + let request = received.recv().await.unwrap(); + assert_eq!(request.method, "OPTIONS"); + assert_eq!(request.uri, "*"); +} + +#[rstest] +#[tokio::test] +async fn an_authority_form_target_is_a_400_python_never_sees() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("CONNECT api.example:443 HTTP/1.1\r\nhost: api.example:443\r\n\r\n") + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::BAD_REQUEST + ); + assert!(received.try_recv().is_err()); +} + +#[rstest] +#[tokio::test] +async fn a_client_connection_close_ends_only_the_client_connection() { + let connections = Arc::new(AtomicUsize::new(0)); + let accepted = connections.clone(); + let upstream = raw_upstream(move |mut python| { + accepted.fetch_add(1, Ordering::SeqCst); + async move { + while python.request_head().await.is_some() { + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\nok") + .await; + } + } + }) + .await; + let gateway = Gateway::start(upstream).await; + for _ in 0..2 { + let mut raw = gateway.connect().await; + raw.send("GET / HTTP/1.1\r\nhost: x\r\nconnection: close\r\n\r\n") + .await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.header("connection"), Some(&b"close"[..])); + assert_eq!(raw.body(2).await, b"ok"); + within(raw.closed()).await; + } + assert_eq!(connections.load(Ordering::SeqCst), 1); +} + +#[rstest] +#[tokio::test] +async fn a_python_connection_close_ends_only_the_upstream_connection() { + let connections = Arc::new(AtomicUsize::new(0)); + let accepted = connections.clone(); + let upstream = raw_upstream(move |mut python| { + accepted.fetch_add(1, Ordering::SeqCst); + async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 500 Internal Server Error\r\ncontent-length: 5\r\nconnection: close\r\n\r\nerror") + .await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + for _ in 0..2 { + raw.send("GET / HTTP/1.1\r\nhost: x\r\n\r\n").await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!(head.header("connection"), None); + assert_eq!(raw.body(5).await, b"error"); + } + assert_eq!(connections.load(Ordering::SeqCst), 2); +} + +/// uvicorn answers HEAD on a streaming route with chunked framing and no body, so the client +/// must not be told the body is empty, and must not wait for one before its next request. +#[rstest] +#[tokio::test] +async fn head_answered_with_chunked_framing_claims_no_length_and_keeps_the_connection() { + let upstream = raw_upstream(|mut python| async move { + while let Some(head) = python.request_head().await { + match head.path.as_str() { + "/next" => { + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 4\r\n\r\nnext") + .await + } + _ => { + python + .send("HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\n\r\n") + .await + } + } + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("HEAD /mcp HTTP/1.1\r\nhost: x\r\n\r\nGET /next HTTP/1.1\r\nhost: x\r\n\r\n") + .await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::OK); + assert_eq!(head.content_length(), None); + let next = within(raw.response_head()).await.unwrap(); + assert_eq!(raw.body(next.content_length().unwrap()).await, b"next"); +} + +const GZIP: &[u8] = &[ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x4b, 0x49, 0x2c, 0x49, 0xb4, 0x52, + 0x30, 0xe4, 0xe2, 0x02, 0x00, 0xce, 0xcd, 0x78, 0x54, 0x09, 0x00, 0x00, 0x00, +]; + +#[rstest] +#[case::fixed_length(false)] +#[case::streamed(true)] +#[tokio::test] +async fn content_codings_cross_unchanged_both_ways(#[case] streamed: bool) { + let (upstream, mut received) = recording(move || { + let body = match streamed { + true => Body::from_stream(futures_util::stream::iter( + GZIP.chunks(5) + .map(|piece| Ok::<_, Infallible>(Bytes::from_static(piece))), + )), + false => Body::from(GZIP), + }; + Response::builder() + .header("content-encoding", "gzip") + .header("vary", "Accept-Encoding") + .body(body) + .unwrap() + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send( + [ + format!( + "POST /v1/chat/completions HTTP/1.1\r\nhost: x\r\ncontent-encoding: gzip\r\naccept-encoding: gzip, br;q=0.5\r\ncontent-length: {}\r\n\r\n", + GZIP.len() + ) + .as_bytes(), + GZIP, + ] + .concat(), + ) + .await; + + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.header("content-encoding"), Some(&b"gzip"[..])); + assert_eq!(head.header("vary"), Some(&b"Accept-Encoding"[..])); + let body = match head.content_length() { + Some(length) => raw.body(length).await, + None => within(raw.chunked_body()).await, + }; + assert_eq!(body, GZIP); + assert_eq!(head.content_length().is_none(), streamed); + let request = received.recv().await.unwrap(); + assert_eq!(request.headers["content-encoding"], "gzip"); + assert_eq!(request.headers["accept-encoding"], "gzip, br;q=0.5"); + assert_eq!(request.body, GZIP); +} + +#[rstest] +#[tokio::test] +async fn python_sees_no_accept_encoding_the_client_did_not_send() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET /metrics/ HTTP/1.1\r\nhost: x\r\n\r\n").await; + within(raw.response_head()).await.unwrap(); + assert!( + !received + .recv() + .await + .unwrap() + .headers + .contains_key("accept-encoding") + ); +} diff --git a/litellm-rust/crates/gateway-forward/tests/framing.rs b/litellm-rust/crates/gateway-forward/tests/framing.rs new file mode 100644 index 00000000000..32a2c3f2b8d --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/framing.rs @@ -0,0 +1,259 @@ +mod support; + +use std::{collections::BTreeSet, net::SocketAddr}; + +use axum::http::StatusCode; +use rstest::rstest; +use support::{Gateway, Head, Raw, raw_upstream, within}; +use tokio::sync::mpsc; + +struct Captured { + head: Head, + body: Vec, +} + +/// A keep-alive upstream that decodes framing like h11 and reports every request it reads. +async fn capturing() -> (SocketAddr, mpsc::UnboundedReceiver) { + let (sender, receiver) = mpsc::unbounded_channel(); + let address = raw_upstream(move |mut python: Raw| { + let sender = sender.clone(); + async move { + while let Some(head) = python.request_head().await { + let body = match head.header("transfer-encoding") { + Some(_) => python.chunked_body().await, + None => python.body(content_length(&head)).await, + }; + sender.send(Captured { head, body }).unwrap(); + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\nok") + .await; + } + } + }) + .await; + (address, receiver) +} + +/// The length h11 reads from every Content-Length field, which may repeat one value. +fn content_length(head: &Head) -> usize { + let lengths: BTreeSet = head + .headers + .iter() + .filter(|(name, _)| name.eq_ignore_ascii_case("content-length")) + .flat_map(|(_, value)| value.split(|byte| *byte == b',')) + .map(|length| { + std::str::from_utf8(length.trim_ascii()) + .unwrap() + .parse() + .unwrap() + }) + .collect(); + assert!( + lengths.len() < 2, + "python got conflicting lengths {lengths:?}" + ); + lengths.first().copied().unwrap_or(0) +} + +/// RFC 9112 section 6.3: Transfer-Encoding overrides Content-Length. Forwarding both would let +/// Python split the stream at a different place than Rust did. Section 6.1 then closes the +/// connection after the response, so a request smuggled behind it never runs. +#[rstest] +#[tokio::test] +async fn a_request_framed_two_ways_reaches_python_framed_by_transfer_encoding_and_closes() { + let (upstream, mut received) = capturing().await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("POST /v1/files HTTP/1.1\r\nhost: x\r\ncontent-length: 3\r\ntransfer-encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\nGET /next HTTP/1.1\r\nhost: x\r\n\r\n") + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + raw.body(2).await; + within(raw.until_closed()).await; + + let upload = received.recv().await.unwrap(); + assert_eq!(upload.head.header("content-length"), None); + assert_eq!( + upload.head.header("transfer-encoding"), + Some(&b"chunked"[..]) + ); + assert_eq!(upload.body, b"hello"); + assert!( + received.try_recv().is_err(), + "python ran the smuggled request" + ); +} + +#[rstest] +#[case::conflicting_content_lengths("content-length: 4\r\ncontent-length: 40\r\n")] +#[case::content_length_list("content-length: 4, 40\r\n")] +#[case::signed_content_length("content-length: +4\r\n")] +#[case::chunked_not_last("transfer-encoding: chunked, identity\r\ncontent-length: 4\r\n")] +#[case::unknown_coding("transfer-encoding: xchunked\r\ncontent-length: 4\r\n")] +#[case::space_before_colon("transfer-encoding : chunked\r\ncontent-length: 4\r\n")] +#[case::folded_transfer_encoding("transfer-encoding:\r\n chunked\r\ncontent-length: 4\r\n")] +#[case::bare_cr("x-a: b\rx-b: c\r\ncontent-length: 4\r\n")] +#[case::nul_byte("x-a: b\x00c\r\ncontent-length: 4\r\n")] +#[tokio::test] +async fn ambiguous_framing_never_reaches_python(#[case] headers: &str) { + let (upstream, mut received) = capturing().await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "POST /v1/files HTTP/1.1\r\nhost: x\r\n{headers}\r\nbodyGET /smuggled HTTP/1.1\r\nhost: x\r\n\r\n" + )) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::BAD_REQUEST + ); + within(raw.closed()).await; + assert!(received.try_recv().is_err(), "python received a request"); +} + +#[rstest] +#[case::bare_lf("4\nbody\n0\n\n")] +#[case::invalid_size("zz\r\nbody\r\n0\r\n\r\n")] +#[case::size_overflow("10000000000000004\r\nbody\r\n0\r\n\r\n")] +#[tokio::test] +async fn a_malformed_request_body_is_a_400_not_a_502(#[case] chunks: &str) { + let upstream = raw_upstream(|mut python| async move { python.closed().await }).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "POST /v1/files HTTP/1.1\r\nhost: x\r\ntransfer-encoding: chunked\r\n\r\n{chunks}" + )) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::BAD_REQUEST + ); +} + +/// hyper decodes only the chunked layer, so any other coding would reach Python still coded but +/// framed as a plain body. uvicorn turns every h11 parse error into a 400, ignoring the 501 h11 +/// suggests for codings. +#[rstest] +#[case::listed_before_chunked("transfer-encoding: gzip, chunked\r\n")] +#[case::split_across_fields("transfer-encoding: gzip\r\ntransfer-encoding: chunked\r\n")] +#[tokio::test] +async fn a_transfer_coding_other_than_chunked_is_the_400_uvicorn_sends_python_never_sees( + #[case] headers: &str, +) { + let (upstream, mut received) = capturing().await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "POST /v1/files HTTP/1.1\r\nhost: x\r\n{headers}\r\n4\r\nbody\r\n0\r\n\r\n" + )) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::BAD_REQUEST + ); + assert!( + received.try_recv().is_err(), + "python received a still-coded body" + ); +} + +async fn assert_python_reads_the_body_and_the_next_request(headers: &str) { + let (upstream, mut received) = capturing().await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "POST /v1/files HTTP/1.1\r\nhost: x\r\n{headers}\r\npayloadGET /next HTTP/1.1\r\nhost: x\r\n\r\n" + )) + .await; + for _ in 0..2 { + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + raw.body(2).await; + } + assert_eq!(received.recv().await.unwrap().body, b"payload"); + assert_eq!(received.recv().await.unwrap().head.path, "/next"); +} + +#[rstest] +#[case::repeated_fields("content-length: 7\r\ncontent-length: 7\r\n")] +#[case::leading_zeros("content-length: 007\r\n")] +#[tokio::test] +async fn a_content_length_h11_reads_one_way_reaches_python_framed_the_same(#[case] headers: &str) { + assert_python_reads_the_body_and_the_next_request(headers).await; +} + +/// h11 accepts a Content-Length list that repeats one value, as RFC 9110 section 8.6 allows. +#[rstest] +#[ignore = "divergence: hyper's server parses each Content-Length field as one number and 400s a list"] +#[tokio::test] +async fn a_content_length_list_of_one_value_reaches_python() { + assert_python_reads_the_body_and_the_next_request("content-length: 7, 7\r\n").await; +} + +#[rstest] +#[case::leading_zeros("0004\r\nbody\r\n0\r\n\r\n", "body")] +#[case::uppercase_hex("A\r\nbody\r\nbody\r\n0\r\n\r\n", "body\r\nbody")] +#[case::whitespace_after_size("4 \r\nbody\r\n0\r\n\r\n", "body")] +#[case::extension_and_trailer("4;a=\"b;c\"\r\nbody\r\n0\r\nx-sum: 1\r\n\r\n", "body")] +#[tokio::test] +async fn chunk_framing_h11_accepts_reaches_python_decoded( + #[case] chunks: &str, + #[case] body: &str, +) { + let (upstream, mut received) = capturing().await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "POST /v1/files HTTP/1.1\r\nhost: x\r\ntransfer-encoding: chunked\r\n\r\n{chunks}GET /next HTTP/1.1\r\nhost: x\r\n\r\n" + )) + .await; + for _ in 0..2 { + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + raw.body(2).await; + } + assert_eq!(received.recv().await.unwrap().body, body.as_bytes()); + assert_eq!(received.recv().await.unwrap().head.path, "/next"); +} + +/// h11 reads a head whose lines end in a bare LF, so Python gets the same request through Rust. +#[rstest] +#[tokio::test] +async fn a_head_with_bare_lf_line_endings_reaches_python() { + let (upstream, mut received) = capturing().await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("POST /v1/files HTTP/1.1\nhost: x\nx-a: b\ncontent-length: 4\n\nbody") + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + let upload = received.recv().await.unwrap(); + assert_eq!(upload.head.header("x-a"), Some(&b"b"[..])); + assert_eq!(upload.body, b"body"); +} + +/// h11 decodes a chunked body from an HTTP/1.0 client, and RFC 9112 section 6.1 only asks the +/// server to close the connection afterwards. +#[rstest] +#[ignore = "divergence: hyper rejects Transfer-Encoding on HTTP/1.0 with a 400 that uvicorn never sends"] +#[tokio::test] +async fn an_http10_chunked_upload_reaches_python() { + let (upstream, mut received) = capturing().await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("POST /v1/files HTTP/1.0\r\nhost: x\r\ntransfer-encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\n\r\n") + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(received.recv().await.unwrap().body, b"body"); +} diff --git a/litellm-rust/crates/gateway-forward/tests/idle_stream.rs b/litellm-rust/crates/gateway-forward/tests/idle_stream.rs new file mode 100644 index 00000000000..a07c0466481 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/idle_stream.rs @@ -0,0 +1,102 @@ +mod support; + +use std::time::Duration; + +use axum::{ + body::Body, + extract::Request, + http::StatusCode, + response::{IntoResponse, Response}, +}; +use http_body_util::BodyExt; +use rstest::rstest; +use support::{Gateway, client, raw_upstream, recording, within}; +use tokio::{sync::mpsc, time::Instant}; + +const SSE_HEAD: &str = "HTTP/1.1 200 OK\r\n\ + date: Tue, 03 Nov 2026 00:00:00 GMT\r\n\ + server: uvicorn\r\n\ + cache-control: no-cache, no-transform\r\n\ + connection: keep-alive\r\n\ + content-type: text/event-stream\r\n\ + mcp-session-id: session-1\r\n\ + x-accel-buffering: no\r\n\ + transfer-encoding: chunked\r\n\r\n"; + +const PING: &str = ": ping - 2026-11-03 00:20:00.000000+00:00\r\n\r\n"; + +const MESSAGE: &str = "event: message\r\ndata: {\"jsonrpc\":\"2.0\",\"method\":\"notifications/tools/list_changed\"}\r\n\r\n"; + +fn chunk(data: &str) -> String { + format!("{:x}\r\n{data}\r\n", data.len()) +} + +/// An MCP client holds a GET stream open for server-initiated messages, which may not come for +/// many minutes, so neither hop may close a stream for being idle. +#[rstest] +#[tokio::test(start_paused = true)] +async fn an_idle_mcp_stream_stays_open_in_both_directions() { + let idle = Duration::from_secs(20 * 60); + let (session, mut sessions) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python| { + let session = session.clone(); + async move { + let head = python.request_head().await.unwrap(); + session + .send(head.header("mcp-session-id").map(<[u8]>::to_vec)) + .unwrap(); + python.send(SSE_HEAD).await; + tokio::time::sleep(idle).await; + python.send(chunk(PING)).await; + python.send(chunk(MESSAGE)).await; + python.send("0\r\n\r\n").await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let started = Instant::now(); + + let response = client() + .request( + Request::get(gateway.url("/github/mcp")) + .header("accept", "text/event-stream") + .header("mcp-session-id", "session-1") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.headers()["mcp-session-id"], "session-1"); + assert_eq!(response.headers()["x-accel-buffering"], "no"); + assert!(!response.headers().contains_key("connection")); + assert_eq!( + sessions.recv().await.unwrap().as_deref(), + Some(&b"session-1"[..]) + ); + let body = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(body, [PING, MESSAGE].concat()); + assert!(started.elapsed() >= idle); +} + +#[rstest] +#[tokio::test] +async fn ends_a_session_with_a_bodiless_delete() { + let (upstream, mut received) = recording(|| -> Response { + ([("mcp-session-id", "session-1")], StatusCode::NO_CONTENT).into_response() + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("DELETE /mcp HTTP/1.1\r\nhost: x\r\nmcp-session-id: session-1\r\n\r\n") + .await; + + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::NO_CONTENT); + assert_eq!(head.header("mcp-session-id"), Some(&b"session-1"[..])); + let request = received.recv().await.unwrap(); + assert_eq!(request.method, "DELETE"); + assert_eq!(request.headers["mcp-session-id"], "session-1"); + assert!(!request.headers.contains_key("transfer-encoding")); + assert!(request.body.is_empty()); +} diff --git a/litellm-rust/crates/gateway-forward/tests/lifecycle.rs b/litellm-rust/crates/gateway-forward/tests/lifecycle.rs new file mode 100644 index 00000000000..85d10a37be4 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/lifecycle.rs @@ -0,0 +1,473 @@ +mod support; + +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use axum::{Router, body::Body, extract::Request, http::StatusCode, response::Response}; +use http_body_util::BodyExt; +use rstest::rstest; +use support::{ + DRAIN, Gateway, Raw, client, raw_upstream, recording, serve, unreachable_upstream, within, +}; +use tokio::{ + net::{TcpListener, TcpStream}, + sync::{Notify, mpsc}, +}; + +fn ok() -> Response { + Response::new(Body::from("ok")) +} + +/// Python stops a provider stream when it sees the client go away, so a client disconnect must +/// close Rust's connection to Python at every stage of the exchange. +#[rstest] +#[case::waiting_for_headers( + "POST /v1/chat/completions HTTP/1.1\r\nhost: x\r\ncontent-length: 0\r\n\r\n", + false +)] +#[case::mid_stream( + "POST /v1/chat/completions HTTP/1.1\r\nhost: x\r\ncontent-length: 0\r\n\r\n", + true +)] +#[case::mid_upload( + "POST /v1/files HTTP/1.1\r\nhost: x\r\ncontent-length: 1000000\r\n\r\npartial", + false +)] +#[case::waiting_for_websocket_handshake( + "GET /v1/realtime HTTP/1.1\r\nhost: x\r\nconnection: Upgrade\r\nupgrade: websocket\r\nsec-websocket-version: 13\r\nsec-websocket-key: dGhlIHNhbXBsZSBub25jZQ==\r\n\r\n", + false +)] +#[tokio::test] +async fn client_disconnect_closes_the_upstream_connection( + #[case] request: &'static str, + #[case] headers_sent: bool, +) { + let (event, mut events) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python: Raw| { + let event = event.clone(); + async move { + python.request_head().await.unwrap(); + if headers_sent { + python + .send("HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n6\r\ndata: \r\n") + .await; + } + event.send("received").unwrap(); + python.closed().await; + event.send("closed").unwrap(); + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(request).await; + assert_eq!(within(events.recv()).await, Some("received")); + if headers_sent { + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + } + drop(raw); + assert_eq!(within(events.recv()).await, Some("closed")); +} + +/// Python treats EOF on its socket as the client leaving, so Rust must not half-close the +/// upstream connection once it has sent the request body. +#[rstest] +#[tokio::test] +async fn python_sees_no_eof_while_its_response_is_pending() { + let (verdict, mut verdicts) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python: Raw| { + let verdict = verdict.clone(); + async move { + let head = python.request_head().await.unwrap(); + python.body(head.content_length().unwrap()).await; + let eof = tokio::time::timeout(Duration::from_millis(200), python.closed()) + .await + .is_ok(); + verdict.send(eof).unwrap(); + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\nok") + .await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + + let response = within( + client().request( + Request::post(gateway.url("/mcp")) + .body(Body::from("{}")) + .unwrap(), + ), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(verdicts.recv().await, Some(false)); +} + +/// uvicorn closes a connection that sat idle past its keep-alive timeout. A request Rust sends on +/// that connection as uvicorn closes it is lost, so Rust must retire idle connections first. +#[rstest] +#[case::reused_while_fresh(Duration::from_millis(0))] +#[case::past_the_pool_timeout(Duration::from_millis(250))] +#[case::past_python_keepalive(Duration::from_millis(450))] +#[tokio::test] +async fn never_reuses_a_connection_python_is_closing_for_idleness(#[case] idle: Duration) { + let keepalive = Duration::from_millis(400); + let upstream = raw_upstream(move |mut python: Raw| async move { + let mut idle_since = None::; + while python.request_head().await.is_some() { + if idle_since.is_some_and(|since| since.elapsed() >= keepalive) { + return; + } + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\nok") + .await; + idle_since = Some(Instant::now()); + } + }) + .await; + let gateway = Gateway::with_keepalive(upstream, keepalive).await; + let client = client(); + let first = within(client.request(Request::get(gateway.url("/")).body(Body::empty()).unwrap())) + .await + .unwrap(); + first.into_body().collect().await.unwrap(); + tokio::time::sleep(idle).await; + let second = + within(client.request(Request::get(gateway.url("/")).body(Body::empty()).unwrap())) + .await + .unwrap(); + assert_eq!(second.status(), StatusCode::OK); +} + +#[rstest] +#[case::health("/health/liveliness")] +#[case::messages("/v1/messages")] +#[tokio::test] +async fn unreachable_python_is_a_502(#[case] path: &str) { + let gateway = Gateway::start(unreachable_upstream()).await; + let response = + within(client().request(Request::get(gateway.url(path)).body(Body::empty()).unwrap())) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); +} + +#[rstest] +#[case::bad_status_line(b"HTTP/1.1 OK\r\ncontent-length: 0\r\n\r\n".to_vec())] +#[case::header_without_colon(b"HTTP/1.1 200 OK\r\nbroken header\r\ncontent-length: 0\r\n\r\n".to_vec())] +#[case::not_http(b"SSH-2.0-OpenSSH_9.6\r\n".to_vec())] +#[case::conflicting_lengths( + b"HTTP/1.1 200 OK\r\ncontent-length: 2\r\ncontent-length: 3\r\n\r\nok".to_vec() +)] +#[case::oversized_head( + [&b"HTTP/1.1 200 OK\r\nx-big: "[..], &vec![b'a'; 1024 * 1024], b"\r\n\r\n"].concat() +)] +#[case::closed_without_reply(Vec::new())] +#[tokio::test] +async fn an_upstream_that_breaks_http_is_a_502_and_the_client_connection_survives( + #[case] reply: Vec, +) { + let upstream = raw_upstream(move |mut python: Raw| { + let reply = reply.clone(); + async move { + python.request_head().await.unwrap(); + if reply.is_empty() { + return; + } + python.send(reply).await; + std::future::pending::<()>().await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + + for _ in 0..2 { + raw.send("GET / HTTP/1.1\r\nhost: x\r\n\r\n").await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::BAD_GATEWAY); + assert_eq!(head.content_length(), Some(0)); + } +} + +#[rstest] +#[tokio::test] +async fn python_restarting_is_a_502_until_it_listens_again() { + let address = unreachable_upstream(); + let gateway = Gateway::start(address).await; + let get = || client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap()); + assert_eq!( + within(get()).await.unwrap().status(), + StatusCode::BAD_GATEWAY + ); + + let listener = TcpListener::bind(address).await.unwrap(); + let router = Router::new().fallback(|| async { "up" }); + tokio::spawn(async move { axum::serve(listener, router).await }); + + assert_eq!(within(get()).await.unwrap().status(), StatusCode::OK); +} + +/// A request Python already read may have run, so it is answered with a 502 and never replayed. +#[rstest] +#[tokio::test] +async fn a_request_python_read_before_closing_is_a_502_and_never_replayed() { + let (event, mut seen) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python: Raw| { + let event = event.clone(); + async move { + while let Some(head) = python.request_head().await { + python.body(head.content_length().unwrap()).await; + event.send(head.path.clone()).unwrap(); + if head.path == "/v1/chat/completions" { + return; + } + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\nok") + .await; + } + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("POST /warm HTTP/1.1\r\nhost: x\r\ncontent-length: 0\r\n\r\n") + .await; + let warm = within(raw.response_head()).await.unwrap(); + raw.body(warm.content_length().unwrap()).await; + + raw.send("POST /v1/chat/completions HTTP/1.1\r\nhost: x\r\ncontent-length: 2\r\n\r\n{}") + .await; + + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::BAD_GATEWAY); + let paths: Vec<_> = std::iter::from_fn(|| seen.try_recv().ok()).collect(); + assert_eq!(paths, ["/warm", "/v1/chat/completions"]); +} + +#[rstest] +#[tokio::test] +async fn shutdown_lets_an_in_flight_stream_finish() { + let release = Arc::new(Notify::new()); + let released = release.clone(); + let upstream = raw_upstream(move |mut python: Raw| { + let released = released.clone(); + async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n5\r\nfirst\r\n") + .await; + released.notified().await; + python.send("4\r\nlast\r\n0\r\n\r\n").await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let response = + within(client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap())) + .await + .unwrap(); + let server = gateway.shutdown(); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(!server.is_finished()); + release.notify_one(); + let body = within(response.into_body().collect()) + .await + .unwrap() + .to_bytes(); + assert_eq!(body, "firstlast"); + within(server).await.unwrap(); +} + +#[rstest] +#[tokio::test] +async fn shutdown_stops_waiting_for_a_stream_still_open_after_the_drain() { + let upstream = raw_upstream(|mut python: Raw| async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n5\r\nfirst\r\n") + .await; + std::future::pending::<()>().await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let response = + within(client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap())) + .await + .unwrap(); + let started = std::time::Instant::now(); + + within(gateway.shutdown()).await.unwrap(); + + assert!(started.elapsed() >= DRAIN); + drop(response); +} + +#[rstest] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_keep_alive_connections_each_get_their_own_responses() { + let upstream = serve(Router::new().fallback(|request: Request| async move { + let index: usize = request.uri().query().unwrap().parse().unwrap(); + Response::builder() + .header("etag", format!("\"{index}\"")) + .body(Body::from(index.to_string().repeat(1 + index * 1000))) + .unwrap() + })) + .await; + let gateway = Gateway::start(upstream).await; + let connections = 6; + + let tasks: Vec<_> = (0..connections) + .map(|first| { + let address = gateway.address; + tokio::spawn(async move { + let mut raw = Raw::new(TcpStream::connect(address).await.unwrap()); + for index in (first..50).step_by(connections) { + raw.send(format!( + "GET /ui/chunk.js?{index} HTTP/1.1\r\nhost: x\r\n\r\n" + )) + .await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.header("etag"), Some(format!("\"{index}\"").as_bytes())); + let body = raw.body(head.content_length().unwrap()).await; + assert_eq!( + body, + index.to_string().repeat(1 + index * 1000).into_bytes() + ); + } + }) + }) + .collect(); + + for task in tasks { + within(task).await.unwrap(); + } +} + +// Python's `--keepalive_timeout` governed idle client connections before Rust took the public +// port, and operators tune it against their load balancer's idle timeout, so it still does. +// Without it, a client that vanishes without closing its connection holds a socket forever and +// the gateway eventually runs out of file descriptors. +const IDLE: Duration = Duration::from_millis(400); + +#[rstest] +#[tokio::test] +async fn closes_a_client_connection_idle_past_python_keepalive() { + let (upstream, _received) = recording(ok).await; + let gateway = Gateway::with_keepalive(upstream, IDLE).await; + let mut raw = gateway.connect().await; + + for _ in 0..2 { + tokio::time::sleep(IDLE / 2).await; + raw.send("GET /health/liveliness HTTP/1.1\r\nhost: x\r\n\r\n") + .await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::OK); + raw.body(head.content_length().unwrap()).await; + } + let idle_since = Instant::now(); + within(raw.closed()).await; + + assert!(idle_since.elapsed() >= IDLE * 3 / 4); +} + +/// uvicorn would hold a connection that never sends a request, which lets a client park sockets +/// for free, so the same limit applies before the first request too. +#[rstest] +#[tokio::test] +async fn closes_a_client_connection_that_never_sends_a_request() { + let (upstream, _received) = recording(ok).await; + let gateway = Gateway::with_keepalive(upstream, IDLE).await; + let mut raw = gateway.connect().await; + + within(raw.closed()).await; +} + +/// The limit only covers the wait for the next request. A response that goes quiet for longer +/// belongs to Python, which decides how long it may take. +#[rstest] +#[tokio::test] +async fn keeps_a_connection_open_while_a_response_goes_quiet() { + let upstream = raw_upstream(|mut python: Raw| async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n") + .await; + tokio::time::sleep(IDLE * 3).await; + python.send("4\r\ndone\r\n0\r\n\r\n").await; + }) + .await; + let gateway = Gateway::with_keepalive(upstream, IDLE).await; + let mut raw = gateway.connect().await; + + raw.send("GET /v1/chat/completions HTTP/1.1\r\nhost: x\r\n\r\n") + .await; + + within(raw.response_head()).await.unwrap(); + assert_eq!(within(raw.chunked_body()).await, b"done"); +} + +/// Likewise for an upload that pauses longer than the limit before it finishes. +#[rstest] +#[tokio::test] +async fn keeps_a_connection_open_while_an_upload_goes_quiet() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::with_keepalive(upstream, IDLE).await; + let mut raw = gateway.connect().await; + + raw.send("POST /v1/files HTTP/1.1\r\nhost: x\r\ncontent-length: 8\r\n\r\nhalf") + .await; + tokio::time::sleep(IDLE * 3).await; + raw.send("done").await; + + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(received.recv().await.unwrap().body, "halfdone"); +} + +/// uvicorn answers a request it rejects (413, 401) as soon as it reads the head and then closes, +/// leaving the upload unread. The client must get that answer, not a 502 or a reset. +#[rstest] +#[case::sized("content-length: 16777216\r\n", vec![b'x'; 256 * 1024])] +#[case::chunked( + "transfer-encoding: chunked\r\n", + [&b"40000\r\n"[..], &[b'x'; 256 * 1024], b"\r\n"].concat() +)] +#[tokio::test] +async fn python_rejecting_an_upload_unread_and_closing_still_answers_the_client( + #[case] framing: &'static str, + #[case] partial_upload: Vec, +) { + let upstream = raw_upstream(|mut python: Raw| async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 413 Payload Too Large\r\ncontent-type: application/json\r\ncontent-length: 26\r\nconnection: close\r\n\r\n{\"error\":\"file too large\"}") + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + + raw.send(format!( + "POST /v1/files HTTP/1.1\r\nhost: x\r\n{framing}\r\n" + )) + .await; + raw.send(partial_upload).await; + + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!( + raw.body(head.content_length().unwrap()).await, + b"{\"error\":\"file too large\"}" + ); +} diff --git a/litellm-rust/crates/gateway-forward/tests/roundtrip.rs b/litellm-rust/crates/gateway-forward/tests/roundtrip.rs new file mode 100644 index 00000000000..c220c7fe8e9 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/roundtrip.rs @@ -0,0 +1,360 @@ +mod support; + +use std::{ + convert::Infallible, + sync::{Arc, Mutex}, +}; + +use axum::{ + body::{Body, Bytes}, + http::{HeaderMap, HeaderName, HeaderValue, Method, Request, StatusCode}, + response::Response, +}; +use http_body_util::BodyExt; +use litellm_gateway_forward::PEER_HEADER; +use proptest::{prelude::*, test_runner::TestRunner}; +use support::{Gateway, client, recording, within}; + +#[derive(Clone, Debug)] +struct Message { + headers: Vec<(String, Vec>)>, + body: Vec, + chunked: bool, +} + +#[derive(Clone, Debug)] +struct Exchange { + method: Method, + target: String, + request: Message, + status: StatusCode, + response: Message, +} + +/// Any value h11 reads back unchanged: visible ASCII, obs-text and interior whitespace. +fn header_value() -> impl Strategy> { + prop::collection::vec( + prop_oneof![8 => 0x21u8..=0x7e, 2 => 0x80u8..=0xff, 1 => Just(b' '), 1 => Just(b'\t')], + 0..24, + ) + .prop_map(|value| value.trim_ascii().to_vec()) +} + +fn method() -> impl Strategy { + prop_oneof![ + Just(Method::GET), + Just(Method::POST), + Just(Method::PUT), + Just(Method::PATCH), + Just(Method::DELETE), + Just(Method::OPTIONS), + "[A-Za-z0-9!#$%&'*+.^_`|~-]{1,16}" + .prop_filter("HEAD and CONNECT frame differently", |method| { + !["HEAD", "CONNECT"].contains(&method.as_str()) + }) + .prop_map(|method| Method::from_bytes(method.as_bytes()).unwrap()), + ] +} + +fn message() -> impl Strategy { + ( + prop::collection::btree_map( + "x-[a-z0-9!#$%&'*+.^_`|~-]{1,12}", + prop::collection::vec(header_value(), 1..4), + 0..12, + ), + prop::collection::vec(any::(), 0..32 * 1024), + any::(), + ) + .prop_map(|(headers, body, chunked)| Message { + headers: headers.into_iter().collect(), + body, + chunked, + }) +} + +fn target() -> impl Strategy { + let segment = prop_oneof![ + "[A-Za-z0-9._~!$&'()*+,;=:@-]{0,8}", + "%[0-9A-Fa-f]{2}", + Just(".".to_owned()), + Just("..".to_owned()), + Just(String::new()), + ]; + ( + prop::collection::vec(segment, 0..6), + prop::option::of("[A-Za-z0-9=&._~%/?:@!$'()*+,;-]{0,40}"), + ) + .prop_map(|(segments, query)| { + let path = format!("/{}", segments.join("/")); + match query { + Some(query) => format!("{path}?{query}"), + None => path, + } + }) +} + +fn exchange() -> impl Strategy { + let status = (200u16..600) + .prop_filter("bodiless status", |code| ![204, 205, 304].contains(code)) + .prop_map(|code| StatusCode::from_u16(code).unwrap()); + (method(), target(), message(), status, message()).prop_map( + |(method, target, request, status, response)| Exchange { + method, + target, + request, + status, + response, + }, + ) +} + +fn body(message: &Message) -> Body { + if !message.chunked { + return Body::from(message.body.clone()); + } + let chunks: Vec> = message + .body + .chunks(1000) + .map(|chunk| Ok(Bytes::copy_from_slice(chunk))) + .collect(); + Body::from_stream(futures_util::stream::iter(chunks)) +} + +fn apply(headers: &mut HeaderMap, message: &Message) { + if message.chunked { + headers.insert("transfer-encoding", HeaderValue::from_static("chunked")); + } + for (name, values) in &message.headers { + for value in values { + headers.append( + HeaderName::from_bytes(name.as_bytes()).unwrap(), + HeaderValue::from_bytes(value).unwrap(), + ); + } + } +} + +fn assert_carried(headers: &HeaderMap, message: &Message, framing_and_hop: &[&str]) { + for (name, values) in &message.headers { + let carried: Vec<&[u8]> = headers + .get_all(name) + .iter() + .map(HeaderValue::as_bytes) + .collect(); + let sent: Vec<&[u8]> = values.iter().map(Vec::as_slice).collect(); + assert_eq!(carried, sent, "header {name}"); + } + for name in headers.keys() { + assert!( + framing_and_hop.contains(&name.as_str()) + || message + .headers + .iter() + .any(|(sent, _)| sent == name.as_str()), + "unexpected header {name}" + ); + } +} + +/// Any request a client can send and any response Python can give cross the forwarder unchanged, +/// apart from the framing and peer headers the forwarder owns. +#[test] +fn every_exchange_crosses_the_forwarder_unchanged() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + let current = Arc::new(Mutex::new(None::)); + let responding = current.clone(); + let (gateway, received) = runtime.block_on(async { + let (upstream, received) = recording(move || { + let exchange = responding.lock().unwrap().clone().unwrap(); + let mut response = Response::new(body(&exchange.response)); + *response.status_mut() = exchange.status; + apply(response.headers_mut(), &exchange.response); + response + }) + .await; + ( + Gateway::start(upstream).await, + tokio::sync::Mutex::new(received), + ) + }); + let client = client(); + TestRunner::new(ProptestConfig::with_cases(256)) + .run(&exchange(), |exchange| { + *current.lock().unwrap() = Some(exchange.clone()); + runtime.block_on(async { + while received.lock().await.try_recv().is_ok() {} + let mut request = Request::builder() + .method(exchange.method.clone()) + .uri(gateway.url(&exchange.target)) + .body(body(&exchange.request)) + .unwrap(); + apply(request.headers_mut(), &exchange.request); + let response = support::within(client.request(request)).await.unwrap(); + + let python = received.lock().await.recv().await.unwrap(); + assert_eq!(python.method, exchange.method); + assert_eq!(python.uri, exchange.target.as_str()); + assert_eq!(python.body, exchange.request.body); + assert_carried( + &python.headers, + &exchange.request, + &["host", "content-length", "transfer-encoding", PEER_HEADER], + ); + assert_eq!(python.headers.get_all(PEER_HEADER).iter().count(), 1); + + assert_eq!(response.status(), exchange.status); + let headers = response.headers().clone(); + assert_carried( + &headers, + &exchange.response, + &["date", "content-length", "transfer-encoding"], + ); + let body = support::within(response.into_body().collect()) + .await + .unwrap(); + assert_eq!(body.to_bytes(), exchange.response.body); + }); + Ok(()) + }) + .unwrap(); +} + +#[derive(Clone, Debug)] +enum Framing { + Unframed, + Length, + Chunked(usize), +} + +#[derive(Clone, Debug)] +struct WireRequest { + method: Method, + target: String, + headers: Vec<(String, Vec)>, + body: Vec, + framing: Framing, +} + +fn wire_request() -> impl Strategy { + let framing = prop_oneof![ + Just(Framing::Unframed), + Just(Framing::Length), + (1usize..2048).prop_map(Framing::Chunked), + ]; + ( + method(), + target(), + prop::collection::vec( + ("[xX]-[A-Za-z0-9!#$%&'*+.^_`|~-]{1,12}", header_value()), + 0..24, + ), + prop::collection::vec(any::(), 0..8 * 1024), + framing, + ) + .prop_map(|(method, target, headers, body, framing)| WireRequest { + method, + target, + headers, + body: match framing { + Framing::Unframed => Vec::new(), + _ => body, + }, + framing, + }) +} + +fn encode(request: &WireRequest) -> Vec { + let framing = match request.framing { + Framing::Unframed => String::new(), + Framing::Length => format!("Content-Length: {}\r\n", request.body.len()), + Framing::Chunked(_) => "Transfer-Encoding: chunked\r\n".to_owned(), + }; + let fields = request + .headers + .iter() + .flat_map(|(name, value)| [name.as_bytes(), b": ", value, b"\r\n"].concat()); + let body = match request.framing { + Framing::Chunked(size) => request + .body + .chunks(size) + .flat_map(|chunk| { + [format!("{:x}\r\n", chunk.len()).as_bytes(), chunk, b"\r\n"].concat() + }) + .chain(*b"0\r\n\r\n") + .collect(), + _ => request.body.clone(), + }; + [ + format!( + "{} {} HTTP/1.1\r\nHost: x\r\n", + request.method, request.target + ) + .into_bytes(), + fields.collect(), + framing.into_bytes(), + b"\r\n".to_vec(), + body, + ] + .concat() +} + +fn by_name<'a>(fields: impl Iterator) -> Vec<(String, &'a [u8])> { + let mut fields: Vec<_> = fields + .map(|(name, value)| (name.to_ascii_lowercase(), value)) + .collect(); + fields.sort_by(|(left, _), (right, _)| left.cmp(right)); + fields +} + +/// Pipelined requests in any framing, written in arbitrary pieces, reach Python one by one with +/// the bytes h11 would have read from the client directly. +#[test] +fn pipelined_wire_requests_reach_python_as_sent() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + let (gateway, received) = runtime.block_on(async { + let (upstream, received) = recording(|| Response::new(Body::from("ok"))).await; + ( + Gateway::start(upstream).await, + tokio::sync::Mutex::new(received), + ) + }); + let requests = (prop::collection::vec(wire_request(), 1..5), 1usize..4096); + TestRunner::new(ProptestConfig::with_cases(256)) + .run(&requests, |(requests, piece)| { + runtime.block_on(async { + let mut received = received.lock().await; + while received.try_recv().is_ok() {} + let mut raw = gateway.connect().await; + let wire: Vec = requests.iter().flat_map(encode).collect(); + for bytes in wire.chunks(piece) { + raw.send(bytes).await; + } + for request in &requests { + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::OK, "{request:?}"); + assert_eq!(raw.body(2).await, b"ok"); + let python = within(received.recv()).await.unwrap(); + assert_eq!(python.method, request.method); + assert_eq!(python.uri, request.target.as_str()); + assert_eq!(python.body, request.body); + assert_eq!(python.headers["host"], "x"); + let forwarded = python.headers.iter().filter(|(name, _)| { + !["host", "content-length", "transfer-encoding", PEER_HEADER] + .contains(&name.as_str()) + }); + assert_eq!( + by_name(forwarded.map(|(name, value)| (name.as_str(), value.as_bytes()))), + by_name( + request + .headers + .iter() + .map(|(name, value)| (name.as_str(), value.as_slice())) + ) + ); + } + }); + Ok(()) + }) + .unwrap(); +} diff --git a/litellm-rust/crates/gateway-forward/tests/settings.rs b/litellm-rust/crates/gateway-forward/tests/settings.rs new file mode 100644 index 00000000000..ce0d95e723f --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/settings.rs @@ -0,0 +1,110 @@ +use std::{ffi::OsString, time::Duration}; + +use litellm_gateway_forward::{Error, Settings}; +use rstest::rstest; + +const COMPLETE: [(&str, &str); 5] = [ + ("LITELLM_FORWARD_UPSTREAM", "http://127.0.0.1:53211"), + ("LITELLM_FORWARD_HOST", "0.0.0.0"), + ("LITELLM_FORWARD_PORT", "4000"), + ("LITELLM_FORWARD_UPSTREAM_KEEPALIVE", "2"), + ("LITELLM_FORWARD_SHUTDOWN_DRAIN", "40"), +]; + +fn parse(variables: &[(&str, &str)]) -> Result, Error> { + Settings::from_vars( + variables + .iter() + .map(|(key, value)| (OsString::from(key), OsString::from(value))), + ) +} + +fn without(variable: &str) -> Vec<(&'static str, &'static str)> { + COMPLETE + .into_iter() + .filter(|(key, _)| *key != variable) + .collect() +} + +fn with<'a>(variable: &'a str, value: &'a str) -> Vec<(&'a str, &'a str)> { + without(variable) + .into_iter() + .chain([(variable, value)]) + .collect() +} + +#[rstest] +fn reads_every_variable() { + assert_eq!( + parse(&COMPLETE).unwrap(), + Some(Settings { + upstream: "http://127.0.0.1:53211".parse().unwrap(), + host: "0.0.0.0".into(), + port: 4000, + upstream_keepalive: Duration::from_secs(2), + shutdown_drain: Duration::from_secs(40), + }) + ); +} + +#[rstest] +fn without_an_upstream_the_gateway_is_not_forwarding() { + assert_eq!(parse(&without("LITELLM_FORWARD_UPSTREAM")).unwrap(), None); +} + +#[rstest] +#[case::host("LITELLM_FORWARD_HOST")] +#[case::port("LITELLM_FORWARD_PORT")] +#[case::upstream_keepalive("LITELLM_FORWARD_UPSTREAM_KEEPALIVE")] +#[case::shutdown_drain("LITELLM_FORWARD_SHUTDOWN_DRAIN")] +fn a_missing_variable_fails_naming_it(#[case] variable: &str) { + let error = parse(&without(variable)).unwrap_err(); + + assert!(error.to_string().contains(variable), "{error}"); +} + +#[rstest] +#[case::port_overflow("LITELLM_FORWARD_PORT", "65536")] +#[case::port_text("LITELLM_FORWARD_PORT", "http")] +#[case::keepalive_text("LITELLM_FORWARD_UPSTREAM_KEEPALIVE", "soon")] +#[case::keepalive_negative("LITELLM_FORWARD_UPSTREAM_KEEPALIVE", "-1")] +#[case::keepalive_zero("LITELLM_FORWARD_UPSTREAM_KEEPALIVE", "0")] +#[case::drain_text("LITELLM_FORWARD_SHUTDOWN_DRAIN", "soon")] +#[case::drain_negative("LITELLM_FORWARD_SHUTDOWN_DRAIN", "-1")] +#[case::drain_infinite("LITELLM_FORWARD_SHUTDOWN_DRAIN", "inf")] +fn an_invalid_variable_fails_naming_it(#[case] variable: &str, #[case] value: &str) { + let error = parse(&with(variable, value)).unwrap_err(); + + assert!(error.to_string().contains(variable), "{error}"); +} + +#[rstest] +fn fractional_drain_seconds_are_used_as_given() { + let settings = parse(&with("LITELLM_FORWARD_SHUTDOWN_DRAIN", "12.5")) + .unwrap() + .unwrap(); + + assert_eq!(settings.shutdown_drain, Duration::from_millis(12_500)); +} + +#[rstest] +#[case::keepalive("KEEPALIVE_TIMEOUT")] +#[case::graceful_shutdown("GRACEFUL_SHUTDOWN_TIMEOUT")] +#[case::host("HOST")] +#[case::port("PORT")] +fn python_settings_are_never_read(#[case] variable: &str) { + let mut variables = COMPLETE.to_vec(); + variables.push((variable, "9")); + + assert_eq!(parse(&variables).unwrap(), parse(&COMPLETE).unwrap()); +} + +#[rstest] +fn a_non_loopback_upstream_fails_startup() { + let error = parse(&with("LITELLM_FORWARD_UPSTREAM", "http://192.0.2.1:4000")).unwrap_err(); + + assert!( + error.to_string().contains("LITELLM_FORWARD_UPSTREAM"), + "{error}" + ); +} diff --git a/litellm-rust/crates/gateway-forward/tests/slow_head.rs b/litellm-rust/crates/gateway-forward/tests/slow_head.rs new file mode 100644 index 00000000000..764733aa524 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/slow_head.rs @@ -0,0 +1,41 @@ +mod support; + +use std::time::Duration; + +use axum::{body::Body, extract::Request, http::StatusCode}; +use http_body_util::BodyExt; +use rstest::rstest; +use support::{Gateway, client, raw_upstream}; +use tokio::time::Instant; + +/// Reasoning models with `stream=false`, image and video generation, and model health checks send +/// no response headers until the provider finishes, which can take many minutes. +#[rstest] +#[tokio::test(start_paused = true)] +async fn waits_for_response_headers_as_long_as_python_takes() { + let thinking = Duration::from_secs(30 * 60); + let upstream = raw_upstream(move |mut python| async move { + python.request_head().await.unwrap(); + tokio::time::sleep(thinking).await; + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 5\r\n\r\nimage") + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let started = Instant::now(); + + let response = client() + .request( + Request::post(gateway.url("/v1/images/generations")) + .body(Body::from("{}")) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + assert!(started.elapsed() >= thinking); + let body = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(body, "image"); +} diff --git a/litellm-rust/crates/gateway-forward/tests/stream.rs b/litellm-rust/crates/gateway-forward/tests/stream.rs new file mode 100644 index 00000000000..eaa12f20b77 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/stream.rs @@ -0,0 +1,313 @@ +mod support; + +use std::{ + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + time::Duration, +}; + +use axum::{body::Body, extract::Request, http::StatusCode}; +use http_body_util::BodyExt; +use hyper::body::Incoming; +use rstest::rstest; +use support::{Gateway, client, raw_upstream, within}; +use tokio::{ + io::AsyncWriteExt, + sync::{Notify, mpsc}, +}; + +fn chunk(bytes: &[u8]) -> Vec { + [format!("{:x}\r\n", bytes.len()).as_bytes(), bytes, b"\r\n"].concat() +} + +const EVENTS: [&str; 3] = ["data: 1\n\n", "data: 2\n\n", "data: [DONE]\n\n"]; + +#[rstest] +#[tokio::test] +async fn each_event_reaches_the_client_before_python_produces_the_next( + #[values(true, false)] chunked: bool, +) { + let produce = Arc::new(Notify::new()); + let next = produce.clone(); + let upstream = raw_upstream(move |mut python| { + let next = next.clone(); + async move { + python.request_head().await.unwrap(); + let framing = match chunked { + true => "transfer-encoding: chunked".to_owned(), + false => format!("content-length: {}", EVENTS.concat().len()), + }; + python + .send(format!( + "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\n{framing}\r\n\r\n" + )) + .await; + for event in EVENTS { + match chunked { + true => python.send(chunk(event.as_bytes())).await, + false => python.send(event).await, + } + next.notified().await; + } + if chunked { + python.send("0\r\n\r\n").await; + } + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut response = within( + client().request( + Request::post(gateway.url("/v1/chat/completions")) + .body(Body::from(r#"{"stream":true}"#)) + .unwrap(), + ), + ) + .await + .unwrap(); + + for event in EVENTS { + let frame = within(response.body_mut().frame()).await.unwrap().unwrap(); + assert_eq!(frame.into_data().unwrap(), event); + produce.notify_one(); + } + let rest = within(response.into_body().collect()).await.unwrap(); + assert!(rest.to_bytes().is_empty()); +} + +#[rstest] +#[tokio::test] +async fn relays_a_binary_framed_stream_unchanged() { + let message: Vec = [ + &[ + 0x00, 0x00, 0x00, 0x3d, 0x00, 0x00, 0x00, 0x20, 0x8a, 0x0d, 0x0a, 0xff, + ][..], + b"\x0b:event-type\x07\x00\x05chunk\x0d:content-type\x07\x00\x10application/json", + b"0\r\n\r\n{\"bytes\":\"eyJ0eXBlIjoiY29udGVudCJ9\"}", + &[0xc3, 0x28, 0x00, 0x0d], + ] + .concat(); + let sent = message.clone(); + let upstream = raw_upstream(move |mut python| { + let sent = sent.clone(); + async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ncontent-type: application/vnd.amazon.eventstream\r\ntransfer-encoding: chunked\r\n\r\n") + .await; + for piece in sent.chunks(7) { + python.send(chunk(piece)).await; + } + python.send("0\r\n\r\n").await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let response = within( + client().request( + Request::post(gateway.url("/bedrock/model/m/converse-stream")) + .body(Body::empty()) + .unwrap(), + ), + ) + .await + .unwrap(); + + assert_eq!( + response.headers()["content-type"], + "application/vnd.amazon.eventstream" + ); + let body = within(response.into_body().collect()) + .await + .unwrap() + .to_bytes(); + assert_eq!(body, message); +} + +#[rstest] +#[tokio::test] +async fn relays_a_body_delimited_by_the_upstream_closing() { + let upstream = raw_upstream(|mut python| async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\nconnection: close\r\n\r\nuntil eof") + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let response = client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap()); + let body = within(response).await.unwrap().into_body().collect(); + assert_eq!(within(body).await.unwrap().to_bytes(), "until eof"); +} + +async fn drain(mut body: Incoming) -> (Vec, Result<(), hyper::Error>) { + let mut delivered = Vec::new(); + loop { + match within(body.frame()).await { + None => return (delivered, Ok(())), + Some(Err(error)) => return (delivered, Err(error)), + Some(Ok(frame)) => delivered.extend_from_slice(&frame.into_data().unwrap()), + } + } +} + +#[rstest] +#[case::after_the_head("transfer-encoding: chunked", "", "")] +#[case::between_events("transfer-encoding: chunked", "6\r\ndata: \r\n", "data: ")] +#[case::mid_event("transfer-encoding: chunked", "a\r\ndata:", "data:")] +#[case::short_of_its_content_length("content-length: 100", "data: 1\n\n", "data: 1\n\n")] +#[tokio::test] +async fn an_upstream_dying_mid_stream_fails_the_client_body_instead_of_ending_it( + #[case] framing: &'static str, + #[case] sent: &'static str, + #[case] delivered: &'static str, +) { + let upstream = raw_upstream(move |mut python| async move { + python.request_head().await.unwrap(); + python + .send(format!( + "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\n{framing}\r\n\r\n{sent}" + )) + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let response = + within(client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap())) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let (received, ended) = drain(response.into_body()).await; + assert_eq!(received, delivered.as_bytes()); + assert!(ended.is_err()); +} + +#[rstest] +#[tokio::test] +async fn response_headers_reach_the_client_before_python_sends_any_body() { + let release = Arc::new(Notify::new()); + let released = release.clone(); + let upstream = raw_upstream(move |mut python| { + let released = released.clone(); + async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ncontent-type: text/event-stream; charset=utf-8\r\nx-accel-buffering: no\r\nx-litellm-call-id: call-1\r\ntransfer-encoding: chunked\r\n\r\n") + .await; + released.notified().await; + python.send(chunk(b"data: 1\n\n")).await; + python.send("0\r\n\r\n").await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + + let response = + within(client().request(Request::post(gateway.url("/")).body(Body::empty()).unwrap())) + .await + .unwrap(); + + assert_eq!(response.headers()["x-litellm-call-id"], "call-1"); + assert_eq!(response.headers()["x-accel-buffering"], "no"); + release.notify_one(); + let body = within(response.into_body().collect()) + .await + .unwrap() + .to_bytes(); + assert_eq!(body, "data: 1\n\n"); +} + +/// A connection closed without any response looks retryable to client SDKs, which would re-run a +/// request Python already ran. Python dying right after its head races the relay, hence the loop. +#[rstest] +#[tokio::test] +async fn a_head_python_sent_right_before_dying_still_reaches_the_client() { + let upstream = raw_upstream(|mut python| async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\n\r\n") + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + for _ in 0..20 { + let mut raw = gateway.connect().await; + raw.send("POST / HTTP/1.1\r\nhost: x\r\ncontent-length: 0\r\n\r\n") + .await; + let head = within(raw.response_head()).await; + assert_eq!(head.map(|head| head.status), Some(StatusCode::OK)); + } +} + +#[rstest] +#[tokio::test] +async fn a_client_that_stops_reading_stalls_python_instead_of_being_buffered_for() { + let written = Arc::new(AtomicUsize::new(0)); + let counted = written.clone(); + let (failed, mut write_failed) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python| { + let counted = counted.clone(); + let failed = failed.clone(); + async move { + python.request_head().await.unwrap(); + let mut python = python.into_stream(); + python + .write_all(b"HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n") + .await + .unwrap(); + let piece = chunk(&[0x5a; 64 * 1024]); + while python.write_all(&piece).await.is_ok() { + counted.fetch_add(piece.len(), Ordering::SeqCst); + } + failed.send(()).unwrap(); + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("POST / HTTP/1.1\r\nhost: x\r\ncontent-length: 0\r\n\r\n") + .await; + within(raw.response_head()).await.unwrap(); + + tokio::time::sleep(Duration::from_secs(1)).await; + let stalled = written.load(Ordering::SeqCst); + tokio::time::sleep(Duration::from_millis(250)).await; + + assert_eq!(written.load(Ordering::SeqCst), stalled); + assert!(stalled < 64 * 1024 * 1024, "{stalled} bytes in flight"); + drop(raw); + within(write_failed.recv()).await.unwrap(); +} + +#[rstest] +#[tokio::test] +async fn an_http10_client_receives_a_streamed_body_delimited_by_close() { + let upstream = raw_upstream(|mut python| async move { + python.request_head().await.unwrap(); + python + .send([ + &b"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\n\r\n"[..], + &chunk(b"data: 1\n"), + &chunk(b"data: 2\n"), + b"0\r\n\r\n", + ] + .concat()) + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET /v1/stream HTTP/1.0\r\nhost: x\r\n\r\n").await; + + let response = String::from_utf8(within(raw.until_closed()).await).unwrap(); + let (head, body) = response.split_once("\r\n\r\n").unwrap(); + assert!(head.starts_with("HTTP/1.0 200 OK"), "{head}"); + assert!( + !head.to_ascii_lowercase().contains("transfer-encoding"), + "{head}" + ); + assert_eq!(body, "data: 1\ndata: 2\n"); +} diff --git a/litellm-rust/crates/gateway-forward/tests/support/mod.rs b/litellm-rust/crates/gateway-forward/tests/support/mod.rs new file mode 100644 index 00000000000..d39532a456e --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/support/mod.rs @@ -0,0 +1,311 @@ +#![allow( + dead_code, + reason = "each test binary uses a different subset of the harness" +)] + +use std::{future::Future, net::SocketAddr, time::Duration}; + +use axum::{ + Router, + body::{Body, Bytes}, + extract::Request, + http::{HeaderMap, Method, StatusCode, Uri}, +}; +use http_body_util::BodyExt; +use hyper_util::{ + client::legacy::{Client, connect::HttpConnector}, + rt::TokioExecutor, +}; +use litellm_gateway_forward::Forward; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, + sync::{mpsc, oneshot}, + task::JoinHandle, +}; + +pub const WAIT: Duration = Duration::from_secs(5); +pub const UVICORN_KEEPALIVE: Duration = Duration::from_secs(5); +pub const DRAIN: Duration = Duration::from_secs(1); + +pub async fn within(future: impl Future) -> T { + tokio::time::timeout(WAIT, future).await.expect("timed out") +} + +/// A running forwarder in front of `upstream`, listening on a real socket like production. +pub struct Gateway { + pub address: SocketAddr, + stop: oneshot::Sender<()>, + server: JoinHandle<()>, +} + +impl Gateway { + pub async fn start(upstream: SocketAddr) -> Self { + Self::with_keepalive(upstream, UVICORN_KEEPALIVE).await + } + + pub async fn with_keepalive(upstream: SocketAddr, keepalive: Duration) -> Self { + let forward = Forward::new(&format!("http://{upstream}").parse().unwrap(), keepalive); + let public = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = public.local_addr().unwrap(); + let (stop, stopped) = oneshot::channel::<()>(); + let server = tokio::spawn(forward.serve( + public, + async { + stopped.await.ok(); + }, + DRAIN, + )); + Self { + address, + stop, + server, + } + } + + pub fn url(&self, path_and_query: &str) -> Uri { + format!("http://{}{path_and_query}", self.address) + .parse() + .unwrap() + } + + pub async fn connect(&self) -> Raw { + Raw::new(TcpStream::connect(self.address).await.unwrap()) + } + + /// Starts graceful shutdown and returns the server task, which ends once in-flight work drains + /// or `DRAIN` passes. + pub fn shutdown(self) -> JoinHandle<()> { + self.stop.send(()).unwrap(); + self.server + } +} + +pub fn client() -> Client { + Client::builder(TokioExecutor::new()).build_http() +} + +/// What the upstream received, after the forwarder rewrote it. +pub struct Received { + pub method: Method, + pub uri: Uri, + pub headers: HeaderMap, + pub body: Bytes, +} + +/// An axum upstream that records every request it fully reads and answers with `respond`. +pub async fn recording( + respond: impl Fn() -> axum::response::Response + Clone + Send + Sync + 'static, +) -> (SocketAddr, mpsc::UnboundedReceiver) { + let (sender, receiver) = mpsc::unbounded_channel(); + let address = serve(Router::new().fallback(move |request: Request| { + let sender = sender.clone(); + let respond = respond.clone(); + async move { + let (parts, body) = request.into_parts(); + let body = body.collect().await.unwrap().to_bytes(); + sender + .send(Received { + method: parts.method, + uri: parts.uri, + headers: parts.headers, + body, + }) + .unwrap(); + respond() + } + })) + .await; + (address, receiver) +} + +pub async fn serve(router: Router) -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + address +} + +/// An upstream that hands each accepted connection to `handle` as raw bytes, for behavior a +/// well-behaved HTTP server would never produce. +pub async fn raw_upstream(handle: F) -> SocketAddr +where + F: Fn(Raw) -> Fut + Send + Sync + 'static, + Fut: Future + Send + 'static, +{ + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.unwrap(); + tokio::spawn(handle(Raw::new(stream))); + } + }); + address +} + +pub fn unreachable_upstream() -> SocketAddr { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.local_addr().unwrap() +} + +pub struct Head { + pub status: StatusCode, + pub headers: Vec<(String, Vec)>, + pub path: String, +} + +impl Head { + pub fn header(&self, name: &str) -> Option<&[u8]> { + self.headers + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_slice()) + } + + pub fn content_length(&self) -> Option { + self.header("content-length") + .map(|value| std::str::from_utf8(value).unwrap().parse().unwrap()) + } +} + +/// One TCP connection driven byte by byte, so tests control framing, timing and disconnects. +/// Bytes read past a message head stay buffered for the next read. +pub struct Raw { + stream: TcpStream, + buffer: Vec, +} + +impl Raw { + pub fn new(stream: TcpStream) -> Self { + Self { + stream, + buffer: Vec::new(), + } + } + + pub fn into_stream(self) -> TcpStream { + assert!(self.buffer.is_empty(), "buffered bytes would be lost"); + self.stream + } + + pub fn local_address(&self) -> SocketAddr { + self.stream.local_addr().unwrap() + } + + pub async fn send(&mut self, bytes: impl AsRef<[u8]>) { + self.stream.write_all(bytes.as_ref()).await.unwrap(); + } + + /// Reads one request head, or `None` when the peer closed the connection first. + pub async fn request_head(&mut self) -> Option { + self.head(|bytes| { + let mut headers = [httparse::EMPTY_HEADER; 256]; + let mut request = httparse::Request::new(&mut headers); + let httparse::Status::Complete(length) = request.parse(bytes).unwrap() else { + return None; + }; + Some(( + length, + Head { + status: StatusCode::OK, + headers: owned(request.headers), + path: request.path.unwrap().to_owned(), + }, + )) + }) + .await + } + + pub async fn response_head(&mut self) -> Option { + self.head(|bytes| { + let mut headers = [httparse::EMPTY_HEADER; 256]; + let mut response = httparse::Response::new(&mut headers); + let httparse::Status::Complete(length) = response.parse(bytes).unwrap() else { + return None; + }; + Some(( + length, + Head { + status: StatusCode::from_u16(response.code.unwrap()).unwrap(), + headers: owned(response.headers), + path: String::new(), + }, + )) + }) + .await + } + + async fn head(&mut self, parse: impl Fn(&[u8]) -> Option<(usize, Head)>) -> Option { + loop { + if let Some((length, head)) = parse(&self.buffer) { + self.buffer.drain(..length); + return Some(head); + } + if self.fill().await == 0 { + return None; + } + } + } + + /// Reads exactly `length` body bytes, using what is already buffered first. + pub async fn body(&mut self, length: usize) -> Vec { + while self.buffer.len() < length { + assert_ne!(self.fill().await, 0, "closed mid-body"); + } + self.buffer.drain(..length).collect() + } + + /// Reads a whole chunked body and returns its decoded bytes. + pub async fn chunked_body(&mut self) -> Vec { + let mut decoded = Vec::new(); + loop { + let size_line = self.line().await; + let size = + usize::from_str_radix(size_line.split(';').next().unwrap().trim(), 16).unwrap(); + if size == 0 { + while !self.line().await.is_empty() {} + return decoded; + } + decoded.extend(self.body(size).await); + assert_eq!(self.body(2).await, b"\r\n"); + } + } + + async fn line(&mut self) -> String { + loop { + if let Some(end) = self.buffer.windows(2).position(|window| window == b"\r\n") { + let line = String::from_utf8(self.buffer[..end].to_vec()).unwrap(); + self.buffer.drain(..end + 2); + return line; + } + assert_ne!(self.fill().await, 0, "closed mid-line"); + } + } + + async fn fill(&mut self) -> usize { + let mut chunk = [0; 8192]; + let read = self.stream.read(&mut chunk).await.unwrap_or(0); + self.buffer.extend_from_slice(&chunk[..read]); + read + } + + /// Everything the peer sends until it closes, including bytes already buffered. + pub async fn until_closed(&mut self) -> Vec { + while self.fill().await != 0 {} + std::mem::take(&mut self.buffer) + } + + /// Resolves once the peer closes the connection, discarding anything it still sends. + pub async fn closed(&mut self) { + let mut chunk = [0; 8192]; + while self.stream.read(&mut chunk).await.unwrap_or(0) != 0 {} + } +} + +fn owned(headers: &[httparse::Header<'_>]) -> Vec<(String, Vec)> { + headers + .iter() + .map(|header| (header.name.to_owned(), header.value.to_vec())) + .collect() +} diff --git a/litellm-rust/crates/gateway-forward/tests/upload.rs b/litellm-rust/crates/gateway-forward/tests/upload.rs new file mode 100644 index 00000000000..3dc865c9c86 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/upload.rs @@ -0,0 +1,339 @@ +mod support; + +use std::{ + convert::Infallible, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + time::Duration, +}; + +use axum::{ + Router, + body::{Body, Bytes}, + extract::Request, + http::StatusCode, + response::Response, +}; +use http_body_util::BodyExt; +use rstest::rstest; +use support::{Gateway, Raw, client, raw_upstream, recording, serve, within}; +use tokio::{ + io::AsyncWriteExt, + sync::{Notify, mpsc}, +}; + +fn ok() -> Response { + Response::new(Body::from("ok")) +} + +#[rstest] +#[tokio::test] +async fn streams_upload_and_response_incrementally_without_a_body_limit() { + let upstream = serve( + Router::new().fallback(|request: Request| async { Response::new(request.into_body()) }), + ) + .await; + let gateway = Gateway::start(upstream).await; + let (sender, receiver) = mpsc::channel::>(1); + let stream = futures_util::stream::unfold(receiver, |mut receiver| async { + receiver.recv().await.map(|chunk| (chunk, receiver)) + }); + sender.send(Ok(Bytes::from_static(b"first"))).await.unwrap(); + let mut response = within( + client().request( + Request::post(gateway.url("/v1/files")) + .body(Body::from_stream(stream)) + .unwrap(), + ), + ) + .await + .unwrap(); + let frame = within(response.body_mut().frame()).await.unwrap().unwrap(); + assert_eq!(frame.into_data().unwrap(), "first"); + let large = Bytes::from(vec![b'x'; 8 * 1024 * 1024]); + sender.send(Ok(large.clone())).await.unwrap(); + drop(sender); + let rest = within(response.into_body().collect()) + .await + .unwrap() + .to_bytes(); + assert_eq!(rest, large); +} + +/// hyper sends the 100 as soon as the request is forwarded, where uvicorn waits for the app to +/// read the body, so an upload Python rejects on headers alone is still sent in full. +#[rstest] +#[tokio::test] +async fn answers_expect_continue_so_large_uploads_start() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("POST /v1/audio/transcriptions HTTP/1.1\r\nhost: x\r\nexpect: 100-continue\r\ncontent-length: 4\r\n\r\n") + .await; + let interim = within(raw.response_head()).await.unwrap(); + assert_eq!(interim.status, StatusCode::CONTINUE); + raw.send("data").await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(received.recv().await.unwrap().body, "data"); +} + +/// curl sends `expect: 100-continue` for any body over 1 MiB and waits for the 100 before sending +/// the multipart body, which must reach Python intact even where it looks like framing. +#[rstest] +#[tokio::test] +async fn expect_continue_delivers_a_large_multipart_upload_byte_for_byte() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let boundary = "------------------------d74496d66958873e"; + let audio: Vec = (0..3 * 1024 * 1024) + .map(|index: u32| (index.wrapping_mul(2_654_435_761) >> 24) as u8) + .chain(*b"\r\n0\r\n\r\nPOST /smuggled HTTP/1.1\r\n\r\n--") + .collect(); + let body = [ + format!("--{boundary}\r\ncontent-disposition: form-data; name=\"model\"\r\n\r\nwhisper-1\r\n--{boundary}\r\ncontent-disposition: form-data; name=\"file\"; filename=\"a b.wav\"\r\ncontent-type: audio/wav\r\n\r\n").into_bytes(), + audio, + format!("\r\n--{boundary}--\r\n").into_bytes(), + ] + .concat(); + let content_type = format!("multipart/form-data; boundary={boundary}"); + let mut raw = gateway.connect().await; + raw.send(format!( + "POST /v1/audio/transcriptions HTTP/1.1\r\nhost: x\r\ncontent-type: {content_type}\r\nexpect: 100-continue\r\ncontent-length: {}\r\n\r\n", + body.len() + )) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::CONTINUE + ); + raw.send(&body).await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + let request = received.recv().await.unwrap(); + assert_eq!(request.headers["content-type"], content_type.as_str()); + assert_eq!(request.body, body); +} + +#[rstest] +#[case::get("GET")] +#[case::delete("DELETE")] +#[case::options("OPTIONS")] +#[case::post("POST")] +#[tokio::test] +async fn forwards_a_chunked_body_on_any_method(#[case] method: &str) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "{method} /v1/search HTTP/1.1\r\nhost: x\r\ntransfer-encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\n\r\n" + )) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(received.recv().await.unwrap().body, "body"); +} + +#[rstest] +#[case::no_framing("", "", &[], "")] +#[case::empty_content_length("content-length: 0\r\n", "", &[("content-length", "0")], "")] +#[case::empty_chunked( + "transfer-encoding: chunked\r\n", + "0\r\n\r\n", + &[("transfer-encoding", "chunked")], + "" +)] +#[case::sized( + "content-length: 7\r\n", + "payload", + &[("content-length", "7")], + "payload" +)] +#[case::chunk_extensions_and_trailers( + "transfer-encoding: chunked\r\n", + "4;name=\"v\"\r\nbody\r\n0\r\nx-checksum: 1\r\n\r\n", + &[("transfer-encoding", "chunked")], + "body" +)] +#[tokio::test] +async fn a_body_reaches_python_with_the_framing_the_client_chose( + #[values("POST", "GET")] method: &str, + #[case] framing: &str, + #[case] wire_body: &str, + #[case] python_framing: &[(&str, &str)], + #[case] python_body: &str, +) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "{method} /v1/files HTTP/1.1\r\nhost: x\r\n{framing}\r\n{wire_body}" + )) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + + let request = received.recv().await.unwrap(); + let forwarded: Vec<(&str, &str)> = ["content-length", "transfer-encoding"] + .into_iter() + .flat_map(|name| { + request + .headers + .get_all(name) + .iter() + .map(move |value| (name, value.to_str().unwrap())) + }) + .collect(); + assert_eq!(request.method, method); + assert_eq!(forwarded, python_framing); + assert_eq!(request.body, python_body); +} + +#[rstest] +#[tokio::test] +async fn a_large_upload_is_paced_by_python_reading_it() { + const LENGTH: usize = 128 * 1024 * 1024; + let start_reading = Arc::new(Notify::new()); + let reading = start_reading.clone(); + let upstream = raw_upstream(move |mut python| { + let reading = reading.clone(); + async move { + let head = python.request_head().await.unwrap(); + reading.notified().await; + let mut remaining = head.content_length().unwrap(); + while remaining > 0 { + remaining -= python.body(remaining.min(1 << 20)).await.len(); + } + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 0\r\n\r\n") + .await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let (read, mut write) = gateway.connect().await.into_stream().into_split(); + let sent = Arc::new(AtomicUsize::new(0)); + let sending = sent.clone(); + let upload = tokio::spawn(async move { + write + .write_all( + format!("POST /v1/files HTTP/1.1\r\nhost: x\r\ncontent-length: {LENGTH}\r\n\r\n") + .as_bytes(), + ) + .await + .unwrap(); + let chunk = [0x5a; 64 * 1024]; + while sending.load(Ordering::SeqCst) < LENGTH { + write.write_all(&chunk).await.unwrap(); + sending.fetch_add(chunk.len(), Ordering::SeqCst); + } + write + }); + + tokio::time::sleep(Duration::from_secs(1)).await; + assert!( + sent.load(Ordering::SeqCst) < LENGTH / 4, + "the forwarder buffered an upload Python was not reading" + ); + start_reading.notify_one(); + let write = tokio::time::timeout(Duration::from_secs(60), upload) + .await + .unwrap() + .unwrap(); + let mut raw = Raw::new(read.reunite(write).unwrap()); + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); +} + +#[rstest] +#[case::between_chunks("5\r\nhello\r\n")] +#[case::mid_chunk("100\r\nhello")] +#[tokio::test] +async fn a_client_abandoning_a_chunked_upload_never_ends_python_body_cleanly( + #[case] sent: &'static str, +) { + let (forwarded, mut received) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python| { + let forwarded = forwarded.clone(); + async move { + python.request_head().await.unwrap(); + forwarded.send(python.until_closed().await).unwrap(); + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "POST /v1/files HTTP/1.1\r\nhost: x\r\ntransfer-encoding: chunked\r\n\r\n{sent}" + )) + .await; + tokio::time::sleep(Duration::from_millis(100)).await; + drop(raw); + + let body = within(received.recv()).await.unwrap(); + assert!(body.starts_with(b"5\r\nhello\r\n")); + assert!( + !body.ends_with(b"0\r\n\r\n"), + "python saw a truncated upload end cleanly" + ); +} + +#[rstest] +#[tokio::test] +async fn an_early_answer_during_an_upload_keeps_both_connections_in_sync() { + let (event, mut events) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python| { + let event = event.clone(); + async move { + while let Some(head) = python.request_head().await { + let length = head.content_length(); + event.send(head.path.into_bytes()).unwrap(); + let Some(length) = length else { + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 4\r\n\r\nnext") + .await; + continue; + }; + python + .send("HTTP/1.1 413 Payload Too Large\r\ncontent-length: 2\r\n\r\n{}") + .await; + event.send(python.body(length).await).unwrap(); + } + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let upload = b"GET /smuggled HTTP/1.1\r\nhost: x\r\n\r\n".repeat(1024); + let (first_half, second_half) = upload.split_at(upload.len() / 2); + let mut raw = gateway.connect().await; + raw.send(format!( + "POST /v1/files HTTP/1.1\r\nhost: x\r\ncontent-length: {}\r\n\r\n", + upload.len() + )) + .await; + raw.send(first_half).await; + + let rejected = within(raw.response_head()).await.unwrap(); + assert_eq!(rejected.status, StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(raw.body(2).await, b"{}"); + raw.send(second_half).await; + raw.send("GET /next HTTP/1.1\r\nhost: x\r\n\r\n").await; + let next = within(raw.response_head()).await.unwrap(); + assert_eq!(raw.body(next.content_length().unwrap()).await, b"next"); + + assert_eq!(within(events.recv()).await.unwrap(), b"/v1/files"); + assert_eq!(within(events.recv()).await.unwrap(), upload); + assert_eq!(within(events.recv()).await.unwrap(), b"/next"); +} diff --git a/litellm-rust/crates/gateway-forward/tests/upstream.rs b/litellm-rust/crates/gateway-forward/tests/upstream.rs new file mode 100644 index 00000000000..25e4768a54e --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/upstream.rs @@ -0,0 +1,21 @@ +use litellm_gateway_forward::Upstream; +use rstest::rstest; + +#[rstest] +#[case::https("https://127.0.0.1:4000")] +#[case::public("http://192.0.2.1:4000")] +#[case::hostname("http://localhost:4000")] +#[case::ipv6("http://[::1]:4000")] +#[case::path("http://127.0.0.1:4000/api")] +#[case::query("http://127.0.0.1:4000/?x=y")] +#[case::credentials("http://user@127.0.0.1:4000")] +fn rejects_anything_but_a_loopback_http_origin(#[case] upstream: &str) { + assert!(upstream.parse::().is_err()); +} + +#[rstest] +#[case::standard("http://127.0.0.1:4000")] +#[case::loopback_range("http://127.0.0.2:4000/")] +fn accepts_a_loopback_http_origin(#[case] upstream: &str) { + assert!(upstream.parse::().is_ok()); +} diff --git a/litellm-rust/crates/gateway-forward/tests/websocket.rs b/litellm-rust/crates/gateway-forward/tests/websocket.rs new file mode 100644 index 00000000000..ff64ddf38d3 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/websocket.rs @@ -0,0 +1,719 @@ +mod support; + +use std::{ + net::SocketAddr, + sync::Arc, + time::{Duration, Instant}, +}; + +use axum::{ + Router, + http::{HeaderValue, StatusCode}, + response::IntoResponse, +}; +use futures_util::{SinkExt, StreamExt, future::join_all}; +use litellm_gateway_forward::PEER_HEADER; +use rstest::rstest; +use support::{DRAIN, Gateway, Head, Raw, raw_upstream, serve, within}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpStream, + sync::{Barrier, mpsc}, +}; +use tokio_tungstenite::{ + WebSocketStream, accept_hdr_async, client_async, + tungstenite::{ + Error, Message, + client::IntoClientRequest, + handshake::{ + derive_accept_key, + server::{ErrorResponse, Request as Handshake, Response as Accepted}, + }, + protocol::{CloseFrame, frame::coding::CloseCode}, + }, +}; + +const KEY: &str = "dGhlIHNhbXBsZSBub25jZQ=="; + +async fn connect( + address: SocketAddr, + connection: &'static str, +) -> Result, Box> { + let mut request = format!("ws://{address}/v1/realtime?model=m") + .into_client_request() + .unwrap(); + let headers = request.headers_mut(); + headers.insert("authorization", HeaderValue::from_static("Bearer test")); + headers.insert( + "sec-websocket-protocol", + HeaderValue::from_static("realtime"), + ); + headers.insert("connection", HeaderValue::from_static(connection)); + let stream = TcpStream::connect(address).await.unwrap(); + let (socket, response) = client_async(request, stream).await.map_err(Box::new)?; + assert_eq!(response.headers()["sec-websocket-protocol"], "realtime"); + Ok(socket) +} + +/// Greets, echoes every message, and closes with 4000 when told to "bye". +#[expect( + clippy::result_large_err, + reason = "tungstenite's handshake callback returns ErrorResponse" +)] +async fn echo() -> SocketAddr { + raw_upstream(|python| async move { + let callback = |request: &Handshake, mut response: Accepted| { + assert_eq!(request.uri(), "/v1/realtime?model=m"); + assert_eq!(request.headers()["authorization"], "Bearer test"); + assert!(request.headers().contains_key(PEER_HEADER)); + response.headers_mut().insert( + "sec-websocket-protocol", + HeaderValue::from_static("realtime"), + ); + Ok::<_, ErrorResponse>(response) + }; + let mut socket = accept_hdr_async(python.into_stream(), callback) + .await + .unwrap(); + socket.send(Message::text("greeting")).await.unwrap(); + while let Some(Ok(message)) = socket.next().await { + if message == Message::text("bye") { + socket + .close(Some(CloseFrame { + code: CloseCode::Library(4000), + reason: "done".into(), + })) + .await + .unwrap(); + continue; + } + if message.is_close() { + break; + } + socket.send(message).await.unwrap(); + } + }) + .await +} + +fn handshake(target: &str, headers: &str) -> String { + format!( + "GET {target} HTTP/1.1\r\nhost: public.example\r\nconnection: Upgrade\r\nupgrade: websocket\r\nsec-websocket-version: 13\r\nsec-websocket-key: {KEY}\r\n{headers}\r\n" + ) +} + +/// The 101 uvicorn writes for `handshake`, with `headers` added. +fn switching(handshake: &Head, headers: &str) -> String { + let accept = derive_accept_key(handshake.header("sec-websocket-key").unwrap()); + format!( + "HTTP/1.1 101 Switching Protocols\r\nupgrade: websocket\r\nconnection: Upgrade\r\nsec-websocket-accept: {accept}\r\n{headers}\r\n" + ) +} + +fn values(head: &Head, name: &str) -> Vec { + head.headers + .iter() + .filter(|(key, _)| key.eq_ignore_ascii_case(name)) + .map(|(_, value)| String::from_utf8(value.clone()).unwrap()) + .collect() +} + +/// Accepts with `headers` in the 101, hands the handshake head to the test, then echoes bytes. +async fn byte_echo(headers: &'static str) -> (SocketAddr, mpsc::UnboundedReceiver) { + let (sender, handshakes) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python: Raw| { + let sender = sender.clone(); + async move { + let handshake = python.request_head().await.unwrap(); + python.send(switching(&handshake, headers)).await; + sender.send(handshake).unwrap(); + echo_bytes(python).await; + } + }) + .await; + (upstream, handshakes) +} + +async fn echo_bytes(python: Raw) { + let mut stream = python.into_stream(); + let (mut read, mut write) = stream.split(); + tokio::io::copy(&mut read, &mut write).await.ok(); +} + +async fn assert_echoes(socket: &mut TcpStream, message: &[u8]) { + socket.write_all(message).await.unwrap(); + let mut echoed = vec![0; message.len()]; + within(socket.read_exact(&mut echoed)).await.unwrap(); + assert_eq!(echoed, message); +} + +async fn open(gateway: &Gateway, handshake: String) -> (TcpStream, Head) { + let mut raw = gateway.connect().await; + raw.send(handshake).await; + let switched = within(raw.response_head()).await.unwrap(); + assert_eq!(switched.status, StatusCode::SWITCHING_PROTOCOLS); + (raw.into_stream(), switched) +} + +#[rstest] +#[tokio::test] +async fn relays_messages_both_ways( + #[values( + "Upgrade", + "keep-alive, Upgrade", + "upgrade", + "Upgrade, Sec-WebSocket-Key, Sec-WebSocket-Version" + )] + connection: &'static str, + #[values( + Message::text("hello"), + Message::binary(vec![0, 1, 2]), + Message::binary(vec![7; 4 * 1024 * 1024]) + )] + message: Message, +) { + let gateway = Gateway::start(echo().await).await; + let mut socket = within(connect(gateway.address, connection)).await.unwrap(); + assert_eq!( + socket.next().await.unwrap().unwrap(), + Message::text("greeting") + ); + socket.send(message.clone()).await.unwrap(); + assert_eq!(within(socket.next()).await.unwrap().unwrap(), message); + socket.close(None).await.unwrap(); + while socket.next().await.is_some() {} + drop(socket); + within(gateway.shutdown()).await.unwrap(); +} + +#[rstest] +#[tokio::test] +async fn relays_a_close_python_initiates() { + let gateway = Gateway::start(echo().await).await; + let mut socket = within(connect(gateway.address, "Upgrade")).await.unwrap(); + socket.next().await.unwrap().unwrap(); + socket.send(Message::text("bye")).await.unwrap(); + let Message::Close(Some(frame)) = within(socket.next()).await.unwrap().unwrap() else { + panic!("expected python's close frame"); + }; + assert_eq!(frame.code, CloseCode::Library(4000)); + assert_eq!(frame.reason, "done"); +} + +#[rstest] +#[tokio::test] +async fn passes_an_upstream_rejection_through() { + let upstream = serve( + Router::new() + .fallback(|| async { (StatusCode::FORBIDDEN, "origin not allowed").into_response() }), + ) + .await; + let gateway = Gateway::start(upstream).await; + let error = within(connect(gateway.address, "Upgrade")) + .await + .unwrap_err(); + let Error::Http(response) = *error else { + panic!("expected the upstream rejection, got {error}"); + }; + assert_eq!(response.status(), StatusCode::FORBIDDEN); + assert_eq!(response.body().as_deref(), Some(&b"origin not allowed"[..])); +} + +/// The idle timeout only covers a connection waiting for its next HTTP request, and a realtime +/// session may sit silent between turns. +#[rstest] +#[tokio::test] +async fn keeps_a_quiet_socket_open_past_the_keepalive() { + let gateway = Gateway::with_keepalive(echo().await, Duration::from_millis(400)).await; + let mut socket = within(connect(gateway.address, "Upgrade")).await.unwrap(); + socket.next().await.unwrap().unwrap(); + tokio::time::sleep(Duration::from_millis(1200)).await; + socket.send(Message::text("still here")).await.unwrap(); + assert_eq!( + within(socket.next()).await.unwrap().unwrap(), + Message::text("still here") + ); +} + +/// Browsers carry the API key in a subprotocol, Python accepts with the first one it was offered, +/// and uvicorn negotiates permessage-deflate, so every handshake header must cross unchanged. +#[rstest] +#[tokio::test] +async fn handshake_headers_cross_unchanged_both_ways() { + let (upstream, mut handshakes) = byte_echo( + "sec-websocket-protocol: realtime\r\nsec-websocket-extensions: permessage-deflate; server_max_window_bits=12; client_max_window_bits=12\r\nset-cookie: a=1\r\nset-cookie: b=2\r\nx-litellm-call-id: abc\r\n", + ) + .await; + let gateway = Gateway::start(upstream).await; + let target = "/v1/realtime?model=gpt-realtime&api-key=sk%2Fa%2Bb%3D&dup=1&dup=2"; + let (_socket, switched) = open( + &gateway, + handshake( + target, + "sec-websocket-protocol: realtime, openai-insecure-api-key.sk-websocket-test, openai-beta.realtime-v1\r\nsec-websocket-protocol: second-line\r\nsec-websocket-extensions: permessage-deflate; client_max_window_bits\r\ncookie: token=abc\r\n", + ), + ) + .await; + let handshake = within(handshakes.recv()).await.unwrap(); + + assert_eq!(handshake.path, target); + assert_eq!( + values(&handshake, "sec-websocket-protocol"), + [ + "realtime, openai-insecure-api-key.sk-websocket-test, openai-beta.realtime-v1", + "second-line" + ] + ); + assert_eq!( + values(&handshake, "sec-websocket-extensions"), + ["permessage-deflate; client_max_window_bits"] + ); + assert_eq!(values(&handshake, "cookie"), ["token=abc"]); + assert_eq!(values(&switched, "sec-websocket-protocol"), ["realtime"]); + assert_eq!( + values(&switched, "sec-websocket-extensions"), + ["permessage-deflate; server_max_window_bits=12; client_max_window_bits=12"] + ); + assert_eq!(values(&switched, "set-cookie"), ["a=1", "b=2"]); + assert_eq!(values(&switched, "x-litellm-call-id"), ["abc"]); +} + +#[rstest] +#[tokio::test] +async fn invents_no_subprotocol_or_extension() { + let (upstream, mut handshakes) = byte_echo("").await; + let gateway = Gateway::start(upstream).await; + let (_socket, switched) = open(&gateway, handshake("/v1/realtime", "")).await; + let handshake = within(handshakes.recv()).await.unwrap(); + for name in ["sec-websocket-protocol", "sec-websocket-extensions"] { + assert!(handshake.header(name).is_none()); + assert!(switched.header(name).is_none()); + } +} + +#[rstest] +#[case::capitalized_upgrade("Upgrade: WebSocket\r\nConnection: keep-alive, Upgrade\r\n")] +#[case::uppercase("UPGRADE: WEBSOCKET\r\nCONNECTION: UPGRADE\r\n")] +#[case::split_connection("Upgrade: websocket\r\nConnection: keep-alive\r\nConnection: Upgrade\r\n")] +#[tokio::test] +async fn accepts_any_spelling_of_the_upgrade_headers(#[case] upgrade: &str) { + let (upstream, _handshakes) = byte_echo("").await; + let gateway = Gateway::start(upstream).await; + let (mut socket, _) = open( + &gateway, + format!( + "GET /v1/realtime HTTP/1.1\r\nhost: x\r\n{upgrade}sec-websocket-version: 13\r\nsec-websocket-key: {KEY}\r\n\r\n" + ), + ) + .await; + socket.write_all(b"ping").await.unwrap(); + let mut echoed = [0; 4]; + within(socket.read_exact(&mut echoed)).await.unwrap(); + assert_eq!(&echoed, b"ping"); +} + +/// Compressed frames set RSV1 and every client frame is masked, so the socket is relayed as bytes, +/// never re-framed. +#[rstest] +#[tokio::test] +async fn relays_socket_bytes_exactly() { + let (upstream, _handshakes) = byte_echo("").await; + let gateway = Gateway::start(upstream).await; + let (socket, _) = open(&gateway, handshake("/v1/realtime", "")).await; + let sent: Vec = (0..3_000_017u32) + .map(|index| index.wrapping_mul(2_654_435_761).to_be_bytes()[1]) + .collect(); + let (mut read, mut write) = socket.into_split(); + let writing = sent.clone(); + let writer = tokio::spawn(async move { + for chunk in writing.chunks(7919) { + write.write_all(chunk).await.unwrap(); + } + write + }); + let mut echoed = vec![0; sent.len()]; + within(read.read_exact(&mut echoed)).await.unwrap(); + assert!(echoed == sent); + drop(writer.await.unwrap()); +} + +/// uvicorn writes the 101, an error event and a close frame back to back and then closes, so they +/// share a segment with the FIN, and a client may send its first frame with the handshake. +#[rstest] +#[tokio::test] +async fn keeps_bytes_that_arrive_with_the_handshake() { + let (early, mut received) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python: Raw| { + let early = early.clone(); + async move { + let handshake = python.request_head().await.unwrap(); + let mut segment = switching(&handshake, "").into_bytes(); + segment.extend(b"\x81\x05error\x88\x08\x03\xf0denied"); + python.send(segment).await; + early.send(within(python.body(6)).await).unwrap(); + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(handshake("/v1/realtime", "") + "early!").await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::SWITCHING_PROTOCOLS + ); + assert_eq!(within(received.recv()).await.unwrap(), b"early!"); + assert_eq!( + within(raw.until_closed()).await, + b"\x81\x05error\x88\x08\x03\xf0denied" + ); +} + +/// Python frees a realtime session's budget and parallel slot when its socket closes, so a client +/// that disappears without a close frame must still close Python's socket. +#[rstest] +#[case::fin(false)] +#[case::reset(true)] +#[tokio::test] +async fn a_vanished_client_closes_the_python_socket(#[case] reset: bool) { + let (event, mut events) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python: Raw| { + let event = event.clone(); + async move { + let handshake = python.request_head().await.unwrap(); + python.send(switching(&handshake, "")).await; + event.send("open").unwrap(); + python.closed().await; + event.send("closed").unwrap(); + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let (socket, _) = open(&gateway, handshake("/v1/realtime", "")).await; + assert_eq!(within(events.recv()).await, Some("open")); + if reset { + #[expect(deprecated, reason = "a zero linger is how a test sends a TCP reset")] + socket.set_linger(Some(Duration::ZERO)).unwrap(); + } + drop(socket); + assert_eq!(within(events.recv()).await, Some("closed")); +} + +#[rstest] +#[case::fin(false)] +#[case::reset(true)] +#[tokio::test] +async fn a_vanished_python_ends_the_client_socket(#[case] reset: bool) { + let upstream = raw_upstream(move |mut python: Raw| async move { + let handshake = python.request_head().await.unwrap(); + python.send(switching(&handshake, "")).await; + let socket = python.into_stream(); + if reset { + #[expect(deprecated, reason = "a zero linger is how a test sends a TCP reset")] + socket.set_linger(Some(Duration::ZERO)).unwrap(); + } + drop(socket); + }) + .await; + let gateway = Gateway::start(upstream).await; + let (mut socket, _) = open(&gateway, handshake("/v1/realtime", "")).await; + let mut rest = Vec::new(); + assert!(matches!( + within(socket.read_to_end(&mut rest)).await, + Ok(0) | Err(_) + )); +} + +#[rstest] +#[tokio::test] +async fn shutdown_closes_open_sockets_after_drain() { + let gateway = Gateway::start(echo().await).await; + let mut socket = within(connect(gateway.address, "Upgrade")).await.unwrap(); + socket.next().await.unwrap().unwrap(); + let started = Instant::now(); + within(gateway.shutdown()).await.unwrap(); + assert!(started.elapsed() >= DRAIN); + assert!(!matches!(socket.next().await, Some(Ok(Message::Text(_))))); +} + +#[rstest] +#[tokio::test] +async fn shutdown_relays_the_close_python_sends_while_draining() { + let gateway = Gateway::start(echo().await).await; + let mut socket = within(connect(gateway.address, "Upgrade")).await.unwrap(); + socket.next().await.unwrap().unwrap(); + let server = gateway.shutdown(); + tokio::time::sleep(Duration::from_millis(100)).await; + socket.send(Message::text("bye")).await.unwrap(); + let Message::Close(Some(frame)) = within(socket.next()).await.unwrap().unwrap() else { + panic!("expected python's close frame"); + }; + assert_eq!(frame.code, CloseCode::Library(4000)); + while socket.next().await.is_some() {} + drop(socket); + let started = Instant::now(); + within(server).await.unwrap(); + assert!(started.elapsed() < DRAIN / 2); +} + +#[rstest] +#[tokio::test] +async fn refuses_new_sockets_while_draining() { + let gateway = Gateway::start(echo().await).await; + let mut open = within(connect(gateway.address, "Upgrade")).await.unwrap(); + open.next().await.unwrap().unwrap(); + let address = gateway.address; + let server = gateway.shutdown(); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!( + TcpStream::connect(address).await.is_err() || connect(address, "Upgrade").await.is_err() + ); + open.send(Message::text("still open")).await.unwrap(); + assert_eq!( + within(open.next()).await.unwrap().unwrap(), + Message::text("still open") + ); + drop(open); + within(server).await.unwrap(); +} + +/// uvicorn never half-closes: once Python closes the socket nothing more can cross it, so the +/// relay must end instead of waiting for the client to close its side too. +#[rstest] +#[tokio::test] +async fn relay_ends_when_python_closes_while_the_client_holds_its_socket() { + let upstream = raw_upstream(move |mut python: Raw| async move { + let handshake = python.request_head().await.unwrap(); + python.send(switching(&handshake, "")).await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let (mut socket, _) = open(&gateway, handshake("/v1/realtime", "")).await; + assert_eq!(within(socket.read(&mut [0; 1])).await.unwrap(), 0); + let started = Instant::now(); + within(gateway.shutdown()).await.unwrap(); + assert!(started.elapsed() < DRAIN / 2); + drop(socket); +} + +/// uvicorn answers a handshake Python closes before accepting with a 403, and one Python denies +/// through `websocket.http.response` with any status, headers and body, then drops the socket. The +/// client gets that response whole and keeps a connection that still reaches Python. +#[rstest] +#[case::closed_before_accept("403 Forbidden", None, "")] +#[case::denial_response( + "401 Unauthorized", + Some("Bearer realm=\"litellm\""), + "{\"error\":\"invalid api key\"}" +)] +#[tokio::test] +async fn a_denied_handshake_reaches_the_client_whole( + #[case] status: &'static str, + #[case] authenticate: Option<&'static str>, + #[case] body: &'static str, +) { + let upstream = raw_upstream(move |mut python: Raw| async move { + python.request_head().await.unwrap(); + let authenticate = authenticate + .map(|value| format!("www-authenticate: {value}\r\n")) + .unwrap_or_default(); + python + .send(format!( + "HTTP/1.1 {status}\r\ndate: Tue, 03 Nov 2026 08:00:00 GMT\r\nconnection: close\r\ncontent-length: {}\r\ncontent-type: text/plain; charset=utf-8\r\n{authenticate}\r\n{body}", + body.len() + )) + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + for _ in 0..2 { + raw.send(handshake("/v1/realtime", "")).await; + let denied = within(raw.response_head()).await.unwrap(); + assert_eq!(denied.status.as_str(), &status[..3]); + assert_eq!( + values(&denied, "www-authenticate"), + Vec::from_iter(authenticate.map(String::from)) + ); + assert!(denied.header("sec-websocket-accept").is_none()); + let length = denied.content_length().unwrap(); + assert_eq!(within(raw.body(length)).await, body.as_bytes()); + } +} + +/// Only Python's EOF ends a relay, so a client that half-closes after its close frame still gets +/// everything Python sends before closing its own side. +#[rstest] +#[tokio::test] +async fn a_client_half_close_still_receives_what_python_sends_after_it() { + let upstream = raw_upstream(|mut python: Raw| async move { + let handshake = python.request_head().await.unwrap(); + python.send(switching(&handshake, "")).await; + let received = python.until_closed().await; + python.send(received).await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let (mut socket, _) = open(&gateway, handshake("/v1/realtime", "")).await; + let close = b"\x88\x82\x01\x02\x03\x04\x02\xea"; + socket.write_all(close).await.unwrap(); + socket.shutdown().await.unwrap(); + let mut replied = Vec::new(); + within(socket.read_to_end(&mut replied)).await.unwrap(); + assert_eq!(replied, close); +} + +/// Python authenticates every handshake before accepting it, so sessions opened together must all +/// reach Python before any of them is accepted, and each must stay on its own socket. +#[rstest] +#[tokio::test] +async fn opens_many_sockets_at_once_without_crossing_them() { + const SOCKETS: usize = 64; + let everyone = Arc::new(Barrier::new(SOCKETS)); + let upstream = raw_upstream(move |mut python: Raw| { + let everyone = everyone.clone(); + async move { + let handshake = python.request_head().await.unwrap(); + everyone.wait().await; + python.send(switching(&handshake, "")).await; + echo_bytes(python).await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let gateway = &gateway; + within(join_all((0..SOCKETS).map(|index| async move { + let (mut socket, _) = open(gateway, handshake("/v1/realtime", "")).await; + assert_echoes(&mut socket, format!("socket {index}").as_bytes()).await; + }))) + .await; +} + +/// Python resolves the key, budget and model before it accepts, and nothing bounds how long that +/// takes, so the idle timeout must not cut a handshake waiting on its 101. +#[rstest] +#[tokio::test] +async fn waits_for_a_101_slower_than_the_keepalive() { + let upstream = raw_upstream(|mut python: Raw| async move { + let handshake = python.request_head().await.unwrap(); + tokio::time::sleep(Duration::from_millis(1200)).await; + python.send(switching(&handshake, "")).await; + echo_bytes(python).await; + }) + .await; + let gateway = Gateway::with_keepalive(upstream, Duration::from_millis(400)).await; + let (mut socket, _) = open(&gateway, handshake("/v1/realtime", "")).await; + assert_echoes(&mut socket, b"late").await; +} + +/// A 101 the client did not ask for, or to a protocol other than the one it asked for, would hand +/// the client a socket it cannot speak, so it is a 502 and Python's socket is closed. +#[rstest] +#[case::plain_request("GET /v1/models HTTP/1.1\r\nhost: x\r\n\r\n".to_owned(), "websocket")] +#[case::other_protocol(handshake("/v1/realtime", ""), "h2c")] +#[tokio::test] +async fn an_unrequested_switch_is_a_502_that_closes_python( + #[case] request: String, + #[case] protocol: &'static str, +) { + let (closed, mut closes) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python: Raw| { + let closed = closed.clone(); + async move { + python.request_head().await.unwrap(); + python + .send(format!( + "HTTP/1.1 101 Switching Protocols\r\nconnection: upgrade\r\nupgrade: {protocol}\r\n\r\n" + )) + .await; + python.closed().await; + closed.send(()).unwrap(); + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(request).await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::BAD_GATEWAY + ); + within(closes.recv()).await.unwrap(); +} + +/// A realtime session streams audio for as long as the client reads it, so a client that stops +/// reading must stall Python's writes instead of piling them up in the gateway. +#[rstest] +#[tokio::test] +async fn a_client_that_stops_reading_stalls_python() { + let (sent, mut totals) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python: Raw| { + let sent = sent.clone(); + async move { + let handshake = python.request_head().await.unwrap(); + python.send(switching(&handshake, "")).await; + python.body(2).await; + let mut stream = python.into_stream(); + let chunk = vec![0x82; 64 * 1024]; + let mut total = 0; + while let Ok(Ok(())) = + tokio::time::timeout(Duration::from_millis(300), stream.write_all(&chunk)).await + { + total += chunk.len(); + } + sent.send(total).unwrap(); + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let (mut socket, _) = open(&gateway, handshake("/v1/realtime", "")).await; + socket.write_all(b"go").await.unwrap(); + let total = within(totals.recv()).await.unwrap(); + assert!( + total < 64 << 20, + "python wrote {total} bytes into a stalled socket" + ); +} + +/// Python closes a session that is still streaming audio by sending an error event and a close +/// frame and dropping its socket with the client's next frames unread, which resets it. Those last +/// bytes reached the client when it talked to uvicorn directly, so they must reach it through the +/// relay too, even though writing the client's frames to Python now fails. +#[rstest] +#[tokio::test] +async fn python_closing_mid_stream_still_delivers_its_last_frames() { + const LAST: &[u8] = b"\x81\x05error\x88\x02\x03\xf0"; + let upstream = raw_upstream(|mut python: Raw| async move { + let handshake = python.request_head().await.unwrap(); + python.send(switching(&handshake, "")).await; + python.body(1).await; + tokio::time::sleep(Duration::from_millis(20)).await; + python.send(LAST).await; + }) + .await; + let gateway = Gateway::start(upstream).await; + for _ in 0..20 { + let (mut socket, _) = open(&gateway, handshake("/v1/realtime", "")).await; + let (mut read, mut write) = socket.split(); + let streaming = async { + while write.write_all(b"\x82\x80\0\0\0\0").await.is_ok() { + tokio::time::sleep(Duration::from_micros(200)).await; + } + std::future::pending().await + }; + let mut received = Vec::new(); + let receiving = async { + let mut chunk = [0; 4096]; + while let Ok(read @ 1..) = read.read(&mut chunk).await { + received.extend_from_slice(&chunk[..read]); + } + }; + within(async { + tokio::select! { + () = streaming => {} + () = receiving => {} + } + }) + .await; + assert_eq!(received, LAST); + } +} diff --git a/litellm-rust/crates/gateway/Cargo.toml b/litellm-rust/crates/gateway/Cargo.toml index 6eeaa43e013..d0cedbb3b36 100644 --- a/litellm-rust/crates/gateway/Cargo.toml +++ b/litellm-rust/crates/gateway/Cargo.toml @@ -12,6 +12,7 @@ envy = "0.4.2" http-body-util = "0.1" litellm-inference.workspace = true litellm-gateway-inference.workspace = true +litellm-gateway-forward.workspace = true litellm-gateway-mcp.workspace = true tokio-util = "0.7" thiserror.workspace = true diff --git a/litellm-rust/crates/gateway/src/main.rs b/litellm-rust/crates/gateway/src/main.rs index 47c1fe83f50..ff528723095 100644 --- a/litellm-rust/crates/gateway/src/main.rs +++ b/litellm-rust/crates/gateway/src/main.rs @@ -10,7 +10,7 @@ use serde_json::json; mod settings; -use settings::{Settings, UiSettings}; +use settings::{LogSettings, Settings, UiSettings}; struct StderrSink { level: Level, @@ -41,6 +41,15 @@ impl Sink for StderrSink { #[tokio::main] async fn main() -> Result<(), Box> { + if let Some(forward) = litellm_gateway_forward::Settings::from_env()? { + let log: LogSettings = settings::from_iter(std::env::vars_os())?; + Logger::new(StderrSink { + level: log.log_level(), + }) + .install_global()?; + litellm_gateway_forward::run(forward, shutdown_signal()).await?; + return Ok(()); + } let settings: Settings = settings::from_iter(std::env::vars_os())?; let level = settings.log_level(); Logger::new(StderrSink { level }).install_global()?; @@ -126,7 +135,20 @@ async fn shutdown_signal() { } } } - #[cfg(not(unix))] + // Windows support exists for local CLI runs only, the proxy is never deployed on Windows. The supervisor + // stops the gateway with CTRL_BREAK_EVENT, which has to drain the same way SIGTERM does. + #[cfg(windows)] + { + match tokio::signal::windows::ctrl_break() { + Ok(mut ctrl_break) => { + tokio::select! { _ = tokio::signal::ctrl_c() => (), _ = ctrl_break.recv() => () } + } + Err(_) => { + let _ = tokio::signal::ctrl_c().await; + } + } + } + #[cfg(not(any(unix, windows)))] { let _ = tokio::signal::ctrl_c().await; } diff --git a/litellm-rust/crates/gateway/src/settings.rs b/litellm-rust/crates/gateway/src/settings.rs index 445071504ed..c2c8bc30934 100644 --- a/litellm-rust/crates/gateway/src/settings.rs +++ b/litellm-rust/crates/gateway/src/settings.rs @@ -4,6 +4,23 @@ use litellm_secret_value::SecretValue; use litellm_tracing::Level; use serde::{Deserialize, de::DeserializeOwned}; +#[derive(Deserialize)] +pub(super) struct LogSettings { + rust_log: Option, +} + +impl LogSettings { + pub fn log_level(&self) -> Level { + log_level(self.rust_log.as_deref()) + } +} + +fn log_level(rust_log: Option<&str>) -> Level { + rust_log + .and_then(|value| value.parse().ok()) + .unwrap_or(Level::INFO) +} + #[derive(Deserialize)] #[serde(default)] pub(super) struct Settings { @@ -26,10 +43,7 @@ impl Default for Settings { impl Settings { pub fn log_level(&self) -> Level { - self.rust_log - .as_deref() - .and_then(|value| value.parse().ok()) - .unwrap_or(Level::INFO) + log_level(self.rust_log.as_deref()) } } From fb74bc902f21486b7ac3d1ed122da8fc6500649d Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:10:09 -0800 Subject: [PATCH 02/40] chore(cost-map): rolling registry audit, add gpt-6-cyber, Kimi K3.1 Pro rows, Jina rerankers and capability flags (#50245) * chore(cost-map): rolling registry audit, add gpt-6-cyber, Kimi K3.1 Pro rows, Jina rerankers and capability flags Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(cost-map): declare gpt-6-cyber PDF input support Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 87 ++++++++++++++++++- model_prices_and_context_window.json | 87 ++++++++++++++++++- 2 files changed, 172 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a5b9a5cb457..708af70985d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -40893,6 +40893,22 @@ "output_cost_per_token": 4e-07, "supports_tool_choice": true }, + "jina-reranker-v3.5": { + "input_cost_per_token": 5e-08, + "litellm_provider": "jina_ai", + "max_input_tokens": 131072, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://api.jina.ai/v1/models" + }, + "jina-reranker-m0": { + "input_cost_per_token": 5e-08, + "litellm_provider": "jina_ai", + "max_input_tokens": 10240, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://api.jina.ai/v1/models" + }, "jina-reranker-v2-base-multilingual": { "input_cost_per_token": 5e-08, "litellm_provider": "jina_ai", @@ -43771,7 +43787,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "mistral/zai-glm-5-3": { "cache_read_input_token_cost": 1.4e-07, @@ -45270,6 +45287,18 @@ "source": "https://tokenfactory.nebius.com/api/public/models_info", "cache_read_input_token_cost": 1.5e-07 }, + "nebius/moonshotai/Kimi-K3.1-Pro": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 2e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" + }, "nebius/moonshotai/Kimi-K3": { "max_input_tokens": 1048576, "max_output_tokens": 1024000, @@ -72148,6 +72177,7 @@ "output_cost_per_token": 4.5e-06, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supports_audio_input": true, + "supports_audio_output": true, "tpm": 250000, "rpm": 10, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, @@ -72170,6 +72200,7 @@ "output_cost_per_token": 4.5e-06, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supports_audio_input": true, + "supports_audio_output": true, "tpm": 250000, "rpm": 10, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, @@ -82782,6 +82813,38 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true }, + "gpt-6-cyber": { + "cache_creation_input_token_cost": 2.5e-05, + "cache_creation_input_token_cost_above_12hr": 5e-05, + "cache_read_input_token_cost": 2e-06, + "input_cost_per_token": 2e-05, + "litellm_provider": "openai", + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.0001, + "source": "https://developers.openai.com/api/docs/models/gpt-6-cyber", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_computer_use": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_search": true, + "supports_vision": true, + "supports_web_search": true + }, "gpt-6-astra-law": { "cache_read_input_token_cost": 1e-06, "input_cost_per_token": 1e-05, @@ -83380,6 +83443,28 @@ "supports_response_schema": true, "source": "https://empiriolabs.ai/models/kimi-k3" }, + "empiriolabs/kimi-k3-1-pro": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.8e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.015, + "search_context_size_medium": 0.015, + "search_context_size_high": 0.015 + }, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_function_calling": true, + "source": "https://empiriolabs.ai/models/kimi-k3-1-pro" + }, "empiriolabs/mimo-v2-6-flash": { "litellm_provider": "empiriolabs", "mode": "chat", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a5b9a5cb457..708af70985d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -40893,6 +40893,22 @@ "output_cost_per_token": 4e-07, "supports_tool_choice": true }, + "jina-reranker-v3.5": { + "input_cost_per_token": 5e-08, + "litellm_provider": "jina_ai", + "max_input_tokens": 131072, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://api.jina.ai/v1/models" + }, + "jina-reranker-m0": { + "input_cost_per_token": 5e-08, + "litellm_provider": "jina_ai", + "max_input_tokens": 10240, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://api.jina.ai/v1/models" + }, "jina-reranker-v2-base-multilingual": { "input_cost_per_token": 5e-08, "litellm_provider": "jina_ai", @@ -43771,7 +43787,8 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "mistral/zai-glm-5-3": { "cache_read_input_token_cost": 1.4e-07, @@ -45270,6 +45287,18 @@ "source": "https://tokenfactory.nebius.com/api/public/models_info", "cache_read_input_token_cost": 1.5e-07 }, + "nebius/moonshotai/Kimi-K3.1-Pro": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 2e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_vision": true, + "supports_prompt_caching": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" + }, "nebius/moonshotai/Kimi-K3": { "max_input_tokens": 1048576, "max_output_tokens": 1024000, @@ -72148,6 +72177,7 @@ "output_cost_per_token": 4.5e-06, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supports_audio_input": true, + "supports_audio_output": true, "tpm": 250000, "rpm": 10, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, @@ -72170,6 +72200,7 @@ "output_cost_per_token": 4.5e-06, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supports_audio_input": true, + "supports_audio_output": true, "tpm": 250000, "rpm": 10, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, @@ -82782,6 +82813,38 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true }, + "gpt-6-cyber": { + "cache_creation_input_token_cost": 2.5e-05, + "cache_creation_input_token_cost_above_12hr": 5e-05, + "cache_read_input_token_cost": 2e-06, + "input_cost_per_token": 2e-05, + "litellm_provider": "openai", + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.0001, + "source": "https://developers.openai.com/api/docs/models/gpt-6-cyber", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_computer_use": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_search": true, + "supports_vision": true, + "supports_web_search": true + }, "gpt-6-astra-law": { "cache_read_input_token_cost": 1e-06, "input_cost_per_token": 1e-05, @@ -83380,6 +83443,28 @@ "supports_response_schema": true, "source": "https://empiriolabs.ai/models/kimi-k3" }, + "empiriolabs/kimi-k3-1-pro": { + "litellm_provider": "empiriolabs", + "mode": "chat", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1.8e-06, + "supports_system_messages": true, + "supports_vision": true, + "supports_reasoning": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.015, + "search_context_size_medium": 0.015, + "search_context_size_high": 0.015 + }, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_function_calling": true, + "source": "https://empiriolabs.ai/models/kimi-k3-1-pro" + }, "empiriolabs/mimo-v2-6-flash": { "litellm_provider": "empiriolabs", "mode": "chat", From 6f9c602650d23bdcc781ceae5f627a0dce830fc7 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 20:16:15 +0000 Subject: [PATCH 03/40] test(skills): assert patch publication and serialize preference writes (#50252) Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ..._claude_code_plugin_installation_preference.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/integration/management/test_claude_code_plugin_installation_preference.py b/tests/integration/management/test_claude_code_plugin_installation_preference.py index aaf6f02ff4e..6d41c690709 100644 --- a/tests/integration/management/test_claude_code_plugin_installation_preference.py +++ b/tests/integration/management/test_claude_code_plugin_installation_preference.py @@ -4,6 +4,7 @@ from collections.abc import Mapping from concurrent.futures import ThreadPoolExecutor from functools import partial from pathlib import Path +from threading import Lock from typing import Final import httpx @@ -248,7 +249,7 @@ def test_duplicate_json_key_keeps_the_last_value(gateway: Gateway) -> None: assert stored_manifest(name)["installation_preference"] == "available" -def test_repeated_identical_update_is_idempotent(gateway: Gateway) -> None: +def test_repeated_identical_updates_publish_patch_versions_without_changing_preference(gateway: Gateway) -> None: name: Final = plugin_name() with gateway.scenario() as scenario: registered(scenario, name, installation_preference="required") @@ -257,7 +258,9 @@ def test_repeated_identical_update_is_idempotent(gateway: Gateway) -> None: replaced(gateway, name, installation_preference="required") second: Final = stored_rows(name) assert len(first) == 1, first - assert second == first + first_manifest: Final = JSON_OBJECT.validate_json(string_value(first[0]["manifest_json"])) + assert first_manifest["version"] == "1.0.1", first + assert second == [{**first[0], "manifest_json": json.dumps({**first_manifest, "version": "1.0.2"})}] assert_preference_served(gateway, name, "required") @@ -442,14 +445,16 @@ def test_burst_reads_across_processes_while_the_preference_flips(gateway: Gatewa name: Final = plugin_name() targets: Final = (gateway, peer) flips: Final = tuple(PREFERENCES[(index + 1) % len(PREFERENCES)] for index in range(13)) + writer: Final = Lock() with gateway.scenario() as scenario: registered(scenario, name, installation_preference="available") def job(index: int) -> tuple[str, JsonValue]: if index % 4 == 3: - return "flip", replace( - gateway, name, replacement(name, installation_preference=flips[index // 4]) - ).status_code + with writer: + return "flip", replace( + gateway, name, replacement(name, installation_preference=flips[index // 4]) + ).status_code return "read", marketplace_entry(targets[index % 2], name).get("installationPreference") with ThreadPoolExecutor(max_workers=8) as pool: From 4cdfaaa334f800b0ce9c9fdb742589a512c0a987 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 20:16:40 +0000 Subject: [PATCH 04/40] test(usage): assert per-user entity spend and metadata (#50251) Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/_support/daily_activity.py | 18 +++++++++++++++++- .../spend/fixtures/daily_activity_user.json | 2 +- .../spend/test_daily_activity_key_owner.py | 2 ++ .../test_daily_activity_key_owner_faults.py | 2 +- 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/integration/_support/daily_activity.py b/tests/integration/_support/daily_activity.py index 346fc156ea9..cedcb80843a 100644 --- a/tests/integration/_support/daily_activity.py +++ b/tests/integration/_support/daily_activity.py @@ -272,16 +272,32 @@ def assert_key_reported( date: str, metadata: Mapping[str, JsonValue], metrics: Mapping[str, float], + *, + entity_rows: Mapping[str, int] | None = None, ) -> None: assert response.status_code == 200, response.text body: Final = object_value(response.json()) records: Final = tuple(object_value(record) for record in records_of_key(body, api_key)) assert records, response.text assert all(record["metadata"] == metadata for record in records), response.text - assert all(counted(record["metrics"]) == pytest.approx(metrics) for record in records), response.text days: Final = body["results"] assert isinstance(days, list) and len(days) == 1, response.text day: Final = object_value(days[0]) + breakdown: Final = object_value(day["breakdown"]) + if entity_rows is None: + assert all(counted(record["metrics"]) == pytest.approx(metrics) for record in records), response.text + else: + totals: Final = records_of_key({name: value for name, value in breakdown.items() if name != "entities"}, api_key) + assert totals, response.text + assert all(counted(object_value(record)["metrics"]) == pytest.approx(metrics) for record in totals), response.text + entities: Final = object_value(breakdown["entities"]) + assert set(entities) == set(entity_rows), response.text + for entity, rows in entity_rows.items(): + bucket: Final = object_value(entities[entity]) + assert counted(bucket["metrics"]) == pytest.approx(seeded_metrics(rows)), response.text + keys: Final = object_value(bucket["api_key_breakdown"]) + assert set(keys) == {api_key}, response.text + assert counted(object_value(keys[api_key])["metrics"]) == pytest.approx(seeded_metrics(rows)), response.text assert day["date"] == date, response.text assert counted(day["metrics"]) == pytest.approx(metrics), response.text assert object_value(body["metadata"])["total_spend"] == pytest.approx(metrics["spend"]), response.text diff --git a/tests/integration/spend/fixtures/daily_activity_user.json b/tests/integration/spend/fixtures/daily_activity_user.json index ae05e661071..058e12a5044 100644 --- a/tests/integration/spend/fixtures/daily_activity_user.json +++ b/tests/integration/spend/fixtures/daily_activity_user.json @@ -1 +1 @@ -{"results":[{"date":"2026-06-01","metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"breakdown":{"mcp_servers":{},"models":{"model-ptu":{"metrics":{"spend":1000.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":0,"completion_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":0,"successful_requests":0,"failed_requests":0,"api_requests":0,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{}},"model-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}},"model-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}},"model-popular":{"metrics":{"spend":270.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":9,"completion_tokens":6,"cache_read_input_tokens":3,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":15,"successful_requests":3,"failed_requests":0,"api_requests":3,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"model_groups":{"model-ptu":{"metrics":{"spend":1000.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":0,"completion_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":0,"successful_requests":0,"failed_requests":0,"api_requests":0,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{}},"model-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}},"model-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}},"model-popular":{"metrics":{"spend":270.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":9,"completion_tokens":6,"cache_read_input_tokens":3,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":15,"successful_requests":3,"failed_requests":0,"api_requests":3,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"providers":{"provider-a":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"endpoints":{"/v1/chat/completions":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"api_keys":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}},"entities":{"user-1":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}}},"guardrails":{}}}],"metadata":{"total_spend":1273.0,"total_flat_cost":0.0,"total_ptu_hours":0.0,"total_prompt_tokens":1014,"total_completion_tokens":10,"total_tokens":1024,"total_api_requests":5,"total_successful_requests":5,"total_failed_requests":0,"total_cache_read_input_tokens":6,"total_cache_creation_input_tokens":0,"total_compression_saved_tokens":0,"total_compression_savings_spend":0.0,"total_prompt_caching_savings_spend":0.0,"total_gateway_injected_caching_savings_spend":0.0,"total_autorouter_savings_spend":0.0,"total_response_time_ms":0,"total_timed_requests":0,"page":1,"total_pages":1,"has_more":false,"api_key_limit":100,"total_api_keys":5,"entity_total_api_keys":{"user-1":5}}} +{"results":[{"date":"2026-06-01","metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"breakdown":{"mcp_servers":{},"models":{"model-ptu":{"metrics":{"spend":1000.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":0,"completion_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":0,"successful_requests":0,"failed_requests":0,"api_requests":0,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{}},"model-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}},"model-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}},"model-popular":{"metrics":{"spend":270.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":9,"completion_tokens":6,"cache_read_input_tokens":3,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":15,"successful_requests":3,"failed_requests":0,"api_requests":3,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"model_groups":{"model-ptu":{"metrics":{"spend":1000.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":0,"completion_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":0,"successful_requests":0,"failed_requests":0,"api_requests":0,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{}},"model-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}},"model-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}},"model-popular":{"metrics":{"spend":270.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":9,"completion_tokens":6,"cache_read_input_tokens":3,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":15,"successful_requests":3,"failed_requests":0,"api_requests":3,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"providers":{"provider-a":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"endpoints":{"/v1/chat/completions":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{},"api_key_breakdown":{"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}}}}},"api_keys":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}},"entities":{"user-1":{"metrics":{"spend":1273.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1014,"completion_tokens":10,"cache_read_input_tokens":6,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1024,"successful_requests":5,"failed_requests":0,"api_requests":5,"total_response_time_ms":0,"timed_requests":0},"metadata":{"user_email":"user@example.com","user_alias":null},"api_key_breakdown":{"key-a":{"metrics":{"spend":100.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":2,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":4,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-a","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-b":{"metrics":{"spend":90.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":3,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":5,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-b","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-c":{"metrics":{"spend":80.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":4,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":6,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-c","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-cache":{"metrics":{"spend":2.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":1000,"completion_tokens":2,"cache_read_input_tokens":1,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":1002,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"alias-cache","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":true}},"key-target":{"metrics":{"spend":1.0,"flat_cost":0.0,"ptu_hours":0.0,"prompt_tokens":5,"completion_tokens":2,"cache_read_input_tokens":2,"cache_creation_input_tokens":0,"compression_saved_tokens":0,"compression_savings_spend":0.0,"prompt_caching_savings_spend":0.0,"gateway_injected_caching_savings_spend":0.0,"autorouter_savings_spend":0.0,"total_tokens":7,"successful_requests":1,"failed_requests":0,"api_requests":1,"total_response_time_ms":0,"timed_requests":0},"metadata":{"key_alias":"deleted-target","team_id":"team-1","user_id":"user-1","user_email":"user@example.com","key_exists":false}}}}},"guardrails":{}}}],"metadata":{"total_spend":1273.0,"total_flat_cost":0.0,"total_ptu_hours":0.0,"total_prompt_tokens":1014,"total_completion_tokens":10,"total_tokens":1024,"total_api_requests":5,"total_successful_requests":5,"total_failed_requests":0,"total_cache_read_input_tokens":6,"total_cache_creation_input_tokens":0,"total_compression_saved_tokens":0,"total_compression_savings_spend":0.0,"total_prompt_caching_savings_spend":0.0,"total_gateway_injected_caching_savings_spend":0.0,"total_autorouter_savings_spend":0.0,"total_response_time_ms":0,"total_timed_requests":0,"page":1,"total_pages":1,"has_more":false,"api_key_limit":100,"total_api_keys":5,"entity_total_api_keys":{"user-1":5}}} diff --git a/tests/integration/spend/test_daily_activity_key_owner.py b/tests/integration/spend/test_daily_activity_key_owner.py index cec19ce5ea0..5a9d8c7467c 100644 --- a/tests/integration/spend/test_daily_activity_key_owner.py +++ b/tests/integration/spend/test_daily_activity_key_owner.py @@ -57,6 +57,7 @@ def test_key_whose_daily_spend_names_two_users_is_reported_with_no_owner(gateway DAY, key_metadata(), seeded_metrics(2), + entity_rows={first: 1, second: 1}, ) @@ -74,6 +75,7 @@ def test_daily_spend_rows_naming_no_user_do_not_hide_the_one_user_the_others_nam DAY, key_metadata(user=owner, email=email), seeded_metrics(2), + entity_rows={owner: 1, "Unassigned": 1}, ) diff --git a/tests/integration/spend/test_daily_activity_key_owner_faults.py b/tests/integration/spend/test_daily_activity_key_owner_faults.py index 1dba97d6ae1..2eeca3c1588 100644 --- a/tests/integration/spend/test_daily_activity_key_owner_faults.py +++ b/tests/integration/spend/test_daily_activity_key_owner_faults.py @@ -199,7 +199,7 @@ def test_key_stops_being_reported_with_an_owner_once_a_second_user_spends_with_i with daily_rows((user_row(second, api_key, DAY),)): shared: Final = activity_of_key(gateway, AGGREGATED_USER_ACTIVITY, api_key) assert_key_reported(alone, api_key, DAY, key_metadata(user=first, email=email), seeded_metrics(1)) - assert_key_reported(shared, api_key, DAY, key_metadata(), seeded_metrics(2)) + assert_key_reported(shared, api_key, DAY, key_metadata(), seeded_metrics(2), entity_rows={first: 1, second: 1}) @pytest.mark.timeout(300) From 26ba02d3ce8a82d5260854c2e4fc32a13c70c3c2 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:20:06 -0800 Subject: [PATCH 05/40] perf(bedrock): avoid copying already isolated schema definitions (#50240) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/prompt_templates/factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 4c3907dc52f..eda98ae2c67 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -5879,7 +5879,7 @@ def _bedrock_tools_pt( description = name defs = parameters.pop("$defs", {}) - defs_copy = copy.deepcopy(defs) if isinstance(defs, dict) else {} + defs_copy = defs if isinstance(defs, dict) else {} # Expand $ref references in parameters using the definitions # Note: We don't pre-flatten defs as that causes exponential memory growth # with circular references (see issue #19098). unpack_defs handles nested From 308fe0792080ee2a87329759d77dafd05e3b74b1 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 20:28:55 +0000 Subject: [PATCH 06/40] fix(openai): add gpt-6-cyber max input tokens and the gpt-5.5-cyber-preview shutdown date (#50254) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 3 +++ model_prices_and_context_window.json | 3 +++ 2 files changed, 6 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 708af70985d..1bda9d24c8e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -71218,6 +71218,8 @@ }, "gpt-5.5-cyber-preview": { "cache_read_input_token_cost": 1.25e-06, + "deprecation_date": "2027-02-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "input_cost_per_token": 1.25e-05, "litellm_provider": "openai", "mode": "chat", @@ -82819,6 +82821,7 @@ "cache_read_input_token_cost": 2e-06, "input_cost_per_token": 2e-05, "litellm_provider": "openai", + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 0.0001, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 708af70985d..1bda9d24c8e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -71218,6 +71218,8 @@ }, "gpt-5.5-cyber-preview": { "cache_read_input_token_cost": 1.25e-06, + "deprecation_date": "2027-02-01", + "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "input_cost_per_token": 1.25e-05, "litellm_provider": "openai", "mode": "chat", @@ -82819,6 +82821,7 @@ "cache_read_input_token_cost": 2e-06, "input_cost_per_token": 2e-05, "litellm_provider": "openai", + "max_input_tokens": 922000, "max_output_tokens": 128000, "mode": "chat", "output_cost_per_token": 0.0001, From f069da8e0ead97381775a6dce8af3675f43fbe13 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 5 Nov 2026 12:38:58 -0800 Subject: [PATCH 07/40] fix(anthropic): report temperature/top_p as supported only when the model keeps them (#50144) * fix(anthropic): report temperature/top_p as supported only when the model keeps them Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(vertex_ai): accept the generic unsupported-param message for Claude batch rows Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/anthropic/chat/transformation.py | 4 ++-- .../test_anthropic_chat_transformation.py | 21 +++++++++++++++++++ .../test_vertex_ai_files_transformation.py | 2 +- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 4f890ed922f..ac4addb0847 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -827,11 +827,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return cls._raise_invalid_reasoning_effort(model, value, llm_provider) def get_supported_openai_params(self, model: str): + sampling_params: Final = ("temperature", "top_p") if self._supports_sampling_params(model) else () params: Final = [ "stream", "stop", - "temperature", - "top_p", + *sampling_params, "max_tokens", "max_completion_tokens", "tools", diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 80845b1f12d..3a4231443dd 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -11047,3 +11047,24 @@ def test_claude_cache_control_cap_does_not_count_a_message_level_hint_that_list_ assert sorted(_cached_texts(body)) == ["system prompt", "turn 1", "turn 2", "turn 3"] assert json.dumps(body).count('"cache_control"') == 4 + + +@pytest.mark.parametrize("model", ["claude-opus-4-7", "claude-haiku-4-5-20251001"]) +def test_get_supported_params_reports_sampling_params_only_when_the_request_keeps_them( + local_model_cost_map, model: str +): + """What get_supported_openai_params says about temperature/top_p must match what + map_openai_params does with them under drop_params, so a caller that pre-flights + from the list is never told a param is fine that the request path then drops.""" + config = AnthropicConfig() + + supported = config.get_supported_openai_params(model=model) + sent = config.map_openai_params( + non_default_params={"temperature": 0.1, "top_p": 0.5}, + optional_params={}, + model=model, + drop_params=True, + ) + + assert ("temperature" in supported) == ("temperature" in sent), (model, supported, sent) + assert ("top_p" in supported) == ("top_p" in sent), (model, supported, sent) diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index 98e204dd44e..842500a7b27 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -2161,7 +2161,7 @@ class TestVertexClaudeBatch: monkeypatch.setattr(litellm, "drop_params", False) entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, top_p=0.5) - with pytest.raises(VertexAIError, match=r"line 1 .*top_p=0.5.*drop_params") as exc_info: + with pytest.raises(VertexAIError, match=r"line 1 .*top_p.*drop_params") as exc_info: _uploaded_rows(config, {**CLAUDE_DEPLOYMENT, "model": "vertex_ai/claude-zeta-9"}, [entry]) assert exc_info.value.status_code == 400 From 41d45079c7b0b71caac21566f7948c7ad269e339 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 5 Nov 2026 12:42:14 -0800 Subject: [PATCH 08/40] refactor(ui): move remaining logs and usage text onto i18n (#50146) Logs and usage option lists, filter chip labels and label maps now hold message keys that their readers translate at render time. Tile titles, the "Claude (unspecified)" agent label and the endpoint chart categories stay English ids and get display maps keyed by them. Lookups keyed by URL or server values check Object.hasOwn first. --- .../components/EndpointUsageBarChart.tsx | 7 +- .../components/EntityUsage/EntityUsage.tsx | 19 +-- .../components/EntityUsage/TopModelView.tsx | 9 +- .../EntityUsage/entityUsageSummary.ts | 12 ++ .../components/errors/ErrorsTab.tsx | 12 +- .../components/errors/errorsData.ts | 16 +-- .../components/overview/BreakdownChart.tsx | 21 +-- .../components/overview/TopAgents.tsx | 30 ++-- .../overview/UsageBreakdownChart.tsx | 16 +-- .../components/overview/agentCatalog.ts | 9 +- .../_components/hooks/clientUsageData.test.ts | 3 +- .../_components/hooks/clientUsageData.ts | 6 +- .../components/EntityUsage/TopKeyView.tsx | 8 +- .../components/chat_ui/ResponseMetrics.tsx | 6 +- .../components/logs/audit/AuditLogsTable.tsx | 83 ++++++----- .../src/components/logs/constants.ts | 56 ++++---- .../logs/detail/LogDetailContent.tsx | 6 +- .../detail/sections/ToolsSection/utils.ts | 3 +- .../logs/request/LogsTableToolbar.tsx | 6 +- .../logs/request/RequestLogsFilters.tsx | 84 ++++++----- .../logs/request/RequestLogsTable.tsx | 16 ++- .../logs/request/useLogFilterLogic.ts | 33 ++--- .../src/components/user_agent_activity.tsx | 9 +- ui/litellm-dashboard/src/i18n/locales/en.json | 134 ++++++++++++++++++ .../src/utils/promptCacheUsage.ts | 8 +- 25 files changed, 411 insertions(+), 201 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EndpointUsage/components/EndpointUsageBarChart.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EndpointUsage/components/EndpointUsageBarChart.tsx index 687571e70be..198d501ba41 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EndpointUsage/components/EndpointUsageBarChart.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EndpointUsage/components/EndpointUsageBarChart.tsx @@ -1,4 +1,5 @@ import React from "react"; +import type { ParseKeys } from "i18next"; import { useTranslation } from "react-i18next"; import { ChartColumnStacked } from "lucide-react"; import { BarChart, CustomTooltip } from "@/components/shared/charts"; @@ -12,6 +13,10 @@ interface EndpointUsageBarChartProps { const SUCCESS_COLOR = "#2b3fd6"; const FAILED_COLOR = "#ef4444"; const CATEGORIES = ["Successful", "Failed"] as const; +const CATEGORY_LABELS: Record<(typeof CATEGORIES)[number], ParseKeys> = { + Successful: "usage.endpointUsage.barChart.successful", + Failed: "usage.endpointUsage.barChart.failed", +}; const COLORS = [SUCCESS_COLOR, FAILED_COLOR] as const; const valueFormatter = (value: number) => value.toLocaleString(); @@ -45,7 +50,7 @@ const EndpointUsageBarChart: React.FC = ({ endpointD {CATEGORIES.map((category, i) => ( ))} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx index 96656225b30..f12f59f9c28 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx @@ -7,6 +7,7 @@ import { buildSummaryTiles, hasFlatCost, hasPtuHours, + SUMMARY_TILE_LABELS, type SummaryTile, } from "./entityUsageSummary"; import { MoneyCell } from "@/components/shared/table_cells"; @@ -74,13 +75,13 @@ const TILE_TRENDS: Readonly> = { "Total Tokens": "tokens", }; const GRANULARITY_OPTIONS = [ - { value: "day", label: "Daily" }, - { value: "week", label: "Weekly" }, -] as const satisfies readonly { value: Granularity; label: string }[]; + { value: "day", labelKey: "usage.entityUsage.chart.granularityOptions.day" }, + { value: "week", labelKey: "usage.entityUsage.chart.granularityOptions.week" }, +] as const satisfies readonly { value: Granularity; labelKey: ParseKeys }[]; const SCALE_OPTIONS = [ - { value: "linear", label: "Linear" }, - { value: "log", label: "Log" }, -] as const satisfies readonly { value: StackedUsageScale; label: string }[]; + { value: "linear", labelKey: "usage.entityUsage.chart.scaleOptions.linear" }, + { value: "log", labelKey: "usage.entityUsage.chart.scaleOptions.log" }, +] as const satisfies readonly { value: StackedUsageScale; labelKey: ParseKeys }[]; const QUIET_HEADER = { headerClassName: "font-normal" }; const FLAT_COST_KEY = "flat_cost"; @@ -433,7 +434,7 @@ const EntityUsage: React.FC = ({ const tileLabel = ({ title, tooltip, expandable }: SummaryTile) => ( <> - {title} + {Object.hasOwn(SUMMARY_TILE_LABELS, title) ? t(SUMMARY_TILE_LABELS[title]) : title} {tooltip ? ( } /> @@ -540,13 +541,13 @@ const EntityUsage: React.FC = ({ ({ value, label: t(labelKey) }))} onChange={setGranularity} /> ({ value, label: t(labelKey) }))} onChange={setScale} /> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/TopModelView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/TopModelView.tsx index ae98dd835b5..c8e42c9a6aa 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/TopModelView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/TopModelView.tsx @@ -2,6 +2,7 @@ import { BarChart } from "@/components/shared/charts"; import { DataTable } from "@/components/shared/DataTable"; import { MoneyCell } from "@/components/shared/table_cells"; import { ProviderLogo } from "@/components/molecules/models/ProviderLogo"; +import type { ParseKeys } from "i18next"; import { useState } from "react"; import { useTranslation } from "react-i18next"; import { formatNumberWithCommas } from "@/utils/dataUtils"; @@ -28,9 +29,9 @@ type ViewMode = "chart" | "table"; const LIMIT_OPTIONS = TOP_MODEL_LIMITS.map((limit) => ({ value: String(limit), label: String(limit) })); const VIEW_OPTIONS = [ - { value: "table", label: "Table View" }, - { value: "chart", label: "Chart View" }, -] as const satisfies readonly { value: ViewMode; label: string }[]; + { value: "table", labelKey: "usage.topModelView.viewOptions.table" }, + { value: "chart", labelKey: "usage.topModelView.viewOptions.chart" }, +] as const satisfies readonly { value: ViewMode; labelKey: ParseKeys }[]; const BAR_COLOR = "#2b3fd6"; const QUIET_HEADER = { headerClassName: "font-normal" }; @@ -100,7 +101,7 @@ export default function TopModelView({ topModels, topModelsLimit, setTopModelsLi ({ value, label: t(labelKey) }))} onChange={setModelViewMode} /> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/entityUsageSummary.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/entityUsageSummary.ts index 60084f7ef66..bf7c272347e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/entityUsageSummary.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/entityUsageSummary.ts @@ -20,6 +20,18 @@ interface SpendSummaryMetadata { total_tokens: number; } +export const SUMMARY_TILE_LABELS: Readonly> = { + "PTU Hours": "usage.entitySummary.titles.ptuHours", + "Total Cost": "usage.entitySummary.titles.totalCost", + "Total Spend": "usage.entitySummary.titles.totalSpend", + "Total Requests": "usage.entitySummary.titles.totalRequests", + "Successful Requests": "usage.entitySummary.titles.successfulRequests", + "Failed Requests": "usage.entitySummary.titles.failedRequests", + "Total Tokens": "usage.entitySummary.titles.totalTokens", + "Request Cost": "usage.entitySummary.titles.requestCost", + "Flat Cost": "usage.entitySummary.titles.flatCost", +}; + export const TOTAL_COST_TOOLTIP: ParseKeys = "usage.entitySummary.tooltips.totalCost"; export const REQUEST_COST_TOOLTIP: ParseKeys = "usage.entitySummary.tooltips.requestCost"; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/errors/ErrorsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/errors/ErrorsTab.tsx index 8f1cc80a81d..8f62e941a73 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/errors/ErrorsTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/errors/ErrorsTab.tsx @@ -35,7 +35,7 @@ interface ErrorsTabProps { userScoped?: boolean; } -const ENTITY_OPTIONS = ERROR_ENTITY_KINDS.map((kind) => ({ value: kind, label: ERROR_ENTITY_LABEL[kind].plural })); +const ENTITY_OPTIONS = ERROR_ENTITY_KINDS.map((kind) => ({ value: kind, labelKey: ERROR_ENTITY_LABEL[kind].plural })); const shortDate = (iso: string): string => new Date(`${iso}T00:00:00`).toLocaleDateString(undefined, { month: "short", day: "numeric" }); @@ -187,7 +187,7 @@ export default function ErrorsTab({ activity, loading, failed, userScoped = fals ({ value, label: t(labelKey) }))} onChange={setRateView} /> } @@ -280,7 +280,7 @@ export default function ErrorsTab({ activity, loading, failed, userScoped = fals ({ value, label: t(labelKey) }))} onChange={setKind} /> } @@ -290,14 +290,14 @@ export default function ErrorsTab({ activity, loading, failed, userScoped = fals height="h-48" empty={ rows.length === 0 - ? t("usage.errors.entities.empty", { entities: ERROR_ENTITY_LABEL[kind].plural.toLowerCase() }) + ? t("usage.errors.entities.empty", { entities: t(ERROR_ENTITY_LABEL[kind].plural).toLowerCase() }) : null } > - +
- {ERROR_ENTITY_LABEL[kind].singular} + {t(ERROR_ENTITY_LABEL[kind].singular)} {t("usage.errors.entities.columns.requests")} {t("usage.errors.entities.columns.failed")} {t("usage.errors.entities.columns.errorRate")} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/errors/errorsData.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/errors/errorsData.ts index c45524b2add..66b69eacaff 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/errors/errorsData.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/errors/errorsData.ts @@ -33,11 +33,11 @@ export interface RequestErrorActivity { export const ERROR_ENTITY_KINDS = ["key", "team", "user", "model"] as const; export type ErrorEntityKind = (typeof ERROR_ENTITY_KINDS)[number]; -export const ERROR_ENTITY_LABEL: Record = { - key: { singular: "Virtual key", plural: "Virtual keys" }, - team: { singular: "Team", plural: "Teams" }, - user: { singular: "User", plural: "Users" }, - model: { singular: "Model", plural: "Models" }, +export const ERROR_ENTITY_LABEL: Record = { + key: { singular: "usage.errors.entityLabels.key.singular", plural: "usage.errors.entityLabels.key.plural" }, + team: { singular: "usage.errors.entityLabels.team.singular", plural: "usage.errors.entityLabels.team.plural" }, + user: { singular: "usage.errors.entityLabels.user.singular", plural: "usage.errors.entityLabels.user.plural" }, + model: { singular: "usage.errors.entityLabels.model.singular", plural: "usage.errors.entityLabels.model.plural" }, }; const STATUS_LABELS: Record = { @@ -153,9 +153,9 @@ export const errorSeries = (activity: RequestErrorActivity): ErrorSeries => { export type ErrorRateView = "total" | "status"; -export const ERROR_RATE_VIEW_OPTIONS: readonly { value: ErrorRateView; label: string }[] = [ - { value: "total", label: "Total" }, - { value: "status", label: "By status code" }, +export const ERROR_RATE_VIEW_OPTIONS: readonly { value: ErrorRateView; labelKey: ParseKeys }[] = [ + { value: "total", labelKey: "usage.errors.rateViews.total" }, + { value: "status", labelKey: "usage.errors.rateViews.status" }, ]; export const rateByStatusRows = (series: ErrorSeries): ErrorSeriesRow[] => diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/BreakdownChart.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/BreakdownChart.tsx index f0a346dc8da..79c64e6079b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/BreakdownChart.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/BreakdownChart.tsx @@ -1,6 +1,7 @@ "use client"; import React, { useMemo } from "react"; +import type { ParseKeys } from "i18next"; import { useTranslation } from "react-i18next"; import { ArrowDownRight, ArrowUpRight } from "lucide-react"; import { ProviderLogo } from "@/components/molecules/models/ProviderLogo"; @@ -22,16 +23,16 @@ import { Segmented } from "./Primitives"; import { providerForModel } from "./modelProvider"; const METRIC_OPTIONS = [ - { value: "spend", label: "Spend" }, - { value: "tokens", label: "Tokens" }, - { value: "requests", label: "Requests" }, -] as const satisfies readonly { value: UsageMetric; label: string }[]; + { value: "spend", labelKey: "usage.breakdown.metricOptions.spend" }, + { value: "tokens", labelKey: "usage.breakdown.metricOptions.tokens" }, + { value: "requests", labelKey: "usage.breakdown.metricOptions.requests" }, +] as const satisfies readonly { value: UsageMetric; labelKey: ParseKeys }[]; const DIMENSION_OPTIONS = [ - { value: "model_groups", label: "Model" }, - { value: "models", label: "Deployment" }, - { value: "providers", label: "Provider" }, -] as const satisfies readonly { value: BreakdownDimension; label: string }[]; + { value: "model_groups", labelKey: "usage.breakdown.dimensionOptions.modelGroups" }, + { value: "models", labelKey: "usage.breakdown.dimensionOptions.models" }, + { value: "providers", labelKey: "usage.breakdown.dimensionOptions.providers" }, +] as const satisfies readonly { value: BreakdownDimension; labelKey: ParseKeys }[]; export interface BreakdownState { metric: UsageMetric; @@ -67,14 +68,14 @@ export function BreakdownControls({ ({ value, label: t(labelKey) }))} onChange={(metric) => onChange({ ...state, metric })} /> {showDimension && ( ({ value, label: t(labelKey) }))} onChange={(dimension) => onChange({ ...state, dimension })} /> )} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/TopAgents.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/TopAgents.tsx index 68ed202a8dc..d5d69216523 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/TopAgents.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/TopAgents.tsx @@ -2,6 +2,7 @@ import { useTerminology } from "@/contexts/TerminologyContext"; import React, { useMemo, useState } from "react"; +import type { ParseKeys } from "i18next"; import { useTranslation } from "react-i18next"; import { ArrowRight, Bot, Monitor, Terminal } from "lucide-react"; import { StackedUsageChart } from "@/components/shared/charts"; @@ -18,17 +19,17 @@ import { type UsageMetric, } from "./overviewData"; import { ChartSkeleton, Panel, PANEL_INSET_X, Segmented } from "./Primitives"; -import { agentDailyData, topAgents, type AgentRow, type TagSummaryRow } from "./agentCatalog"; +import { agentDailyData, agentDisplayLabel, topAgents, type AgentRow, type TagSummaryRow } from "./agentCatalog"; const ROWS = 10; const CHART_SERIES = 6; /** Below this share of tokens carrying a User-Agent, say so rather than imply the list is the whole picture. */ const COVERAGE_NOTE_BELOW = 0.5; const METRICS = [ - { value: "tokens", label: "Tokens" }, - { value: "spend", label: "Spend" }, - { value: "requests", label: "Requests" }, -] as const satisfies readonly { value: UsageMetric; label: string }[]; + { value: "tokens", labelKey: "usage.topAgents.metricOptions.tokens" }, + { value: "spend", labelKey: "usage.topAgents.metricOptions.spend" }, + { value: "requests", labelKey: "usage.topAgents.metricOptions.requests" }, +] as const satisfies readonly { value: UsageMetric; labelKey: ParseKeys }[]; /** Brand-tinted monograms for runtimes that have no logo file, so a top spender never shows a blank tile. */ const MONOGRAMS: Readonly> = { @@ -42,7 +43,9 @@ const CLIENT_SURFACES: Readonly>> = { }; function AgentGlyph({ agent }: { agent: Pick }) { - if (agent.logo) return ; + const { t } = useTranslation(); + const label = agentDisplayLabel(agent.label, t); + if (agent.logo) return ; const monogram = MONOGRAMS[agent.id]; if (monogram) { return ( @@ -56,7 +59,7 @@ function AgentGlyph({ agent }: { agent: Pick ); } - return {agent.label.charAt(0)}; + return {label.charAt(0)}; } export function AgentMark({ agent }: { agent: Pick }) { @@ -103,7 +106,7 @@ function AgentItem({ )} - {agent.label} + {agentDisplayLabel(agent.label, t)} {openRow ? ( {t("usage.topAgents.viewActivity")} @@ -130,7 +133,7 @@ function AgentItem({ {openRow ? ( ))}
diff --git a/ui/litellm-dashboard/src/components/logs/request/RequestLogsFilters.tsx b/ui/litellm-dashboard/src/components/logs/request/RequestLogsFilters.tsx index f444d21d6e8..26f84386c0c 100644 --- a/ui/litellm-dashboard/src/components/logs/request/RequestLogsFilters.tsx +++ b/ui/litellm-dashboard/src/components/logs/request/RequestLogsFilters.tsx @@ -1,5 +1,6 @@ "use client"; +import type { ParseKeys } from "i18next"; import { useMemo, useState } from "react"; import { useTranslation } from "react-i18next"; @@ -27,30 +28,35 @@ import { LOG_FILTER_IDS, type LogsWindow } from "./useLogFilterLogic"; const ALL_VALUE = "all"; -const STATUS_FILTER_ITEMS = [ - { value: ALL_VALUE, label: "All Statuses" }, - { value: "success", label: "Success" }, - { value: "failure", label: "Failure" }, -] as const; +interface FilterItem { + value: string; + labelKey: ParseKeys; +} -const CACHE_FILTER_ITEMS = [ - { value: ALL_VALUE, label: "All Requests" }, - { value: "hit", label: "Cache Hit" }, - { value: "miss", label: "Cache Miss" }, -] as const; +const STATUS_FILTER_ITEMS: readonly FilterItem[] = [ + { value: ALL_VALUE, labelKey: "logs.request.filters.allStatuses" }, + { value: "success", labelKey: "logs.request.filters.statusOptions.success" }, + { value: "failure", labelKey: "logs.request.filters.statusOptions.failure" }, +]; -const CREDENTIAL_FILTER_ITEMS = [ - { value: ALL_VALUE, label: "All Credentials" }, - ...Object.entries(CREDENTIAL_LABELS).map(([value, label]) => ({ value, label })), -] as const; +const CACHE_FILTER_ITEMS: readonly FilterItem[] = [ + { value: ALL_VALUE, labelKey: "logs.request.filters.allRequests" }, + { value: "hit", labelKey: "logs.request.filters.cacheOptions.hit" }, + { value: "miss", labelKey: "logs.request.filters.cacheOptions.miss" }, +]; -const SPAN_TYPE_FILTER_ITEMS = [ - { value: ALL_VALUE, label: "All Types" }, - { value: "llm", label: "LLM" }, - { value: "agent", label: "Agent" }, - { value: "mcp", label: "MCP" }, - { value: "batch", label: "Batch" }, -] as const; +const CREDENTIAL_FILTER_ITEMS: readonly FilterItem[] = [ + { value: ALL_VALUE, labelKey: "logs.request.filters.allCredentials" }, + ...Object.entries(CREDENTIAL_LABELS).map(([value, labelKey]) => ({ value, labelKey })), +]; + +const SPAN_TYPE_FILTER_ITEMS: readonly FilterItem[] = [ + { value: ALL_VALUE, labelKey: "logs.request.filters.allTypes" }, + { value: "llm", labelKey: "logs.common.spanTypes.llm" }, + { value: "agent", labelKey: "logs.common.spanTypes.agent" }, + { value: "mcp", labelKey: "logs.common.spanTypes.mcp" }, + { value: "batch", labelKey: "logs.common.spanTypes.batch" }, +]; const PAGE_SIZE = 50; const SEARCH_INPUT_REASONS: ReadonlySet = new Set(["input-change", "input-clear", "clear-press"]); @@ -278,21 +284,26 @@ function ErrorCodeFilterField({ value, onChange }: { value: string; onChange: (v const { t } = useTranslation(); const [query, setQuery] = useState(""); + const errorCodeOptions = useMemo( + () => ERROR_CODE_OPTIONS.map(({ labelKey, value: code }) => ({ label: t(labelKey), value: code })), + [t], + ); + const options = useMemo(() => { const trimmed = query.trim(); const lowered = trimmed.toLowerCase(); - const matches = ERROR_CODE_OPTIONS.filter((option) => option.label.toLowerCase().includes(lowered)); - const isKnownCode = ERROR_CODE_OPTIONS.some( + const matches = errorCodeOptions.filter((option) => option.label.toLowerCase().includes(lowered)); + const isKnownCode = errorCodeOptions.some( (option) => option.value === trimmed || option.label.toLowerCase() === lowered, ); if (trimmed === "" || isKnownCode) return matches; return [...matches, { label: t("logs.request.filters.customErrorCode", { code: trimmed }), value: trimmed }]; - }, [query, t]); + }, [query, t, errorCodeOptions]); const selected = useMemo(() => { if (value === "") return null; - return ERROR_CODE_OPTIONS.find((option) => option.value === value) ?? { label: value, value }; - }, [value]); + return errorCodeOptions.find((option) => option.value === value) ?? { label: value, value }; + }, [value, errorCodeOptions]); const items = useMemo(() => { if (selected === null) return options; @@ -346,6 +357,11 @@ export function RequestLogsFilters({ get, set, teams, logsWindow }: RequestLogsF const { t } = useTranslation(); const valueOf = (id: string): string => asString(get(id)); const setter = (id: string) => (next: string | undefined) => set(id, next); + const withLabel = ({ value, labelKey }: FilterItem) => ({ value, label: t(labelKey) }); + const spanTypeItems = SPAN_TYPE_FILTER_ITEMS.map(withLabel); + const statusItems = STATUS_FILTER_ITEMS.map(withLabel); + const cacheItems = CACHE_FILTER_ITEMS.map(withLabel); + const credentialItems = CREDENTIAL_FILTER_ITEMS.map(withLabel); return ( <> @@ -357,7 +373,7 @@ export function RequestLogsFilters({ get, set, teams, logsWindow }: RequestLogsF set(LOG_FILTER_IDS.STATUS, next === null || next === ALL_VALUE ? undefined : next)} > @@ -386,7 +402,7 @@ export function RequestLogsFilters({ get, set, teams, logsWindow }: RequestLogsF - {STATUS_FILTER_ITEMS.map((item) => ( + {statusItems.map((item) => ( {item.label} @@ -397,7 +413,7 @@ export function RequestLogsFilters({ get, set, teams, logsWindow }: RequestLogsF set(LOG_FILTER_IDS.CREDENTIAL, next === null || next === ALL_VALUE ? undefined : next) @@ -428,7 +444,7 @@ export function RequestLogsFilters({ get, set, teams, logsWindow }: RequestLogsF - {CREDENTIAL_FILTER_ITEMS.map((item) => ( + {credentialItems.map((item) => ( {item.label} diff --git a/ui/litellm-dashboard/src/components/logs/request/RequestLogsTable.tsx b/ui/litellm-dashboard/src/components/logs/request/RequestLogsTable.tsx index 31b494448c2..01314fe3221 100644 --- a/ui/litellm-dashboard/src/components/logs/request/RequestLogsTable.tsx +++ b/ui/litellm-dashboard/src/components/logs/request/RequestLogsTable.tsx @@ -1,6 +1,7 @@ "use client"; import type { ColumnFiltersState, OnChangeFn, PaginationState, SortingState } from "@tanstack/react-table"; +import type { TFunction } from "i18next"; import { ScrollText } from "lucide-react"; import { useMemo, useState, type ReactNode } from "react"; import { useTranslation } from "react-i18next"; @@ -39,12 +40,12 @@ interface RequestLogsTableProps { toolbarChildren?: ReactNode; } -const formatFilterValue = (columnId: string, value: unknown): string => { +const formatFilterValue = (columnId: string, value: unknown, t: TFunction): string => { if (columnId === LOG_FILTER_IDS.SPAN_TYPE) { - return SPAN_TYPE_LABELS[String(value)] ?? String(value); + return Object.hasOwn(SPAN_TYPE_LABELS, String(value)) ? t(SPAN_TYPE_LABELS[String(value)]) : String(value); } if (columnId === LOG_FILTER_IDS.CREDENTIAL) { - return CREDENTIAL_LABELS[String(value)] ?? String(value); + return Object.hasOwn(CREDENTIAL_LABELS, String(value)) ? t(CREDENTIAL_LABELS[String(value)]) : String(value); } return Array.isArray(value) ? value.join(", ") : String(value); }; @@ -100,6 +101,11 @@ export function RequestLogsTable({ return getRequestLogsTableColumns({ onKeyHashClick, onSessionClick, resolveUserEmail }, t); }, [onKeyHashClick, onSessionClick, emailByUserId, t]); + const filterLabels = useMemo( + () => Object.fromEntries(Object.entries(LOG_FILTER_LABELS).map(([id, labelKey]) => [id, t(labelKey)])), + [t], + ); + const isFiltered = columnFilters.length > 0 || searchValue !== ""; return ( @@ -136,8 +142,8 @@ export function RequestLogsTable({ onRefresh={onRefresh} isRefreshing={isRefreshing} onOpenFilters={() => setFiltersOpen(true)} - filterLabels={LOG_FILTER_LABELS} - formatFilterValue={formatFilterValue} + filterLabels={filterLabels} + formatFilterValue={(columnId, value) => formatFilterValue(columnId, value, t)} showViewOptions={false} > {toolbarChildren} diff --git a/ui/litellm-dashboard/src/components/logs/request/useLogFilterLogic.ts b/ui/litellm-dashboard/src/components/logs/request/useLogFilterLogic.ts index 17d7eb4b860..f50780539ac 100644 --- a/ui/litellm-dashboard/src/components/logs/request/useLogFilterLogic.ts +++ b/ui/litellm-dashboard/src/components/logs/request/useLogFilterLogic.ts @@ -1,4 +1,5 @@ import moment from "moment"; +import type { ParseKeys } from "i18next"; import { keepPreviousData, useQuery, type UseQueryOptions } from "@tanstack/react-query"; import type { ColumnFiltersState, PaginationState, SortingState } from "@tanstack/react-table"; import { uiSpendLogsCall } from "../../networking"; @@ -38,22 +39,22 @@ export const LOG_FILTER_IDS = { SEARCH: "search", } as const; -export const LOG_FILTER_LABELS: Record = { - [LOG_FILTER_IDS.TEAM_ID]: "Team ID", - [LOG_FILTER_IDS.SPAN_TYPE]: "Span Type", - [LOG_FILTER_IDS.STATUS]: "Status", - [LOG_FILTER_IDS.CACHE_STATUS]: "Cache", - [LOG_FILTER_IDS.CREDENTIAL]: "Credential", - [LOG_FILTER_IDS.KEY_ALIAS]: "Key Alias", - [LOG_FILTER_IDS.USER_ID]: "User ID", - [LOG_FILTER_IDS.END_USER]: "End User", - [LOG_FILTER_IDS.ERROR_CODE]: "Error Code", - [LOG_FILTER_IDS.ERROR_MESSAGE]: "Error Message", - [LOG_FILTER_IDS.KEY_HASH]: "Key Hash", - [LOG_FILTER_IDS.SESSION_ID]: "Session ID", - [LOG_FILTER_IDS.MODEL_ID]: "Model", - [LOG_FILTER_IDS.PUBLIC_MODEL_OR_SEARCH_TOOL]: "Public model / search tool", - [LOG_FILTER_IDS.SEARCH]: "Search", +export const LOG_FILTER_LABELS: Record = { + [LOG_FILTER_IDS.TEAM_ID]: "logs.request.filters.teamId", + [LOG_FILTER_IDS.SPAN_TYPE]: "logs.request.filters.spanType", + [LOG_FILTER_IDS.STATUS]: "logs.request.filters.status", + [LOG_FILTER_IDS.CACHE_STATUS]: "logs.request.filters.cache", + [LOG_FILTER_IDS.CREDENTIAL]: "logs.request.filters.credential", + [LOG_FILTER_IDS.KEY_ALIAS]: "logs.request.filters.keyAlias", + [LOG_FILTER_IDS.USER_ID]: "logs.request.filters.userId", + [LOG_FILTER_IDS.END_USER]: "logs.request.filters.endUser", + [LOG_FILTER_IDS.ERROR_CODE]: "logs.request.filters.errorCode", + [LOG_FILTER_IDS.ERROR_MESSAGE]: "logs.request.filters.errorMessage", + [LOG_FILTER_IDS.KEY_HASH]: "logs.request.filters.keyHash", + [LOG_FILTER_IDS.SESSION_ID]: "logs.request.filters.sessionId", + [LOG_FILTER_IDS.MODEL_ID]: "logs.request.filters.model", + [LOG_FILTER_IDS.PUBLIC_MODEL_OR_SEARCH_TOOL]: "logs.request.filters.publicModelOrSearchTool", + [LOG_FILTER_IDS.SEARCH]: "logs.request.filters.search", }; export interface LogsWindow { diff --git a/ui/litellm-dashboard/src/components/user_agent_activity.tsx b/ui/litellm-dashboard/src/components/user_agent_activity.tsx index 27e9f7373dc..6904d6b323d 100644 --- a/ui/litellm-dashboard/src/components/user_agent_activity.tsx +++ b/ui/litellm-dashboard/src/components/user_agent_activity.tsx @@ -1,3 +1,4 @@ +import type { ParseKeys } from "i18next"; import React, { useState, useEffect } from "react"; import { useTranslation } from "react-i18next"; import { Bot, Users } from "lucide-react"; @@ -25,9 +26,9 @@ type ActivityView = "active-users" | "per-user"; type ActivePeriod = "dau" | "wau" | "mau"; const VIEW_OPTIONS = [ - { value: "active-users", label: "Active users" }, - { value: "per-user", label: "Per user" }, -] as const satisfies readonly { value: ActivityView; label: string }[]; + { value: "active-users", labelKey: "usage.userAgentActivity.viewOptions.activeUsers" }, + { value: "per-user", labelKey: "usage.userAgentActivity.viewOptions.perUser" }, +] as const satisfies readonly { value: ActivityView; labelKey: ParseKeys }[]; const PERIOD_OPTIONS = [ { value: "dau", label: "DAU" }, @@ -576,7 +577,7 @@ const UserAgentActivity: React.FC = ({ ({ value, label: t(labelKey) }))} onChange={setActivityView} /> diff --git a/ui/litellm-dashboard/src/i18n/locales/en.json b/ui/litellm-dashboard/src/i18n/locales/en.json index f9a7882b2de..2f61c51d819 100644 --- a/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/ui/litellm-dashboard/src/i18n/locales/en.json @@ -12972,6 +12972,37 @@ }, "logs": { "common": { + "errorCodes": { + "badRequest": "400 - Bad Request", + "invalidAuthentication": "401 - Invalid Authentication", + "permissionDenied": "403 - Permission Denied", + "notFound": "404 - Not Found", + "requestTimeout": "408 - Request Timeout", + "unprocessableEntity": "422 - Unprocessable Entity", + "rateLimited": "429 - Rate Limited", + "internalServerError": "500 - Internal Server Error", + "badGateway": "502 - Bad Gateway", + "serviceUnavailable": "503 - Service Unavailable", + "overloaded": "529 - Overloaded" + }, + "spanTypes": { + "llm": "LLM", + "agent": "Agent", + "mcp": "MCP", + "batch": "Batch" + }, + "credentials": { + "clientOauthToken": "Client OAuth token", + "configuredKey": "Configured key" + }, + "quickSelect": { + "lastMinute": "Last Minute", + "last15Minutes": "Last 15 Minutes", + "lastHour": "Last Hour", + "last4Hours": "Last 4 Hours", + "last24Hours": "Last 24 Hours", + "last7Days": "Last 7 Days" + }, "podRouting": { "roundRobin": "Round robin", "sessionAffinity": "Session affinity", @@ -13377,6 +13408,7 @@ "note": "Note: This will only affect new requests after the configuration change." }, "tools": { + "fallbackName": "Tool {{index}}", "title": "Tools", "summary": "{{provided}} provided, {{called}} called", "called": "called", @@ -13509,6 +13541,15 @@ "filtersDescription": "Narrow down request logs" }, "filters": { + "search": "Search", + "statusOptions": { + "success": "Success", + "failure": "Failure" + }, + "cacheOptions": { + "hit": "Cache Hit", + "miss": "Cache Miss" + }, "teamId": "Team ID", "teamPlaceholder": "Search or select a team", "noTeams": "No teams found", @@ -13589,6 +13630,13 @@ } }, "audit": { + "actionOptions": { + "created": "Created", + "updated": "Updated", + "deleted": "Deleted", + "rotated": "Rotated", + "killSwitchFired": "Kill switch fired" + }, "panel": { "enterpriseTitle": "✨ Enterprise Feature.", "enterpriseDescription": "This is a LiteLLM Enterprise feature, and requires a valid key to use.", @@ -13973,6 +14021,14 @@ "weekOf": "Wk of {{date}}" }, "breakdownChart": { + "granularityOptions": { + "day": "Daily", + "week": "Weekly" + }, + "scaleOptions": { + "linear": "Linear", + "log": "Log" + }, "metricNoun": { "spend": "spend", "tokens": "tokens", @@ -13990,12 +14046,27 @@ "loadingTopModels": "Loading top models" }, "breakdown": { + "metricOptions": { + "spend": "Spend", + "tokens": "Tokens", + "requests": "Requests" + }, + "dimensionOptions": { + "modelGroups": "Model", + "models": "Deployment", + "providers": "Provider" + }, "metricLabel": "Metric", "groupByLabel": "Group by", "empty": "No usage in this range", "byProvider": "by {{provider}}" }, "topAgents": { + "metricOptions": { + "tokens": "Tokens", + "spend": "Spend", + "requests": "Requests" + }, "viewActivity": "View activity", "tokens": "{{tokens}} tokens", "viewAgentUsage": "View {{agent}} usage", @@ -14006,6 +14077,9 @@ "metricLabel": "Agent metric" }, "agentCatalog": { + "labels": { + "claudeUnspecified": "Claude (unspecified)" + }, "descriptions": { "claudeDesktop": "Anthropic's desktop assistant", "claudeCode": "Anthropic's coding agent", @@ -14033,6 +14107,28 @@ } }, "errors": { + "entityLabels": { + "key": { + "singular": "Virtual key", + "plural": "Virtual keys" + }, + "team": { + "singular": "Team", + "plural": "Teams" + }, + "user": { + "singular": "User", + "plural": "Users" + }, + "model": { + "singular": "Model", + "plural": "Models" + } + }, + "rateViews": { + "total": "Total", + "status": "By status code" + }, "entities": { "title": "Failures by identity", "subtitle": "Keys, teams, users and models ranked by failed requests", @@ -14165,6 +14261,17 @@ } }, "entitySummary": { + "titles": { + "ptuHours": "PTU Hours", + "totalCost": "Total Cost", + "totalSpend": "Total Spend", + "totalRequests": "Total Requests", + "successfulRequests": "Successful Requests", + "failedRequests": "Failed Requests", + "totalTokens": "Total Tokens", + "requestCost": "Request Cost", + "flatCost": "Flat Cost" + }, "tooltips": { "totalCost": "Request cost plus flat cost for reserved capacity. Select this tile to see the breakdown.", "requestCost": "Usage-based cost of the requests this entity sent during the selected period, priced per token.", @@ -14174,6 +14281,14 @@ }, "entityUsage": { "chart": { + "granularityOptions": { + "day": "Daily", + "week": "Weekly" + }, + "scaleOptions": { + "linear": "Linear", + "log": "Log" + }, "flatCostSeries": "Flat cost", "metricTitle": { "spend": "Spend", @@ -14259,6 +14374,10 @@ "noUser": "(no user)" }, "topModelView": { + "viewOptions": { + "table": "Table View", + "chart": "Chart View" + }, "columns": { "model": "Model", "spendUsd": "Spend (USD)", @@ -14286,6 +14405,8 @@ "noData": "No endpoint usage data" }, "barChart": { + "successful": "Successful", + "failed": "Failed", "title": "Success vs Failed Requests by Endpoint" }, "lineChart": { @@ -14296,6 +14417,7 @@ "unknownModel": "Unknown model", "unknownProvider": "Unknown provider", "errors": { + "narrowRange": "Narrow the date range to fewer than 1,000 requests to see complete client usage", "selectUserOrTeam": "Select a user or team to view client usage", "incompleteData": "The proxy returned incomplete client usage data", "outsideUser": "The proxy returned usage outside the selected user", @@ -14361,6 +14483,10 @@ "loadMoreFailed": "Could not load more keys." }, "topKeys": { + "viewOptions": { + "table": "Table View", + "chart": "Chart View" + }, "keyNotInDatabase": "This key is no longer in the database (deleted, or a CLI/SSO session key), so its details can't be opened", "columns": { "keyId": "Key ID", @@ -14554,6 +14680,10 @@ } }, "userAgentActivity": { + "viewOptions": { + "activeUsers": "Active users", + "perUser": "Per user" + }, "noActiveUsers": "No active users in this period", "weekLabel": "Week {{number}}", "monthLabel": "Month {{number}}", @@ -14695,6 +14825,10 @@ } } }, + "promptCacheUsage": { + "readTooltip": "Input tokens read from the LLM provider's prompt cache (e.g. Anthropic / OpenAI), billed at a discounted rate. Reported by the provider.", + "creationTooltip": "Input tokens written to the LLM provider's prompt cache for reuse by later requests." + }, "organizations": { "view": { "loading": "Loading...", diff --git a/ui/litellm-dashboard/src/utils/promptCacheUsage.ts b/ui/litellm-dashboard/src/utils/promptCacheUsage.ts index f7c25d43e5f..9209e60485c 100644 --- a/ui/litellm-dashboard/src/utils/promptCacheUsage.ts +++ b/ui/litellm-dashboard/src/utils/promptCacheUsage.ts @@ -1,7 +1,7 @@ -export const PROMPT_CACHE_READ_TOOLTIP = - "Input tokens read from the LLM provider's prompt cache (e.g. Anthropic / OpenAI), billed at a discounted rate. Reported by the provider."; -export const PROMPT_CACHE_CREATION_TOOLTIP = - "Input tokens written to the LLM provider's prompt cache for reuse by later requests."; +import type { ParseKeys } from "i18next"; + +export const PROMPT_CACHE_READ_TOOLTIP: ParseKeys = "promptCacheUsage.readTooltip"; +export const PROMPT_CACHE_CREATION_TOOLTIP: ParseKeys = "promptCacheUsage.creationTooltip"; interface CachedTokenDetails { cached_tokens?: number | null; From 625353fabe24a7c3bafc76bd7f7258309118e6f4 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 5 Nov 2026 12:42:26 -0800 Subject: [PATCH 09/40] refactor(ui): move remaining models text onto i18n (#50147) * refactor(ui): move remaining models text onto i18n Moves the leftover English on the models pages, mostly labels and hints held in constants, onto react-i18next. Constants now hold message keys and the code that renders them translates. Where a string is both a value and a label (tier names, health check sentinels), the English value stays for comparisons, sorting and payloads, and a separate label key drives display. Lookups keyed by server values check Object.hasOwn first. * refactor(ui): translate the tier name in display name inputs --- .../_components/TierTurnsChart.tsx | 13 +- .../components/AllModelsTable.tsx | 12 +- .../components/AutoRouters/autoRouterRows.ts | 30 +-- .../src/components/ModelInfoEditForm.tsx | 4 +- .../src/components/add_model/AddModelForm.tsx | 4 +- .../add_model/AutoRouterSimpleSetup.tsx | 2 +- .../add_model/ClassificationMethodConfig.tsx | 12 +- .../add_model/ComplexityRouterConfig.tsx | 74 +++--- .../add_model/CompressionControls.tsx | 5 +- .../add_model/CustomDimensionRows.tsx | 8 +- .../add_model/HeuristicScoringConfig.test.tsx | 3 +- .../add_model/OpeningPromptEditor.tsx | 10 +- .../components/add_model/TierRestrictions.tsx | 3 +- .../components/add_model/add_model_modes.tsx | 34 +-- .../add_model/advanced_settings.tsx | 10 +- .../src/components/add_model/auto_setup.ts | 8 +- .../add_model/cache_control_settings.tsx | 25 +- .../add_model/heuristic_scoring_knobs.ts | 20 +- .../add_model/provider_auth_types.ts | 24 +- .../add_model/provider_specific_fields.tsx | 6 +- .../src/components/add_model/tier_rows.ts | 24 +- .../components/model_add/CredentialModal.tsx | 11 +- .../components/model_add/FederationFields.tsx | 2 +- .../model_add/anthropic_federation.ts | 95 ++++---- .../components/model_add/federation_field.ts | 6 +- .../components/model_add/openai_federation.ts | 15 +- .../HealthChecksTableColumns.tsx | 15 +- ui/litellm-dashboard/src/i18n/locales/en.json | 217 ++++++++++++++++++ 28 files changed, 471 insertions(+), 221 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.tsx index 87d689e3162..c1a89f4cdad 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.tsx @@ -1,5 +1,6 @@ "use client"; +import type { TFunction } from "i18next"; import React from "react"; import { useTranslation } from "react-i18next"; @@ -7,11 +8,12 @@ import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useMod import { hydrateTierLabels } from "@/components/add_model/build_complexity_router_config"; import { TIER_KEYS, - effectiveTierLabel, + effectiveTierDisplayLabel, type ComplexityTierLabels, type ComplexityTiers, } from "@/components/add_model/ComplexityRouterConfig"; import { normalizeTierModels } from "@/components/add_model/complexity_router_tiers"; +import i18n from "@/i18n/i18n"; import { chartColorValue, DEFAULT_COLOR_CYCLE, DonutChart } from "@/components/shared/charts"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; @@ -35,8 +37,11 @@ const asRecord = (value: unknown): Record => { const isComplexityTier = (tier: string): tier is keyof ComplexityTiers => (TIER_KEYS as readonly string[]).includes(tier); -export const tierDisplayLabel = (tier: string, tierLabels: ComplexityTierLabels | undefined): string => - isComplexityTier(tier) ? effectiveTierLabel(tier, tierLabels) : tier; +export const tierDisplayLabel = ( + tier: string, + tierLabels: ComplexityTierLabels | undefined, + t: TFunction = i18n.t.bind(i18n), +): string => (isComplexityTier(tier) ? effectiveTierDisplayLabel(tier, tierLabels, t) : tier); const CONFIG_KEY_BY_ROUTER_TYPE: Record> = { complexity: "complexity_router_config", @@ -93,7 +98,7 @@ const TierTurnsChart: React.FC = ({ view, autoRouters }) => const tierLabels = tierLabelsFor(group.router_name, group.router_type, autoRouters); const total = entries.reduce((sum, [, turns]) => sum + turns, 0); const slices = entries.map(([tier, turns]) => ({ - tier: tierDisplayLabel(tier, tierLabels), + tier: tierDisplayLabel(tier, tierLabels, t), turns, models: tierModelsFor(tier, group.router_name, group.router_type, autoRouters), })); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTable.tsx index 3687ef963c0..df433afd2b9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTable.tsx @@ -38,10 +38,10 @@ export const WILDCARD_MODEL_GROUP_VALUE = "wildcard"; const MODEL_TABLE_BODY_HEIGHT = 600; const ALL_PROXY_MODELS_LABEL: ParseKeys = "modelsAndEndpoints.allModelsTable.allProxyModels"; -const FILTER_LABELS: Record = { - [MODEL_NAME_COLUMN_ID]: "Public Model Name", - [ACCESS_GROUPS_COLUMN_ID]: "Model Access Group", - [ROUTING_STATUS_FILTER_ID]: "Status", +const FILTER_LABELS: Record = { + [MODEL_NAME_COLUMN_ID]: "modelsAndEndpoints.allModelsTable.filterLabels.publicModelName", + [ACCESS_GROUPS_COLUMN_ID]: "modelsAndEndpoints.allModelsTable.filterLabels.modelAccessGroup", + [ROUTING_STATUS_FILTER_ID]: "modelsAndEndpoints.allModelsTable.filterLabels.status", }; const ROUTING_STATUS_FILTER_VALUES = { @@ -241,7 +241,9 @@ export function AllModelsTable({ onOpenFilters={() => setFiltersOpen(true)} onRefresh={onRefresh} isRefreshing={isRefreshing} - filterLabels={FILTER_LABELS} + filterLabels={Object.fromEntries( + Object.entries(FILTER_LABELS).map(([columnId, labelKey]) => [columnId, t(labelKey)]), + )} formatFilterValue={formatFilterValue} > ({ value: mode.value, label: t(mode.labelKey) }))} value={(control.value as string | undefined) ?? null} onValueChange={(value: string | null) => { control.onChange(value); @@ -326,7 +326,7 @@ const AddModelForm: React.FC = ({ {TEST_MODES.map((mode) => ( - {mode.label} + {t(mode.labelKey)} ))} diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterSimpleSetup.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterSimpleSetup.tsx index 04ee03f7572..092bf41c444 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoRouterSimpleSetup.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterSimpleSetup.tsx @@ -83,7 +83,7 @@ export default function AutoRouterSimpleSetup({ aria-pressed={goal === key} onClick={() => onGoalChange(key as RouterGoal)} > - {label} + {t(label)} ))}
diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx index ea70d499099..6f514785bb9 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx @@ -42,7 +42,7 @@ import { DEFAULT_CLASSIFIER_TIMEOUT_MS, DEFAULT_CLASSIFICATION_RUBRIC, ClassificationRubric, - effectiveTierLabel, + effectiveTierDisplayLabel, heuristicScoringRole, heuristicTuningType, usesHeuristicV2, @@ -119,25 +119,25 @@ const HowClassificationWorks: React.FC<{ value: ComplexityRouterConfigValue }> = {scorerRuns && ranges && (
  • - {effectiveTierLabel("SIMPLE", value.tier_labels)}:{" "} + {effectiveTierDisplayLabel("SIMPLE", value.tier_labels, t)}:{" "} {t("addModel.classificationMethodConfig.howItWorks.scoreBelow", { max: ranges.simpleMedium })}
  • - {effectiveTierLabel("MEDIUM", value.tier_labels)}:{" "} + {effectiveTierDisplayLabel("MEDIUM", value.tier_labels, t)}:{" "} {t("addModel.classificationMethodConfig.howItWorks.scoreBetween", { min: ranges.simpleMedium, max: ranges.mediumComplex, })}
  • - {effectiveTierLabel("COMPLEX", value.tier_labels)}:{" "} + {effectiveTierDisplayLabel("COMPLEX", value.tier_labels, t)}:{" "} {t("addModel.classificationMethodConfig.howItWorks.scoreBetween", { min: ranges.mediumComplex, max: ranges.complexReasoning, })}
  • - {effectiveTierLabel("REASONING", value.tier_labels)}:{" "} + {effectiveTierDisplayLabel("REASONING", value.tier_labels, t)}:{" "} {t("addModel.classificationMethodConfig.howItWorks.scoreAbove", { min: ranges.complexReasoning, floor: ranges.reasoningOverrideFloor, @@ -506,7 +506,7 @@ const ClassificationMethodConfig: React.FC = ({ {HEURISTIC_FIRST_MAX_TIER_KEYS.map((tier) => ( - {effectiveTierLabel(tier, value.tier_labels)} + {effectiveTierDisplayLabel(tier, value.tier_labels, t)} ))} diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index e757813783d..e99a8c6e949 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -104,36 +104,27 @@ export const DEFAULT_CLASSIFICATION_RUBRIC: ClassificationRubric = "legacy"; */ export const NEW_CLASSIFIER_CLASSIFICATION_RUBRIC: ClassificationRubric = "agentic"; -export const CLASSIFICATION_RUBRIC_DESCRIPTIONS: Record = - { - legacy: { - label: "Legacy (uncalibrated)", - description: - "The rubric as it shipped before calibration examples, with no worked examples at all. Routers created " + - "before this setting existed use it, so their tier decisions and spend are unchanged. It over-routes " + - "ordinary engineering to the most expensive tier.", - }, - agentic: { - label: "Agentic", - description: - "Anchors routine installs, builds, multi-file edits, and standard debugging at " + - "Medium, so ordinary engineering does not route to your most expensive tier. Suits agent, terminal, and " + - "coding-assistant traffic, and mixed traffic.", - }, - chat: { - label: "Chat", - description: - "Drops the engineering examples, for a router serving only conversational traffic that never sees those " + - "requests.", - }, - business: { - label: "Business", - description: - "Business and sales examples plus business-oriented tier definitions: routine drafting and summarizing " + - "stay at Medium, data-determined analysis is Complex, and only decisions under conflicting tradeoffs " + - "reach Reasoning. Suits sales, support, and go-to-market traffic.", - }, - }; +export const CLASSIFICATION_RUBRIC_DESCRIPTIONS: Record< + ClassificationRubric, + { labelKey: ParseKeys; descriptionKey: ParseKeys } +> = { + legacy: { + labelKey: "addModel.complexityRouterConfig.rubrics.legacy.label", + descriptionKey: "addModel.complexityRouterConfig.rubrics.legacy.description", + }, + agentic: { + labelKey: "addModel.complexityRouterConfig.rubrics.agentic.label", + descriptionKey: "addModel.complexityRouterConfig.rubrics.agentic.description", + }, + chat: { + labelKey: "addModel.complexityRouterConfig.rubrics.chat.label", + descriptionKey: "addModel.complexityRouterConfig.rubrics.chat.description", + }, + business: { + labelKey: "addModel.complexityRouterConfig.rubrics.business.label", + descriptionKey: "addModel.complexityRouterConfig.rubrics.business.description", + }, +}; export const CLASSIFICATION_RUBRIC_KEYS = Object.keys(CLASSIFICATION_RUBRIC_DESCRIPTIONS) as ClassificationRubric[]; @@ -204,7 +195,9 @@ const rowOrigin = (row: TierRow, editing: boolean, t: TFunction = i18n.t.bind(i1 : t("addModel.complexityRouterConfig.rowOrigin.custom"); }; -const builtInTierInfo = (rowId: string): { label: string; description: ParseKeys; examples: ParseKeys } | undefined => { +const builtInTierInfo = ( + rowId: string, +): { labelKey: ParseKeys; description: ParseKeys; examples: ParseKeys } | undefined => { const builtIn = ALL_BUILT_IN_TIERS.find((tier) => tier === rowId); return builtIn ? TIER_DESCRIPTIONS[builtIn] : undefined; }; @@ -531,30 +524,35 @@ interface ComplexityRouterConfigProps { export const TIER_DESCRIPTIONS: Record< keyof ComplexityTiers, - { label: string; description: ParseKeys; examples: ParseKeys } + { label: string; labelKey: ParseKeys; description: ParseKeys; examples: ParseKeys } > = { NON_REASONING: { label: "Non-reasoning", + labelKey: "addModel.complexityRouterConfig.tiers.nonReasoning.label", description: "addModel.complexityRouterConfig.tiers.nonReasoning.description", examples: "addModel.complexityRouterConfig.tiers.nonReasoning.examples", }, SIMPLE: { label: "Simple", + labelKey: "addModel.complexityRouterConfig.tiers.simple.label", description: "addModel.complexityRouterConfig.tiers.simple.description", examples: "addModel.complexityRouterConfig.tiers.simple.examples", }, MEDIUM: { label: "Medium", + labelKey: "addModel.complexityRouterConfig.tiers.medium.label", description: "addModel.complexityRouterConfig.tiers.medium.description", examples: "addModel.complexityRouterConfig.tiers.medium.examples", }, COMPLEX: { label: "Complex", + labelKey: "addModel.complexityRouterConfig.tiers.complex.label", description: "addModel.complexityRouterConfig.tiers.complex.description", examples: "addModel.complexityRouterConfig.tiers.complex.examples", }, REASONING: { label: "Reasoning", + labelKey: "addModel.complexityRouterConfig.tiers.reasoning.label", description: "addModel.complexityRouterConfig.tiers.reasoning.description", examples: "addModel.complexityRouterConfig.tiers.reasoning.examples", }, @@ -565,6 +563,12 @@ export const TIER_KEYS = Object.keys(TIER_DESCRIPTIONS) as Array tierLabels?.[tier]?.trim() || TIER_DESCRIPTIONS[tier].label; +export const effectiveTierDisplayLabel = ( + tier: keyof ComplexityTiers, + tierLabels: ComplexityTierLabels | undefined, + t: TFunction = i18n.t.bind(i18n), +): string => tierLabels?.[tier]?.trim() || t(TIER_DESCRIPTIONS[tier].labelKey); + export const DEFAULT_HEURISTIC_FIRST_MAX_TIER = "SIMPLE"; /** What the Hybrid radio starts at. Required by that type, so the form always has a value to send. */ @@ -797,10 +801,10 @@ const ComplexityRouterConfig: React.FC = ({ handleTierLabelChange(row.id as keyof ComplexityTiers, event.target.value) } placeholder={t("addModel.complexityRouterConfig.displayName.placeholder", { - label: tierInfo.label, + label: t(tierInfo.labelKey), })} aria-label={t("addModel.complexityRouterConfig.displayName.aria", { - label: tierInfo.label, + label: t(tierInfo.labelKey), })} /> {value.tier_labels?.[row.id as keyof ComplexityTiers] && ( @@ -808,7 +812,7 @@ const ComplexityRouterConfig: React.FC = ({ handleTierLabelChange(row.id as keyof ComplexityTiers, "")} > diff --git a/ui/litellm-dashboard/src/components/add_model/CompressionControls.tsx b/ui/litellm-dashboard/src/components/add_model/CompressionControls.tsx index 2c8ef000e61..6df8c3ad13d 100644 --- a/ui/litellm-dashboard/src/components/add_model/CompressionControls.tsx +++ b/ui/litellm-dashboard/src/components/add_model/CompressionControls.tsx @@ -3,6 +3,7 @@ import { SearchSelect, SearchSelectOption } from "@/components/shared/SearchSele import { Label } from "@/components/ui/label"; import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group"; import { Info } from "lucide-react"; +import type { ParseKeys } from "i18next"; import React from "react"; import { useTranslation } from "react-i18next"; import { useGuardrails } from "@/app/(dashboard)/hooks/guardrails/useGuardrails"; @@ -17,7 +18,7 @@ interface CompressionControlsProps { onChange: (state: AutoRouterCompressionState) => void; } -const NONE_OPTION: SearchSelectOption = { label: "None (no compression)", value: NO_COMPRESSION }; +const NONE_OPTION_LABEL: ParseKeys = "addModel.compressionControls.noneOption"; const CompressionControls: React.FC = ({ value, onChange }) => { const { t } = useTranslation(); @@ -30,7 +31,7 @@ const CompressionControls: React.FC = ({ value, onChan const compressionOptions: SearchSelectOption[] = (data?.guardrails ?? []) .filter((g) => isCompressionGuardrailProvider(g.litellm_params?.guardrail)) .map((g) => ({ label: g.guardrail_name, value: g.guardrail_name })); - const options: SearchSelectOption[] = [NONE_OPTION, ...compressionOptions]; + const options: SearchSelectOption[] = [{ label: t(NONE_OPTION_LABEL), value: NO_COMPRESSION }, ...compressionOptions]; return (
    diff --git a/ui/litellm-dashboard/src/components/add_model/CustomDimensionRows.tsx b/ui/litellm-dashboard/src/components/add_model/CustomDimensionRows.tsx index 2dfd5c3ce10..ebb5af749bd 100644 --- a/ui/litellm-dashboard/src/components/add_model/CustomDimensionRows.tsx +++ b/ui/litellm-dashboard/src/components/add_model/CustomDimensionRows.tsx @@ -10,8 +10,8 @@ import { Slider } from "@/components/ui/slider"; import type { CustomDimensionRow } from "./custom_dimensions"; const SCORING_MODES = [ - { value: "binary", label: "Binary" }, - { value: "match_count", label: "Match count" }, + { value: "binary", labelKey: "addModel.customDimensionRows.scoringModes.binary" }, + { value: "match_count", labelKey: "addModel.customDimensionRows.scoringModes.matchCount" }, ] as const; interface Props { @@ -109,7 +109,7 @@ export default function CustomDimensionRows({ rows, disabled, onChange, onWeight
    ({ value: rubric, - label: description.label, + label: t(description.labelKey), }))} value={classificationRubric ?? tierSource.classificationRubric} onValueChange={(rubric: ClassificationRubric | null) => rubric && setRubricDraft(rubric)} @@ -210,13 +210,13 @@ const OpeningPromptEditor: React.FC = ({ > {Object.entries(CLASSIFICATION_RUBRIC_DESCRIPTIONS).map(([rubric, description]) => ( - {description.label} + {t(description.labelKey)} ))}

    - {tierSource.rubricRestriction ?? draftRubricSummary?.description} + {tierSource.rubricRestriction ?? (draftRubricSummary && t(draftRubricSummary.descriptionKey))}

    )} diff --git a/ui/litellm-dashboard/src/components/add_model/TierRestrictions.tsx b/ui/litellm-dashboard/src/components/add_model/TierRestrictions.tsx index 4383bea5972..78551f6a398 100644 --- a/ui/litellm-dashboard/src/components/add_model/TierRestrictions.tsx +++ b/ui/litellm-dashboard/src/components/add_model/TierRestrictions.tsx @@ -8,7 +8,8 @@ export const restrictedBy = ( key: keyof typeof CUSTOM_TIER_RESTRICTIONS, t: TFunction = i18n.t.bind(i18n), ): TierRestriction | undefined => { - if (value.custom_tier_set) return CUSTOM_TIER_RESTRICTIONS[key]; + if (value.custom_tier_set) + return { omit: CUSTOM_TIER_RESTRICTIONS[key].omit, reason: t(CUSTOM_TIER_RESTRICTIONS[key].reason) }; if (value.classifier_type === "llm_v2" && key === "adaptive") return { omit: ["adaptive", "adaptive_weights", "adaptive_eligible", "tier_distance_penalty"], diff --git a/ui/litellm-dashboard/src/components/add_model/add_model_modes.tsx b/ui/litellm-dashboard/src/components/add_model/add_model_modes.tsx index 63def76db8f..ef651b9f7dc 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_model_modes.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_model_modes.tsx @@ -1,20 +1,22 @@ +import type { ParseKeys } from "i18next"; + // Define the available test modes -export const TEST_MODES = [ - { value: "chat", label: "Chat - /chat/completions" }, - { value: "completion", label: "Completion - /completions" }, - { value: "embedding", label: "Embedding - /embeddings" }, - { value: "audio_speech", label: "Audio Speech - /audio/speech" }, - { value: "audio_transcription", label: "Audio Transcription - /audio/transcriptions" }, - { value: "image_generation", label: "Image Generation - /images/generations" }, - { value: "image_edit", label: "Image Edit - /images/edits" }, - { value: "video_generation", label: "Video Generation - /videos" }, - { value: "rerank", label: "Rerank - /rerank" }, - { value: "realtime", label: "Realtime - /realtime" }, - { value: "batch", label: "Batch - /batch" }, - { value: "anthropic_messages", label: "Anthropic Messages - /v1/messages" }, - { value: "ocr", label: "OCR - /ocr" }, - { value: "decisions", label: "Decisions - /v1/decisions" }, - { value: "extraction", label: "Extraction - /v1/extraction" }, +export const TEST_MODES: readonly { value: string; labelKey: ParseKeys }[] = [ + { value: "chat", labelKey: "addModel.addModelModes.chat" }, + { value: "completion", labelKey: "addModel.addModelModes.completion" }, + { value: "embedding", labelKey: "addModel.addModelModes.embedding" }, + { value: "audio_speech", labelKey: "addModel.addModelModes.audioSpeech" }, + { value: "audio_transcription", labelKey: "addModel.addModelModes.audioTranscription" }, + { value: "image_generation", labelKey: "addModel.addModelModes.imageGeneration" }, + { value: "image_edit", labelKey: "addModel.addModelModes.imageEdit" }, + { value: "video_generation", labelKey: "addModel.addModelModes.videoGeneration" }, + { value: "rerank", labelKey: "addModel.addModelModes.rerank" }, + { value: "realtime", labelKey: "addModel.addModelModes.realtime" }, + { value: "batch", labelKey: "addModel.addModelModes.batch" }, + { value: "anthropic_messages", labelKey: "addModel.addModelModes.anthropicMessages" }, + { value: "ocr", labelKey: "addModel.addModelModes.ocr" }, + { value: "decisions", labelKey: "addModel.addModelModes.decisions" }, + { value: "extraction", labelKey: "addModel.addModelModes.extraction" }, ]; // Define the available auto router routing strategies diff --git a/ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx b/ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx index 28cf847587e..83ac6ff465e 100644 --- a/ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx +++ b/ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx @@ -58,8 +58,8 @@ const USAGE_COST_FIELDS = [ const REVALIDATED_WHEN_PTU_COUNT_CHANGES = [PTU_RATE_FIELD, PTU_START_FIELD, ...USAGE_COST_FIELDS]; const PRICING_MODEL_ITEMS = [ - { value: "per_token", label: "Per Million Tokens" }, - { value: "per_second", label: "Per Second" }, + { value: "per_token", labelKey: "addModel.advancedSettings.pricingModels.perToken" }, + { value: "per_second", labelKey: "addModel.advancedSettings.pricingModels.perSecond" }, ] as const; const validateNumber = (_: unknown, value: unknown) => { @@ -320,7 +320,7 @@ const AdvancedSettings: React.FC = ({ > {(control) => ( + ({ value: item.value, label: t(item.labelKey) }))} value={point.role ?? null} onValueChange={(selected) => replaceAt(index, { ...point, role: (selected as CacheControlRole | null) ?? undefined }) @@ -119,7 +122,7 @@ const CacheControlInjectionPoints: React.FC = {t("addModel.cacheControlSettings.noRole")} {ROLE_ITEMS.map((item) => ( - {item.label} + {t(item.labelKey)} ))} diff --git a/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.ts b/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.ts index 1249323aa39..185785fd120 100644 --- a/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.ts +++ b/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.ts @@ -1,3 +1,4 @@ +import type { ParseKeys } from "i18next"; import i18n from "@/i18n/i18n"; import { serializeCustomDimensions, type CustomDimensionRow } from "./custom_dimensions"; import { heuristicScoringRoleFor, type ClassifierFallback } from "./ComplexityRouterConfig"; @@ -43,17 +44,18 @@ export const scorerKnobPayload = (knobs: ScorerKnobInputs) => { * shipped weights come from the proxy (GET /public/complexity_router/scorer_defaults), so a dimension * added backend-side still renders, under its raw key until it is given a label here. */ -export const DIMENSION_LABELS: Record = { - codePresence: "Code presence", - reasoningMarkers: "Reasoning markers", - technicalTerms: "Technical terms", - tokenCount: "Token count", - simpleIndicators: "Simple indicators", - multiStepPatterns: "Multi-step patterns", - questionComplexity: "Question complexity", +export const DIMENSION_LABELS: Record = { + codePresence: "addModel.heuristicScoringKnobs.dimensions.codePresence", + reasoningMarkers: "addModel.heuristicScoringKnobs.dimensions.reasoningMarkers", + technicalTerms: "addModel.heuristicScoringKnobs.dimensions.technicalTerms", + tokenCount: "addModel.heuristicScoringKnobs.dimensions.tokenCount", + simpleIndicators: "addModel.heuristicScoringKnobs.dimensions.simpleIndicators", + multiStepPatterns: "addModel.heuristicScoringKnobs.dimensions.multiStepPatterns", + questionComplexity: "addModel.heuristicScoringKnobs.dimensions.questionComplexity", }; -export const dimensionLabel = (key: string): string => DIMENSION_LABELS[key] ?? key; +export const dimensionLabel = (key: string): string => + Object.hasOwn(DIMENSION_LABELS, key) ? i18n.t(DIMENSION_LABELS[key]) : key; const asRecord = (raw: unknown): Record | undefined => typeof raw === "object" && raw !== null && !Array.isArray(raw) ? (raw as Record) : undefined; diff --git a/ui/litellm-dashboard/src/components/add_model/provider_auth_types.ts b/ui/litellm-dashboard/src/components/add_model/provider_auth_types.ts index 0bf98407bfa..77a08c45393 100644 --- a/ui/litellm-dashboard/src/components/add_model/provider_auth_types.ts +++ b/ui/litellm-dashboard/src/components/add_model/provider_auth_types.ts @@ -1,9 +1,10 @@ +import type { ParseKeys } from "i18next"; import { Providers } from "../provider_info_helpers"; export interface ProviderAuthType { readonly id: string; - readonly label: string; - readonly description: string; + readonly labelKey: ParseKeys; + readonly descriptionKey: ParseKeys; readonly fieldKeys: readonly string[]; readonly requiredFieldKeys: readonly string[]; readonly fixedValues?: Readonly>; @@ -20,18 +21,16 @@ export const PROVIDER_AUTH_TYPES: Partial = ({ )} -

    {selectedAuthType.description}

    +

    {t(selectedAuthType.descriptionKey)}

    )} {credentialOnlySelected ? ( diff --git a/ui/litellm-dashboard/src/components/add_model/tier_rows.ts b/ui/litellm-dashboard/src/components/add_model/tier_rows.ts index a9ea085854c..092dc8a73c5 100644 --- a/ui/litellm-dashboard/src/components/add_model/tier_rows.ts +++ b/ui/litellm-dashboard/src/components/add_model/tier_rows.ts @@ -1,4 +1,4 @@ -import type { TFunction } from "i18next"; +import type { ParseKeys, TFunction } from "i18next"; import i18n from "@/i18n/i18n"; import { isForecastClassifier } from "./forecast_classifier_config"; import type { ClassifierType, ComplexityTiers } from "./ComplexityRouterConfig"; @@ -126,29 +126,27 @@ export interface TierRestriction { export const CUSTOM_TIER_RESTRICTIONS = { displayNames: { omit: ["tier_labels"], - reason: "Display names rename the built-in tiers, which your tier set replaces. Name each tier directly", + reason: "addModel.tierRows.restrictions.displayNames", }, escalation: { omit: ["escalation_keywords"], - reason: "Escalation bumps a request along the built-in tier ladder, which your tier set replaces", + reason: "addModel.tierRows.restrictions.escalation", }, stallEscalation: { omit: ["stall_escalation_enabled", "stall_escalation_window", "stall_escalation_repeat_threshold"], - reason: "Stall escalation bumps a request along the built-in tier ladder, which your tier set replaces", + reason: "addModel.tierRows.restrictions.stallEscalation", }, adaptive: { omit: ["adaptive", "adaptive_weights", "tier_distance_penalty", "adaptive_eligible"], - reason: "Adaptive routing scores models along the built-in tier ladder, which your tier set replaces", + reason: "addModel.tierRows.restrictions.adaptive", }, sessionAffinity: { omit: [], - reason: "Session pinning escalates along the built-in tier ladder, which your tier set replaces", + reason: "addModel.tierRows.restrictions.sessionAffinity", }, heuristicClassifier: { omit: ["heuristic_first_max_tier", "hybrid_boundary_margin", "local_heuristic"], - reason: - "The heuristic scorer only produces the built-in tiers, so an edited set needs the LLM or JEV classifier. " + - "Heuristic first and hybrid are out for the same reason: their local scorer decides the traffic it is sure of", + reason: "addModel.tierRows.restrictions.heuristicClassifier", }, heuristicScoring: { omit: [ @@ -159,17 +157,17 @@ export const CUSTOM_TIER_RESTRICTIONS = { "reasoning_override_min_score", "custom_technical_keywords", ], - reason: "The heuristic scorer never runs under an edited tier set, so its inputs have no effect", + reason: "addModel.tierRows.restrictions.heuristicScoring", }, classificationRubric: { omit: [], - reason: "The preset calibration examples are written against the built-in tiers, which your tier set replaces", + reason: "addModel.tierRows.restrictions.classificationRubric", }, classifierFallback: { omit: ["classifier_fallback"], - reason: "Fallback Tier is where an edited tier set routes when the classifier fails", + reason: "addModel.tierRows.restrictions.classifierFallback", }, -} as const satisfies Record; +} as const satisfies Record; export const CUSTOM_TIER_OMITTED_KEYS: readonly string[] = Object.values(CUSTOM_TIER_RESTRICTIONS).flatMap( (restriction) => restriction.omit, diff --git a/ui/litellm-dashboard/src/components/model_add/CredentialModal.tsx b/ui/litellm-dashboard/src/components/model_add/CredentialModal.tsx index 26c0ca07b07..a7e1d8546ef 100644 --- a/ui/litellm-dashboard/src/components/model_add/CredentialModal.tsx +++ b/ui/litellm-dashboard/src/components/model_add/CredentialModal.tsx @@ -1,3 +1,4 @@ +import type { ParseKeys } from "i18next"; import { Input } from "@/components/ui/input"; import { SearchSelect, type SearchSelectOption } from "@/components/shared/SearchSelect"; import { SimpleTooltip } from "@/components/ui/tooltip"; @@ -54,9 +55,9 @@ const AUTH_METHOD_SELECT_ID = "credential_auth_method"; const API_KEY_FIELDS: readonly string[] = ["api_key"]; const NO_HIDDEN_FIELDS: readonly string[] = []; -const authMethodItems: { value: AuthMethod; label: string }[] = [ - { value: "api_key", label: "API key" }, - { value: "federation", label: "Workload identity federation" }, +const authMethodItems: { value: AuthMethod; labelKey: ParseKeys }[] = [ + { value: "api_key", labelKey: "modelAdd.credentialModal.authMethods.apiKey" }, + { value: "federation", labelKey: "modelAdd.credentialModal.authMethods.federation" }, ]; interface CredentialModalProps { @@ -242,7 +243,7 @@ export default function CredentialModal({ )} ({ ...item, label: t(item.label) }))} value={(control.value as string | undefined) ?? null} onValueChange={handleTransportSelected(control.onChange)} > @@ -879,14 +879,17 @@ const CreateMCPServer: React.FC = ({ }} > {(control) => ( - (control)} + items={AUTH_TYPE_ITEMS.map((item) => ({ ...item, label: t(item.label) }))} + > {AUTH_TYPE_ITEMS.map((item) => ( - {item.label} + {t(item.label)} ))} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPServerCard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPServerCard.tsx index 6524985b74a..62e6782bcfc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPServerCard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPServerCard.tsx @@ -379,7 +379,7 @@ const HealthChip: FC = ({ />
    {t("mcpServers.serverCard.healthChip.health", { status })}
    - {status === "reachable" &&
    {MCP_REACHABLE_DESCRIPTION}
    } + {status === "reachable" &&
    {t(MCP_REACHABLE_DESCRIPTION)}
    } {lastCheck && (
    {t("mcpServers.serverCard.healthChip.lastCheck", { time: new Date(lastCheck).toLocaleString() })} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/StdioAvailability.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/StdioAvailability.tsx index 4c2ef63c63f..ecae5d89042 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/StdioAvailability.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/StdioAvailability.tsx @@ -24,7 +24,7 @@ export const TransportSelectItems: FC<{ stdioEnabled: boolean }> = ({ stdioEnabl const disabled = item.value === TRANSPORT.STDIO && !stdioEnabled; return ( - {item.label} + {t(item.label)} {disabled && } ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx index ee27d23bf8e..2b86095c1e4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx @@ -896,7 +896,7 @@ const MCPServerEdit: React.FC = ({ > {(control) => ( (control)} items={AUTH_TYPE_ITEMS}> + ({ value: option.settingValue, label: option.label }))} + items={relativeTimeOptions.map((option) => ({ value: option.settingValue, label: t(option.label) }))} onValueChange={(value) => { if (value === null) return; updateSettings( @@ -354,7 +354,7 @@ export default function UISettings() { {relativeTimeOptions.map((option) => ( - {option.label} + {t(option.label)} ))} diff --git a/ui/litellm-dashboard/src/components/SidebarAccountMenu/SidebarAccountMenu.tsx b/ui/litellm-dashboard/src/components/SidebarAccountMenu/SidebarAccountMenu.tsx index 173e3b3f07e..5d419274bc7 100644 --- a/ui/litellm-dashboard/src/components/SidebarAccountMenu/SidebarAccountMenu.tsx +++ b/ui/litellm-dashboard/src/components/SidebarAccountMenu/SidebarAccountMenu.tsx @@ -21,6 +21,7 @@ import { isProxyAdminRole } from "@/utils/roles"; import { ChevronsUpDown, Crown, IdCard, KeyRound, LogOut, Mail, ShieldCheck } from "lucide-react"; import { useRouter } from "next/navigation"; import React from "react"; +import { Trans, useTranslation } from "react-i18next"; const RELEASE_NOTES_URL = "https://docs.litellm.ai/release_notes"; @@ -96,6 +97,7 @@ const SidebarAccountMenu: React.FC = ({ onLogout, colla accessToken, loginMethod, } = useAuthorized(); + const { t } = useTranslation(); const router = useRouter(); const [open, setOpen] = React.useState(false); const { data: healthData } = useHealthReadinessDetails(accessToken); @@ -119,29 +121,29 @@ const SidebarAccountMenu: React.FC = ({ onLogout, colla const toggles = [ { key: "disableShowNewBadge", - label: "Hide New Feature Indicators", - ariaLabel: "Toggle hide new feature indicators", + label: t("shell.sidebarAccountMenu.toggles.newBadge.label"), + ariaLabel: t("shell.sidebarAccountMenu.toggles.newBadge.ariaLabel"), checked: disableShowNewBadge, onCheckedChange: (checked: boolean) => setFlag("disableShowNewBadge", checked), }, { key: "disableShowPrompts", - label: "Hide All Prompts", - ariaLabel: "Toggle hide all prompts", + label: t("shell.sidebarAccountMenu.toggles.prompts.label"), + ariaLabel: t("shell.sidebarAccountMenu.toggles.prompts.ariaLabel"), checked: disableShowPrompts, onCheckedChange: (checked: boolean) => setFlag("disableShowPrompts", checked), }, { key: "disableBlogPosts", - label: "Hide Blog Posts", - ariaLabel: "Toggle hide blog posts", + label: t("shell.sidebarAccountMenu.toggles.blogPosts.label"), + ariaLabel: t("shell.sidebarAccountMenu.toggles.blogPosts.ariaLabel"), checked: disableBlogPosts, onCheckedChange: (checked: boolean) => setFlag("disableBlogPosts", checked), }, { key: "disableBouncingIcon", - label: "Hide Bouncing Icon", - ariaLabel: "Toggle hide bouncing icon", + label: t("shell.sidebarAccountMenu.toggles.bouncingIcon.label"), + ariaLabel: t("shell.sidebarAccountMenu.toggles.bouncingIcon.ariaLabel"), checked: disableBouncingIcon, onCheckedChange: (checked: boolean) => setFlag("disableBouncingIcon", checked), }, @@ -151,7 +153,10 @@ const SidebarAccountMenu: React.FC = ({ onLogout, colla const initials = initialsFromIdentity(userEmail, userId); const hue = hueFromString(seed); const displayName = navAccountDisplayName(userEmail, userId); - const triggerLabel = `Account menu — ${userRole ?? "Unknown role"} — signed in as ${userEmail || userId || "unknown"}`; + const triggerLabel = t("shell.sidebarAccountMenu.triggerLabel", { + role: userRole ?? t("shell.sidebarAccountMenu.unknownRole"), + identity: userEmail || userId || t("shell.sidebarAccountMenu.unknownIdentity"), + }); return ( @@ -192,7 +197,7 @@ const SidebarAccountMenu: React.FC = ({ onLogout, colla 🌴 @@ -211,27 +216,27 @@ const SidebarAccountMenu: React.FC = ({ onLogout, colla
    - } label="Tier"> + } label={t("shell.sidebarAccountMenu.tier")}> {premiumUser ? ( - Premium + {t("shell.sidebarAccountMenu.premium")} ) : ( - + - Standard + {t("shell.sidebarAccountMenu.standard")} )} - } label="Role"> + } label={t("shell.sidebarAccountMenu.role")}> {userRole} - } label="Email"> - + } label={t("shell.sidebarAccountMenu.email")}> + - } label="User ID"> - + } label={t("shell.sidebarAccountMenu.userId")}> +
    @@ -251,12 +256,14 @@ const SidebarAccountMenu: React.FC = ({ onLogout, colla ))} {premiumUser === true && canUseLiteAdmin && (
    - Hide {ARIA_AGENT_LABEL} + + {ARIA_AGENT_LABEL} }} /> +
    )} @@ -274,7 +281,7 @@ const SidebarAccountMenu: React.FC = ({ onLogout, colla className="h-[42px] w-full justify-start gap-2.5 rounded-none px-3 text-sm font-medium text-foreground" > - Change Password + {t("shell.sidebarAccountMenu.changePassword")} )} @@ -284,7 +291,7 @@ const SidebarAccountMenu: React.FC = ({ onLogout, colla className="h-[42px] w-full justify-start gap-2.5 rounded-none px-3 text-sm font-medium text-foreground" > - Logout + {t("shell.sidebarAccountMenu.logout")} diff --git a/ui/litellm-dashboard/src/components/SidebarUsageCard.tsx b/ui/litellm-dashboard/src/components/SidebarUsageCard.tsx index effb6c29e8e..ba1ce4a286b 100644 --- a/ui/litellm-dashboard/src/components/SidebarUsageCard.tsx +++ b/ui/litellm-dashboard/src/components/SidebarUsageCard.tsx @@ -4,7 +4,9 @@ import { Button } from "@/components/ui/button"; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; import { Meter, MeterIndicator, MeterLabel, MeterTrack } from "@/components/shared/Meter"; import { useQuery } from "@tanstack/react-query"; +import type { TFunction } from "i18next"; import { Award, ChevronDown, Loader2 } from "lucide-react"; +import { useTranslation } from "react-i18next"; import { getRemainingUsers } from "./networking"; interface SidebarUsageCardProps { @@ -26,9 +28,17 @@ const meterTone = (pct: number): "default" | "warning" | "over" => { }; const UsageMeter = ({ label, used, total }: MeterData) => { + const { t } = useTranslation(); const pct = total > 0 ? (used / total) * 100 : 0; return ( - +
    {label} @@ -53,11 +63,15 @@ const remainingUsersQuery = (accessToken: string | null) => ({ staleTime: 5 * 60 * 1000, }); -const buildMeters = (data: RemainingUsage | null): MeterData[] => { +const buildMeters = (data: RemainingUsage | null, t: TFunction): MeterData[] => { if (!data) return []; return [ - ...(data.total_users != null ? [{ label: "Seats", used: data.total_users_used, total: data.total_users }] : []), - ...(data.total_teams != null ? [{ label: "Teams", used: data.total_teams_used, total: data.total_teams }] : []), + ...(data.total_users != null + ? [{ label: t("shell.sidebarUsageCard.seats"), used: data.total_users_used, total: data.total_users }] + : []), + ...(data.total_teams != null + ? [{ label: t("shell.sidebarUsageCard.teams"), used: data.total_teams_used, total: data.total_teams }] + : []), ]; }; @@ -68,6 +82,7 @@ const buildMeters = (data: RemainingUsage | null): MeterData[] => { * design's Spend / API-request meters are intentionally omitted. */ export default function SidebarUsageCard({ accessToken, collapsed, onExpandRail }: SidebarUsageCardProps) { + const { t } = useTranslation(); const licenseInfo = useLicenseInfo(accessToken).data ?? null; const { data: usageData, isLoading } = useQuery(remainingUsersQuery(accessToken)); const data = usageData ?? null; @@ -84,7 +99,7 @@ export default function SidebarUsageCard({ accessToken, collapsed, onExpandRail