diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 1b71232bc2e..9b8b132df62 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -74,12 +74,19 @@ jobs: - name: Run Clippy with Bedrock auth run: cargo clippy -p litellm-core --all-targets --features bedrock-auth --locked -- -D warnings + - name: Run Clippy with all gateway features + run: cargo clippy -p litellm-ai-gateway --all-targets --all-features --locked -- -D warnings + - name: Run Rust tests run: cargo test --workspace --locked - name: Run core tests with Bedrock auth run: cargo test -p litellm-core --features bedrock-auth --locked + # Not --all-features: python-config links libpython, which this job does not install. + - name: Run gateway tests with the server feature + run: cargo test -p litellm-ai-gateway --features server --locked + release-wheel: name: release wheel runs-on: ubuntu-latest diff --git a/litellm-rust/ADDING_A_PROVIDER.md b/litellm-rust/ADDING_A_PROVIDER.md index 857a744e014..ae8ae5a6870 100644 --- a/litellm-rust/ADDING_A_PROVIDER.md +++ b/litellm-rust/ADDING_A_PROVIDER.md @@ -26,4 +26,4 @@ variants of it. The test for a good abstraction is that adding the next provider is a few declarative lines, not a new file of duplicated flow. Only diverge from the base when behavior is genuinely different, and say so explicitly in the PR. -**Calling:** hosts invoke the core entrypoint — the Python bridge and the `ai-gateway` route service both call `litellm_core::messages::messages`. Never add a provider handler to `ai-gateway`. Register new modules in `lib.rs` / `mod.rs`, then run `cargo fmt && cargo clippy --workspace -- -D warnings && cargo test --workspace`. +**Calling:** hosts invoke the core entrypoint — the Python bridge and the `ai-gateway` route service both call `litellm_core::messages::messages`. Never add a provider handler to `ai-gateway`. Register new modules in `lib.rs` / `mod.rs`, then run the commands under "Checks" in [CLAUDE.md](CLAUDE.md). diff --git a/litellm-rust/CLAUDE.md b/litellm-rust/CLAUDE.md index 3dcf1853efc..d9c944529df 100644 --- a/litellm-rust/CLAUDE.md +++ b/litellm-rust/CLAUDE.md @@ -174,10 +174,14 @@ for changes under `litellm-rust/`. ```bash cd litellm-rust cargo fmt --check +cargo clippy --workspace --all-targets -- -D warnings +cargo clippy -p litellm-core --all-targets --features bedrock-auth -- -D warnings # the ai-gateway binary + server code is behind the `server` feature -cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings -cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings +cargo clippy -p litellm-ai-gateway --all-targets --all-features -- -D warnings cargo test --workspace +cargo test -p litellm-core --features bedrock-auth +# the `auth`, `routes`, `state` and `realtime` tests only exist under `server` +cargo test -p litellm-ai-gateway --features server ``` When a Rust path is exposed through Python, add Python parity tests that compare diff --git a/litellm-rust/README.md b/litellm-rust/README.md index a0d79c6f0a5..e43dc7ea6ad 100644 --- a/litellm-rust/README.md +++ b/litellm-rust/README.md @@ -49,11 +49,6 @@ function per top-level route, mirroring the core entrypoints. ## Checks -Run these before pushing Rust changes. GitHub Actions runs the same checks for -changes under `litellm-rust/`. - -```bash -cargo fmt --check -cargo clippy --workspace --all-targets -- -D warnings -cargo test --workspace -``` +Run the commands under "Checks" in [CLAUDE.md](CLAUDE.md) before pushing Rust +changes. That list is the single source of truth and matches what GitHub Actions +runs for changes under `litellm-rust/`. diff --git a/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md b/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md index 4a689cb9579..c0a29ab14bc 100644 --- a/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md +++ b/litellm-rust/crates/CODING_STANDARDS/PROVIDER_CODING_STANDARDS.md @@ -49,11 +49,5 @@ Rules for adding or changing an LLM provider/route in `litellm-rust`. `messages` ## Checks before push -25. Run, and keep green: - ```bash - cd litellm-rust - cargo fmt --check - cargo clippy -p litellm-ai-gateway --all-targets --features server -- -D warnings - cargo clippy -p litellm-core -p litellm-python-interop -p litellm-python-bridge --all-targets -- -D warnings - cargo test --workspace - ``` +25. Run, and keep green, the commands under "Checks" in `litellm-rust/CLAUDE.md`. + That list is the single source of truth and matches what GitHub Actions runs. diff --git a/litellm-rust/crates/ai-gateway/Dockerfile b/litellm-rust/crates/ai-gateway/Dockerfile index adf6fca0741..2bc3c05ad7e 100644 --- a/litellm-rust/crates/ai-gateway/Dockerfile +++ b/litellm-rust/crates/ai-gateway/Dockerfile @@ -36,12 +36,12 @@ FROM chef AS builder # whenever only gateway source changes. COPY --from=planner /build/litellm-rust/recipe.json recipe.json RUN cargo chef cook --locked --release \ - -p litellm-ai-gateway --features python-config \ + -p litellm-ai-gateway --features server,python-config \ --recipe-path recipe.json # Now copy the real sources and build the gateway binary. Deps are already cooked # above, so this step only recompiles the gateway crate. COPY litellm-rust/ . -RUN cargo build --locked --release -p litellm-ai-gateway --features python-config +RUN cargo build --locked --release -p litellm-ai-gateway --bin litellm-ai-gateway --features server,python-config # ---- Runtime ---------------------------------------------------------------- # python:3.11-slim-bookworm ships libpython3.11, matching the builder's PyO3 diff --git a/litellm-rust/crates/ai-gateway/README.md b/litellm-rust/crates/ai-gateway/README.md index 5cbb47220be..1675e6f1b16 100644 --- a/litellm-rust/crates/ai-gateway/README.md +++ b/litellm-rust/crates/ai-gateway/README.md @@ -100,7 +100,7 @@ Worker tuning, rarely needed: `LITELLM_LOG_CHANNEL_CAPACITY` (4096), ## Build & run with Docker -The image is built `--features python-config` and installs litellm **from this +The image is built `--features server,python-config` and installs litellm **from this repo's source** (the config reader is newer than any PyPI release), so the build **context is the repo root**: @@ -135,10 +135,10 @@ docker run --rm -p 4001:4001 \ ```bash # config.yaml mode — needs litellm importable in the active python env LITELLM_CONFIG_PATH=./crates/ai-gateway/config.yaml \ - cargo run --release -p litellm-ai-gateway --features python-config + cargo run --release -p litellm-ai-gateway --features server,python-config # env stand-in mode — no python, no config -cargo run --release -p litellm-ai-gateway +cargo run --release -p litellm-ai-gateway --features server ``` ## Deploy on Render diff --git a/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs b/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs index c32e727de54..c0d72e90b77 100644 --- a/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs +++ b/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs @@ -106,16 +106,16 @@ impl RealTimeStreaming { /// `litellm_call_id`, replacing the gateway-generated fallback. fn on_session(&mut self, event: &RealtimeEvent) { let session = event.data.get("session").and_then(Value::as_object); - if let Some(id) = session.and_then(|s| s.get("id")).and_then(Value::as_str) { - if !id.is_empty() { - self.id = id.to_string(); - self.litellm_call_id = id.to_string(); - } + if let Some(id) = session.and_then(|s| s.get("id")).and_then(Value::as_str) + && !id.is_empty() + { + self.id = id.to_string(); + self.litellm_call_id = id.to_string(); } - if let Some(model) = session.and_then(|s| s.get("model")).and_then(Value::as_str) { - if !model.is_empty() { - self.model = model.to_string(); - } + if let Some(model) = session.and_then(|s| s.get("model")).and_then(Value::as_str) + && !model.is_empty() + { + self.model = model.to_string(); } } @@ -323,6 +323,32 @@ mod tests { assert_eq!(streaming.dropped(), 0); } + #[test] + fn blank_session_id_and_model_keep_the_gateway_fallbacks() { + let mut streaming = RealTimeStreaming::new( + Vec::new(), + "call_fallback".to_string(), + "gpt-realtime".to_string(), + RequestMetadata::default(), + ); + + streaming.observe(&event( + r#"{"type":"session.created","session":{"id":"","model":""}}"#, + )); + let payload = streaming.build_payload(); + assert_eq!(payload.id, "call_fallback"); + assert_eq!(payload.litellm_call_id, "call_fallback"); + assert_eq!(payload.model, "gpt-realtime"); + + streaming.observe(&event( + r#"{"type":"session.updated","session":{"id":"sess_002","model":""}}"#, + )); + let payload = streaming.build_payload(); + assert_eq!(payload.id, "sess_002"); + assert_eq!(payload.litellm_call_id, "sess_002"); + assert_eq!(payload.model, "gpt-realtime"); + } + #[test] fn payload_serializes_with_camelcase_times_and_realtime_call_type() { let mut streaming = RealTimeStreaming::new( diff --git a/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs b/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs index b8ee77c4269..f7bbb37dff4 100644 --- a/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs +++ b/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs @@ -50,18 +50,17 @@ where provider_model, params.api_key.as_deref(), params.api_base.as_deref(), - ) { - if let Some(handoff) = pool.take(&key) { - return crate::io::realtime::realtime_warm( - provider_model, - handoff, - idle_timeout, - observe, - client_in, - client_out, - ) - .await; - } + ) && let Some(handoff) = pool.take(&key) + { + return crate::io::realtime::realtime_warm( + provider_model, + handoff, + idle_timeout, + observe, + client_in, + client_out, + ) + .await; } // Cold path: fresh dial (the original behavior).