mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
feat: add openai realtime translation layer to litellm-rust (1/2) (#31129)
* add RealtimeTransformResult type for realtime transforms * add RealtimeProviderConfig pure trait in litellm-core * add core realtime module * register realtime module in litellm-core lib * add OpenAI realtime transform + complete_url parity in providers * add openai realtime module * add openai provider module * register openai provider module in providers lib * add realtime() fn that invokes OpenAI GA realtime API end to end * register realtime route module in providers lib * wire tokio/tokio-tungstenite/futures-util into providers crate * add tokio, tokio-tungstenite, futures-util to rust workspace deps * update Cargo.lock for realtime websocket deps * docs: add litellm-rust provider/route contributor guide * add typed RealtimeEvent; make RealtimeTransformResult hold typed events * type RealtimeProviderConfig trait on RealtimeEvent instead of raw strings * type OpenAI realtime passthrough transforms on RealtimeEvent * type realtime() fn on RealtimeEvent end to end (parse/serialize at host edge) * docs: add typed-contracts core rule to core CLAUDE.md * harden complete_url: default bare host / unknown scheme to wss:// --------- Co-authored-by: Ishaan Jaffer <ishaanjaffer0324@gmail.com>
This commit is contained in:
ishaan-berri
GitHub
parent
c2e06890ad
commit
18406c2bad
14 changed files with 759 additions and 8 deletions
9
litellm-rust/ADDING_A_PROVIDER.md
Normal file
9
litellm-rust/ADDING_A_PROVIDER.md
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
# Adding a provider / route to litellm-rust
|
||||
|
||||
Three layers, same for every route (see `ocr` and `realtime` as references):
|
||||
|
||||
1. **Transform contract (pure)** — `crates/core/src/<route>/transformation.rs`: a `…ProviderConfig` trait (URL build + request/response transforms) + types in `types.rs`. No network, env, or auth.
|
||||
2. **Provider config (pure)** — `crates/providers/src/<provider>/<route>/transformation.rs`: implement that trait as a `const <PROVIDER>_<ROUTE>_CONFIG`, mirroring the Python provider tree. Add parity unit tests.
|
||||
3. **HTTP / transport (the host)** — `crates/providers/src/<route>.rs` (e.g. `ocr.rs`, `realtime.rs`): the callable fn (`run_ocr`, `realtime`). It resolves the key, builds the auth header, builds URL + transforms via the config, then does the network call. This is the only layer allowed to do I/O.
|
||||
|
||||
**Calling:** the host invokes the route fn — the Python bridge calls `run_ocr`; the `ai-gateway` server calls `realtime`. Register new modules in `lib.rs` / `mod.rs`, then run `cargo fmt && cargo clippy --workspace -- -D warnings && cargo test --workspace`.
|
||||
272
litellm-rust/Cargo.lock
generated
272
litellm-rust/Cargo.lock
generated
|
|
@ -26,12 +26,27 @@ version = "2.13.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.20.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.12.0"
|
||||
|
|
@ -60,6 +75,57 @@ version = "0.2.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation"
|
||||
version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"
|
||||
dependencies = [
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation-sys"
|
||||
version = "0.8.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "data-encoding"
|
||||
version = "2.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8"
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer",
|
||||
"crypto-common",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "displaydoc"
|
||||
version = "0.2.6"
|
||||
|
|
@ -135,6 +201,16 @@ dependencies = [
|
|||
"slab",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.17"
|
||||
|
|
@ -413,16 +489,19 @@ version = "0.1.0"
|
|||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
"thiserror 2.0.18",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-providers"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"litellm-core",
|
||||
"reqwest",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"tokio-tungstenite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -485,6 +564,12 @@ version = "1.21.4"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||
|
||||
[[package]]
|
||||
name = "openssl-probe"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "2.3.2"
|
||||
|
|
@ -607,7 +692,7 @@ dependencies = [
|
|||
"rustc-hash",
|
||||
"rustls",
|
||||
"socket2",
|
||||
"thiserror",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"web-time",
|
||||
|
|
@ -622,13 +707,13 @@ dependencies = [
|
|||
"bytes",
|
||||
"getrandom 0.3.4",
|
||||
"lru-slab",
|
||||
"rand",
|
||||
"rand 0.9.4",
|
||||
"ring",
|
||||
"rustc-hash",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"slab",
|
||||
"thiserror",
|
||||
"thiserror 2.0.18",
|
||||
"tinyvec",
|
||||
"tracing",
|
||||
"web-time",
|
||||
|
|
@ -663,14 +748,35 @@ version = "5.3.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rand_chacha 0.3.1",
|
||||
"rand_core 0.6.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.9.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
|
||||
dependencies = [
|
||||
"rand_chacha",
|
||||
"rand_core",
|
||||
"rand_chacha 0.9.0",
|
||||
"rand_core 0.9.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core 0.6.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -680,7 +786,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core",
|
||||
"rand_core 0.9.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
|
||||
dependencies = [
|
||||
"getrandom 0.2.17",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -766,6 +881,18 @@ dependencies = [
|
|||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-native-certs"
|
||||
version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d"
|
||||
dependencies = [
|
||||
"openssl-probe",
|
||||
"rustls-pki-types",
|
||||
"schannel",
|
||||
"security-framework",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-pki-types"
|
||||
version = "1.14.1"
|
||||
|
|
@ -799,6 +926,38 @@ version = "1.0.23"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
|
||||
|
||||
[[package]]
|
||||
name = "schannel"
|
||||
version = "0.1.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "security-framework"
|
||||
version = "3.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"core-foundation",
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
"security-framework-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "security-framework-sys"
|
||||
version = "2.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3"
|
||||
dependencies = [
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.228"
|
||||
|
|
@ -854,6 +1013,17 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha1"
|
||||
version = "0.10.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "2.0.1"
|
||||
|
|
@ -931,13 +1101,33 @@ version = "0.12.16"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "1.0.69"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
|
||||
dependencies = [
|
||||
"thiserror-impl 1.0.69",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "2.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
"thiserror-impl 2.0.18",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "1.0.69"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -987,9 +1177,21 @@ dependencies = [
|
|||
"mio",
|
||||
"pin-project-lite",
|
||||
"socket2",
|
||||
"tokio-macros",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-macros"
|
||||
version = "2.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-rustls"
|
||||
version = "0.26.4"
|
||||
|
|
@ -1000,6 +1202,22 @@ dependencies = [
|
|||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-tungstenite"
|
||||
version = "0.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"log",
|
||||
"rustls",
|
||||
"rustls-native-certs",
|
||||
"rustls-pki-types",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tungstenite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tower"
|
||||
version = "0.5.3"
|
||||
|
|
@ -1070,6 +1288,32 @@ version = "0.2.5"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
|
||||
|
||||
[[package]]
|
||||
name = "tungstenite"
|
||||
version = "0.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"bytes",
|
||||
"data-encoding",
|
||||
"http",
|
||||
"httparse",
|
||||
"log",
|
||||
"rand 0.8.6",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"sha1",
|
||||
"thiserror 1.0.69",
|
||||
"utf-8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.20.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
|
|
@ -1100,12 +1344,24 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "utf-8"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
|
||||
|
||||
[[package]]
|
||||
name = "utf8_iter"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "want"
|
||||
version = "0.3.1"
|
||||
|
|
|
|||
|
|
@ -19,3 +19,6 @@ reqwest = { version = "0.12", default-features = false, features = ["blocking",
|
|||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
thiserror = "2.0"
|
||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "time"] }
|
||||
tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] }
|
||||
futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] }
|
||||
|
|
|
|||
|
|
@ -22,6 +22,15 @@ Not allowed:
|
|||
- Provider-specific branching that belongs in `providers`.
|
||||
- Panics for user/provider-controlled input.
|
||||
|
||||
## Typed Contracts (core rule)
|
||||
|
||||
Trait and function boundaries MUST be strongly typed. No stringly-typed JSON
|
||||
(`&str` / `String` / `Vec<String>` / bare `serde_json::Value`) as a transform
|
||||
input or output. Parse wire bytes into typed structs/enums at the host edge;
|
||||
`core` and `providers` operate only on those types (e.g. `RealtimeEvent`,
|
||||
`RealtimeTransformResult`, `OcrRequestData`). A `type`-style discriminator is a
|
||||
typed field on a struct, not a raw string threaded through the API.
|
||||
|
||||
## Structure
|
||||
|
||||
Use route names directly under `src/`: `ocr`, future `messages`,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
pub mod error;
|
||||
pub mod ocr;
|
||||
pub mod realtime;
|
||||
|
||||
pub use error::{CoreError, CoreResult};
|
||||
|
|
|
|||
2
litellm-rust/crates/core/src/realtime/mod.rs
Normal file
2
litellm-rust/crates/core/src/realtime/mod.rs
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
pub mod transformation;
|
||||
pub mod types;
|
||||
22
litellm-rust/crates/core/src/realtime/transformation.rs
Normal file
22
litellm-rust/crates/core/src/realtime/transformation.rs
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
use crate::realtime::types::{RealtimeEvent, RealtimeTransformResult};
|
||||
use crate::CoreResult;
|
||||
|
||||
pub trait RealtimeProviderConfig {
|
||||
/// Build the upstream WebSocket URL (e.g. `wss://api.openai.com/v1/realtime?model=…`).
|
||||
/// Pure string construction only — no network, no env.
|
||||
fn complete_url(&self, api_base: Option<&str>, model: &str) -> String;
|
||||
|
||||
/// Transform a client → backend event before it is forwarded upstream.
|
||||
fn transform_realtime_request(
|
||||
&self,
|
||||
event: &RealtimeEvent,
|
||||
model: &str,
|
||||
) -> CoreResult<RealtimeTransformResult>;
|
||||
|
||||
/// Transform a backend → client event before it is forwarded downstream.
|
||||
fn transform_realtime_response(
|
||||
&self,
|
||||
event: &RealtimeEvent,
|
||||
model: &str,
|
||||
) -> CoreResult<RealtimeTransformResult>;
|
||||
}
|
||||
60
litellm-rust/crates/core/src/realtime/types.rs
Normal file
60
litellm-rust/crates/core/src/realtime/types.rs
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
/// A single realtime event exchanged over the WebSocket.
|
||||
///
|
||||
/// The `type` discriminator is a typed field; the remaining fields are
|
||||
/// preserved losslessly in `data` so a transform can pass an event through, or
|
||||
/// inspect/modify specific fields, without enumerating every event variant.
|
||||
/// Wire (de)serialization happens at the host edge — `core`/`providers` operate
|
||||
/// only on this typed form.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct RealtimeEvent {
|
||||
#[serde(rename = "type")]
|
||||
pub event_type: String,
|
||||
#[serde(flatten)]
|
||||
pub data: Map<String, Value>,
|
||||
}
|
||||
|
||||
/// One or more typed events produced by a realtime transform.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct RealtimeTransformResult {
|
||||
pub events: Vec<RealtimeEvent>,
|
||||
}
|
||||
|
||||
impl RealtimeTransformResult {
|
||||
/// Forward a single event unchanged (the OpenAI baseline).
|
||||
pub fn passthrough(event: RealtimeEvent) -> Self {
|
||||
Self {
|
||||
events: vec![event],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn event(raw: &str) -> RealtimeEvent {
|
||||
serde_json::from_str(raw).expect("valid event json")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn realtime_event_round_trips_type_and_extra_fields() {
|
||||
let raw = r#"{"type":"response.output_text.delta","delta":"hi","response_id":"r1"}"#;
|
||||
let parsed = event(raw);
|
||||
assert_eq!(parsed.event_type, "response.output_text.delta");
|
||||
assert_eq!(parsed.data.get("delta"), Some(&Value::String("hi".into())));
|
||||
// Re-serializing yields a semantically-equal event (key order may differ).
|
||||
let reparsed: RealtimeEvent =
|
||||
serde_json::from_str(&serde_json::to_string(&parsed).unwrap()).unwrap();
|
||||
assert_eq!(parsed, reparsed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn passthrough_produces_single_element_vec() {
|
||||
let parsed = event(r#"{"type":"session.update"}"#);
|
||||
let result = RealtimeTransformResult::passthrough(parsed.clone());
|
||||
assert_eq!(result.events, vec![parsed]);
|
||||
}
|
||||
}
|
||||
|
|
@ -9,6 +9,9 @@ repository.workspace = true
|
|||
litellm-core.workspace = true
|
||||
reqwest.workspace = true
|
||||
serde_json.workspace = true
|
||||
tokio.workspace = true
|
||||
tokio-tungstenite.workspace = true
|
||||
futures-util.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,2 +1,4 @@
|
|||
pub mod mistral;
|
||||
pub mod ocr;
|
||||
pub mod openai;
|
||||
pub mod realtime;
|
||||
|
|
|
|||
1
litellm-rust/crates/providers/src/openai/mod.rs
Normal file
1
litellm-rust/crates/providers/src/openai/mod.rs
Normal file
|
|
@ -0,0 +1 @@
|
|||
pub mod realtime;
|
||||
1
litellm-rust/crates/providers/src/openai/realtime/mod.rs
Normal file
1
litellm-rust/crates/providers/src/openai/realtime/mod.rs
Normal file
|
|
@ -0,0 +1 @@
|
|||
pub mod transformation;
|
||||
|
|
@ -0,0 +1,189 @@
|
|||
use litellm_core::realtime::transformation::RealtimeProviderConfig;
|
||||
use litellm_core::realtime::types::{RealtimeEvent, RealtimeTransformResult};
|
||||
use litellm_core::CoreResult;
|
||||
|
||||
/// Default OpenAI API base, used when the caller does not override `api_base`.
|
||||
pub const OPENAI_REALTIME_DEFAULT_API_BASE: &str = "https://api.openai.com";
|
||||
|
||||
/// Path appended to the resolved host base to reach the realtime endpoint.
|
||||
pub const OPENAI_REALTIME_PATH: &str = "/v1/realtime";
|
||||
|
||||
/// Percent-encode a query value, escaping any char outside the RFC 3986
|
||||
/// unreserved set (`A-Za-z0-9-._~`). Keeps us dependency-free; common realtime
|
||||
/// model slugs have no special chars, but this stays correct for the rest.
|
||||
fn percent_encode(value: &str) -> String {
|
||||
let mut encoded = String::with_capacity(value.len());
|
||||
for byte in value.bytes() {
|
||||
let unreserved = byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~');
|
||||
if unreserved {
|
||||
encoded.push(byte as char);
|
||||
} else {
|
||||
encoded.push('%');
|
||||
encoded.push_str(&format!("{byte:02X}"));
|
||||
}
|
||||
}
|
||||
encoded
|
||||
}
|
||||
|
||||
/// Build the realtime WebSocket URL, porting Python's `OpenAIRealtime._construct_url`.
|
||||
///
|
||||
/// Blank/whitespace `api_base` is treated as absent (guard at resolution time),
|
||||
/// falling back to the default. The scheme is swapped to its WebSocket
|
||||
/// equivalent (`https://`→`wss://`, `http://`→`ws://`); bases already using
|
||||
/// `ws`/`wss` are left untouched. A bare host or unrecognized scheme defaults to
|
||||
/// secure `wss://` so we never hand a scheme-less URL to the connector (this is
|
||||
/// a deliberate hardening over Python's `_construct_url`, which would emit a
|
||||
/// scheme-less URL here). A trailing `/` is trimmed before the path and
|
||||
/// `?model=<encoded>` are appended.
|
||||
pub fn complete_url(api_base: Option<&str>, model: &str) -> String {
|
||||
let base = api_base
|
||||
.map(str::trim)
|
||||
.filter(|base| !base.is_empty())
|
||||
.unwrap_or(OPENAI_REALTIME_DEFAULT_API_BASE);
|
||||
|
||||
let base = if let Some(rest) = base.strip_prefix("https://") {
|
||||
format!("wss://{rest}")
|
||||
} else if let Some(rest) = base.strip_prefix("http://") {
|
||||
format!("ws://{rest}")
|
||||
} else if base.starts_with("wss://") || base.starts_with("ws://") {
|
||||
base.to_string()
|
||||
} else {
|
||||
format!("wss://{base}")
|
||||
};
|
||||
|
||||
let base = base.trim_end_matches('/');
|
||||
|
||||
format!(
|
||||
"{base}{OPENAI_REALTIME_PATH}?model={}",
|
||||
percent_encode(model)
|
||||
)
|
||||
}
|
||||
|
||||
pub struct OpenAiRealtimeConfig;
|
||||
|
||||
pub const OPENAI_REALTIME_CONFIG: OpenAiRealtimeConfig = OpenAiRealtimeConfig;
|
||||
|
||||
impl RealtimeProviderConfig for OpenAiRealtimeConfig {
|
||||
fn complete_url(&self, api_base: Option<&str>, model: &str) -> String {
|
||||
complete_url(api_base, model)
|
||||
}
|
||||
|
||||
fn transform_realtime_request(
|
||||
&self,
|
||||
event: &RealtimeEvent,
|
||||
_model: &str,
|
||||
) -> CoreResult<RealtimeTransformResult> {
|
||||
Ok(RealtimeTransformResult::passthrough(event.clone()))
|
||||
}
|
||||
|
||||
fn transform_realtime_response(
|
||||
&self,
|
||||
event: &RealtimeEvent,
|
||||
_model: &str,
|
||||
) -> CoreResult<RealtimeTransformResult> {
|
||||
Ok(RealtimeTransformResult::passthrough(event.clone()))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn transform_realtime_request(
|
||||
event: &RealtimeEvent,
|
||||
model: &str,
|
||||
) -> CoreResult<RealtimeTransformResult> {
|
||||
OPENAI_REALTIME_CONFIG.transform_realtime_request(event, model)
|
||||
}
|
||||
|
||||
pub fn transform_realtime_response(
|
||||
event: &RealtimeEvent,
|
||||
model: &str,
|
||||
) -> CoreResult<RealtimeTransformResult> {
|
||||
OPENAI_REALTIME_CONFIG.transform_realtime_response(event, model)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn complete_url_defaults_to_openai_wss() {
|
||||
assert_eq!(
|
||||
complete_url(None, "gpt-4o-realtime-preview"),
|
||||
"wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn complete_url_blank_base_uses_default() {
|
||||
assert_eq!(
|
||||
complete_url(Some(" "), "gpt-4o-realtime-preview"),
|
||||
"wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn complete_url_swaps_http_to_ws() {
|
||||
assert_eq!(
|
||||
complete_url(Some("http://localhost:8080"), "gpt-4o-realtime-preview"),
|
||||
"ws://localhost:8080/v1/realtime?model=gpt-4o-realtime-preview"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn complete_url_dedupes_trailing_slash() {
|
||||
assert_eq!(
|
||||
complete_url(Some("https://api.openai.com/"), "gpt-4o-realtime-preview"),
|
||||
"wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn complete_url_custom_base() {
|
||||
assert_eq!(
|
||||
complete_url(Some("https://oai.azure.example"), "gpt-4o-realtime-preview"),
|
||||
"wss://oai.azure.example/v1/realtime?model=gpt-4o-realtime-preview"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn complete_url_preserves_existing_wss_scheme() {
|
||||
assert_eq!(
|
||||
complete_url(Some("wss://api.openai.com"), "gpt-realtime"),
|
||||
"wss://api.openai.com/v1/realtime?model=gpt-realtime"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn complete_url_bare_host_defaults_to_wss() {
|
||||
assert_eq!(
|
||||
complete_url(Some("api.openai.com"), "gpt-realtime"),
|
||||
"wss://api.openai.com/v1/realtime?model=gpt-realtime"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn complete_url_percent_encodes_model_space() {
|
||||
assert_eq!(
|
||||
complete_url(None, "gpt 4o"),
|
||||
"wss://api.openai.com/v1/realtime?model=gpt%204o"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transform_realtime_request_passthrough_preserves_event() {
|
||||
let event: RealtimeEvent =
|
||||
serde_json::from_str(r#"{"type":"session.update","session":{"voice":"alloy"}}"#)
|
||||
.expect("valid event");
|
||||
let result =
|
||||
transform_realtime_request(&event, "gpt-realtime").expect("passthrough is infallible");
|
||||
assert_eq!(result.events, vec![event]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transform_realtime_response_passthrough_preserves_event() {
|
||||
let event: RealtimeEvent =
|
||||
serde_json::from_str(r#"{"type":"response.output_audio.delta","delta":"abc=="}"#)
|
||||
.expect("valid event");
|
||||
let result =
|
||||
transform_realtime_response(&event, "gpt-realtime").expect("passthrough is infallible");
|
||||
assert_eq!(result.events, vec![event]);
|
||||
}
|
||||
}
|
||||
193
litellm-rust/crates/providers/src/realtime.rs
Normal file
193
litellm-rust/crates/providers/src/realtime.rs
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
//! End-to-end OpenAI realtime invocation.
|
||||
//!
|
||||
//! The host-facing entry point, mirroring `providers::ocr::run_ocr`: open the
|
||||
//! WebSocket to OpenAI, drive typed events through the pure
|
||||
//! `OPENAI_REALTIME_CONFIG` transforms, and collect the response events.
|
||||
//! Network, auth header, key resolution, and wire (de)serialization live here so
|
||||
//! the `transformation` module stays pure and typed.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use litellm_core::error::CoreError;
|
||||
use litellm_core::realtime::transformation::RealtimeProviderConfig;
|
||||
use litellm_core::realtime::types::RealtimeEvent;
|
||||
use litellm_core::CoreResult;
|
||||
use tokio_tungstenite::connect_async;
|
||||
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
|
||||
use tokio_tungstenite::tungstenite::http::header::AUTHORIZATION;
|
||||
use tokio_tungstenite::tungstenite::http::HeaderValue;
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
|
||||
use crate::openai::realtime::transformation::OPENAI_REALTIME_CONFIG;
|
||||
|
||||
/// Environment variable holding the OpenAI API key (last-resort fallback).
|
||||
const OPENAI_API_KEY_ENV: &str = "OPENAI_API_KEY";
|
||||
|
||||
/// Default overall ceiling for a single realtime invocation.
|
||||
const DEFAULT_TIMEOUT_SECS: u64 = 60;
|
||||
|
||||
const MISSING_KEY_MESSAGE: &str = "Missing OpenAI API Key - a realtime call is being made but no key was passed via params or the OPENAI_API_KEY environment variable";
|
||||
|
||||
/// Resolve the OpenAI API key from the explicit param or the environment.
|
||||
///
|
||||
/// Blank/whitespace values are treated as absent (guard at resolution time).
|
||||
fn resolve_api_key(api_key: Option<&str>) -> CoreResult<String> {
|
||||
api_key
|
||||
.map(str::trim)
|
||||
.filter(|key| !key.is_empty())
|
||||
.map(str::to_string)
|
||||
.or_else(|| {
|
||||
std::env::var(OPENAI_API_KEY_ENV)
|
||||
.ok()
|
||||
.filter(|key| !key.trim().is_empty())
|
||||
})
|
||||
.ok_or_else(|| CoreError::Auth(MISSING_KEY_MESSAGE.to_string()))
|
||||
}
|
||||
|
||||
/// True for events that end a realtime turn: a completed response or an error.
|
||||
fn is_terminal_event(event: &RealtimeEvent) -> bool {
|
||||
event.event_type == "response.done" || event.event_type == "error"
|
||||
}
|
||||
|
||||
/// Invoke the OpenAI realtime API end to end over a WebSocket.
|
||||
///
|
||||
/// Sends each `input_events` entry after passing it through
|
||||
/// `transform_realtime_request`, then collects backend events — each passed
|
||||
/// through `transform_realtime_response` — until a terminal event
|
||||
/// (`response.done` / `error`) arrives, the socket closes, or the `timeout`
|
||||
/// elapses. Returns the transformed backend events in arrival order.
|
||||
///
|
||||
/// Mirrors `run_ocr`: pure transforms come from `core`/`providers`; the network,
|
||||
/// auth header, key resolution, and JSON (de)serialization are owned here.
|
||||
pub async fn realtime(
|
||||
model: &str,
|
||||
input_events: Vec<RealtimeEvent>,
|
||||
api_key: Option<&str>,
|
||||
api_base: Option<&str>,
|
||||
timeout: Option<Duration>,
|
||||
) -> CoreResult<Vec<RealtimeEvent>> {
|
||||
let config = &OPENAI_REALTIME_CONFIG;
|
||||
let api_key = resolve_api_key(api_key)?;
|
||||
let url = config.complete_url(api_base, model);
|
||||
|
||||
let mut request = url
|
||||
.as_str()
|
||||
.into_client_request()
|
||||
.map_err(|err| CoreError::Network(err.to_string()))?;
|
||||
// GA realtime API: only Authorization is needed. The legacy
|
||||
// `OpenAI-Beta: realtime=v1` header opts into the now-removed beta request
|
||||
// shape and triggers `beta_api_shape_disabled`, so we do not send it.
|
||||
request.headers_mut().insert(
|
||||
AUTHORIZATION,
|
||||
HeaderValue::from_str(&format!("Bearer {api_key}"))
|
||||
.map_err(|err| CoreError::Auth(err.to_string()))?,
|
||||
);
|
||||
|
||||
let (mut ws, _response) = connect_async(request)
|
||||
.await
|
||||
.map_err(|err| CoreError::Network(err.to_string()))?;
|
||||
|
||||
for event in &input_events {
|
||||
for outbound in config.transform_realtime_request(event, model)?.events {
|
||||
let payload = serde_json::to_string(&outbound)
|
||||
.map_err(|err| CoreError::InvalidResponse(err.to_string()))?;
|
||||
ws.send(Message::Text(payload))
|
||||
.await
|
||||
.map_err(|err| CoreError::Network(err.to_string()))?;
|
||||
}
|
||||
}
|
||||
|
||||
let deadline = timeout.unwrap_or_else(|| Duration::from_secs(DEFAULT_TIMEOUT_SECS));
|
||||
let mut received: Vec<RealtimeEvent> = Vec::new();
|
||||
|
||||
let collect = async {
|
||||
while let Some(message) = ws.next().await {
|
||||
match message.map_err(|err| CoreError::Network(err.to_string()))? {
|
||||
Message::Text(text) => {
|
||||
let event: RealtimeEvent = serde_json::from_str(text.as_str())
|
||||
.map_err(|err| CoreError::InvalidResponse(err.to_string()))?;
|
||||
for outbound in config.transform_realtime_response(&event, model)?.events {
|
||||
let terminal = is_terminal_event(&outbound);
|
||||
received.push(outbound);
|
||||
if terminal {
|
||||
return Ok::<(), CoreError>(());
|
||||
}
|
||||
}
|
||||
}
|
||||
Message::Close(_) => return Ok(()),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
|
||||
tokio::time::timeout(deadline, collect)
|
||||
.await
|
||||
.map_err(|_| CoreError::Network("realtime call timed out".to_string()))??;
|
||||
|
||||
Ok(received)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn event(raw: &str) -> RealtimeEvent {
|
||||
serde_json::from_str(raw).expect("valid event json")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_api_key_prefers_param_then_blank_falls_through() {
|
||||
assert_eq!(resolve_api_key(Some("sk-test")).unwrap(), "sk-test");
|
||||
// A blank param with no env set should error.
|
||||
if std::env::var(OPENAI_API_KEY_ENV).is_err() {
|
||||
assert!(resolve_api_key(Some(" ")).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_terminal_event_matches_done_and_error_only() {
|
||||
assert!(is_terminal_event(&event(r#"{"type":"response.done"}"#)));
|
||||
assert!(is_terminal_event(&event(r#"{"type":"error","error":{}}"#)));
|
||||
assert!(!is_terminal_event(&event(
|
||||
r#"{"type":"response.output_text.delta"}"#
|
||||
)));
|
||||
}
|
||||
|
||||
/// Live end-to-end check against OpenAI. Ignored by default (CI never runs
|
||||
/// it); run explicitly with `OPENAI_API_KEY` set:
|
||||
/// `cargo test -p litellm-providers realtime_invokes_openai -- --ignored --nocapture`
|
||||
#[tokio::test]
|
||||
#[ignore = "hits the live OpenAI realtime API; needs OPENAI_API_KEY"]
|
||||
async fn realtime_invokes_openai_and_responds() {
|
||||
let key =
|
||||
std::env::var(OPENAI_API_KEY_ENV).expect("set OPENAI_API_KEY to run this ignored test");
|
||||
|
||||
let response_create = event(
|
||||
r#"{"type":"response.create","response":{"output_modalities":["text"],"instructions":"Respond with exactly: hello world"}}"#,
|
||||
);
|
||||
|
||||
let events = realtime(
|
||||
"gpt-realtime",
|
||||
vec![response_create],
|
||||
Some(&key),
|
||||
None,
|
||||
Some(Duration::from_secs(30)),
|
||||
)
|
||||
.await
|
||||
.expect("realtime call should succeed");
|
||||
|
||||
let types: Vec<&str> = events.iter().map(|e| e.event_type.as_str()).collect();
|
||||
eprintln!("received {} events: {:?}", events.len(), types);
|
||||
|
||||
assert!(
|
||||
types.contains(&"response.done"),
|
||||
"expected a response.done event, got: {types:?}"
|
||||
);
|
||||
assert!(
|
||||
types.contains(&"response.output_text.delta"),
|
||||
"expected streamed text output, got: {types:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue