mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
fix(http): open WebSockets through the pooled reqwest client
The Responses WebSocket built its own rustls config from native roots, so it ignored ssl_verify, the CA bundle, the client certificate and HTTPS_PROXY/NO_PROXY. The upgrade GET now goes through a ClientVariant::WebSocket client (HTTP/1.1 only, no redirects) and tungstenite only frames the upgraded stream, since tungstenite has no proxy support (snapview/tungstenite-rs#177) tests/websocket.rs covers a private CA, ssl_verify off, mTLS, a CONNECT tunnel proxy, an h2-offering server, a refused upgrade and a wrong accept key Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
626f4d7724
commit
8c4d08f1ca
11 changed files with 486 additions and 150 deletions
8
litellm-rust/Cargo.lock
generated
8
litellm-rust/Cargo.lock
generated
|
|
@ -3967,13 +3967,13 @@ dependencies = [
|
|||
"rmcp",
|
||||
"rstest",
|
||||
"rustls 0.23.42",
|
||||
"rustls-native-certs",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"strum",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"tokio-rustls 0.26.4",
|
||||
"tokio-tungstenite",
|
||||
"tracing",
|
||||
"veil",
|
||||
|
|
@ -7430,11 +7430,7 @@ checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9"
|
|||
dependencies = [
|
||||
"futures-util",
|
||||
"log",
|
||||
"rustls 0.23.42",
|
||||
"rustls-native-certs",
|
||||
"rustls-pki-types",
|
||||
"tokio",
|
||||
"tokio-rustls 0.26.4",
|
||||
"tungstenite",
|
||||
]
|
||||
|
||||
|
|
@ -7779,8 +7775,6 @@ dependencies = [
|
|||
"httparse",
|
||||
"log",
|
||||
"rand 0.8.7",
|
||||
"rustls 0.23.42",
|
||||
"rustls-pki-types",
|
||||
"sha1 0.10.7",
|
||||
"thiserror 1.0.69",
|
||||
"utf-8",
|
||||
|
|
|
|||
|
|
@ -99,7 +99,6 @@ rustls = { version = "0.23", default-features = false, features = ["ring", "std"
|
|||
rustify = "=0.7.0"
|
||||
rustify_derive = "=0.5.5"
|
||||
vaultrs = { version = "=0.8.0", default-features = false, features = ["rustls"] }
|
||||
rustls-native-certs = "0.8"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = { version = "1.0", features = ["float_roundtrip"] }
|
||||
serde_with = { version = "=3.21.0", default-features = false, features = ["std", "macros"] }
|
||||
|
|
@ -113,7 +112,7 @@ thiserror = "2.0"
|
|||
tokenizers = { version = "0.23.1", default-features = false, features = ["onig"] }
|
||||
tiktoken-rs = "0.12.0"
|
||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"] }
|
||||
tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] }
|
||||
tokio-tungstenite = { version = "0.24", default-features = false, features = ["handshake"] }
|
||||
futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] }
|
||||
base64 = "0.22"
|
||||
flate2 = "1"
|
||||
|
|
|
|||
|
|
@ -8,14 +8,17 @@
|
|||
|
||||
## What lives here
|
||||
|
||||
- `pool.rs`: `HttpClientPool`, keyed by `(HttpClientConfig, ClientVariant)` with a TTL. `ClientVariant` is the full list of ways a client may differ from the configured one (redirect policy, pinned DNS, dropped client certificate and proxies). Add a variant only for a difference that must be fixed when the client is built, otherwise set it on the request
|
||||
- `pool.rs`: `HttpClientPool`, keyed by `(HttpClientConfig, ClientVariant)` with a TTL. `ClientVariant` is the full list of ways a client may differ from the configured one (redirect policy, pinned DNS, dropped client certificate and proxies, HTTP/1.1 only for WebSocket). Add a variant only for a difference that must be fixed when the client is built, otherwise set it on the request
|
||||
- `config.rs` and `tls.rs`: `HttpClientConfig` to `reqwest::ClientBuilder` and to `rustls::ClientConfig`
|
||||
- `proxy.rs`: `EnvironmentProxies` to reqwest proxies, and the URL matcher the media fetcher uses to decide whether a URL leaves through the proxy
|
||||
- `media.rs`: the SSRF-safe `MediaFetcher`. It is here because address pinning is a build-time property of the client (`dns_resolver2`) and because it needs the proxy matcher. `UrlPolicy` and `DownloadPolicy` are the host's inputs, the host never decides a URL is safe on its own
|
||||
- `request.rs`: case-insensitive header helpers over `Vec<(String, String)>` and `HeaderPolicy` for forwarding. They keep existing precedence, duplicate handling and unrelated forwarded headers. `string_headers` and `HeaderError` validate the shape of a caller's `extra_headers` and are the only place a request field is named
|
||||
- `outbound.rs`: `OutboundRequest` serializes the body once and a `RequestSigner` sees those exact bytes. A route builds it after its hooks ran and cannot change it afterwards
|
||||
- `transport.rs`: `Error::{Http, Network, Connect}` is the one transport error every crate above nests. `Connect` means the request never reached the provider, so a caller may reroute. Messages never carry URLs
|
||||
- `websocket.rs` and `mcp.rs`: the two connections the reqwest 0.12 builder cannot make. `mcp.rs` repeats the builder chain for the reqwest 0.13 that rmcp needs, delete it when the two versions converge
|
||||
- `websocket.rs`: the WebSocket client handshake over a pooled `ClientVariant::WebSocket` client. reqwest sends the upgrade GET and owns TCP, TLS and proxies, so a WebSocket honors `ssl_verify`, the CA bundle, the client certificate and `HTTPS_PROXY`/`NO_PROXY` exactly like an HTTP call. tungstenite only frames the upgraded stream (`from_raw_socket`) and supplies the key and accept-key math. Never give tungstenite its own TCP or TLS connector (`connect_async*`, a rustls config, native roots): tungstenite has no proxy support (snapview/tungstenite-rs#177) and a second TLS stack drifts from `tls.rs`
|
||||
- The WebSocket variant forces `http2: false` because an upgrade only exists in HTTP/1.1, and with `h2` in ALPN a server may pick h2 and the upgrade fails. Redirects are off, a WebSocket handshake never follows one
|
||||
- A refused upgrade is `transport::Error::Http` with the status and body, a wrong `Sec-WebSocket-Accept` is `Network`, and anything before the request left (a bad URL or scheme included) is `Connect`
|
||||
- `mcp.rs`: the connection the reqwest 0.12 builder cannot make. It repeats the builder chain for the reqwest 0.13 that rmcp needs, delete it when the two versions converge
|
||||
|
||||
## Boundary with consumers
|
||||
|
||||
|
|
@ -23,15 +26,11 @@
|
|||
- A consumer builds its request from the `Client` it was given. It never reads the environment, a config file or a settings snapshot to decide TLS, proxy or timeouts
|
||||
- Tests take `Client::for_test`, `plain_for_test` or `no_redirect_for_test` behind the `test-support` feature. Production code never calls them
|
||||
|
||||
## Known gaps
|
||||
|
||||
- `websocket.rs` builds its own rustls config from native roots and ignores `HttpClientConfig` (`ssl_verify`, CA bundle, client certificate, proxies). Make it take an `HttpClientConfig` like the pool does before adding another WebSocket caller
|
||||
- `HttpSettings::connect_timeout` has no layer field, so it is always the default
|
||||
|
||||
## Tests
|
||||
|
||||
- Verify observable request behavior (which headers were sent, which proxy was picked, whether a client was reused), never helper structure
|
||||
- TLS tests generate certificates with `rcgen` and run a local listener. They never depend on a public host
|
||||
- `tests/websocket.rs` is the transport contract for WebSockets. It runs a private CA with a `tokio-rustls` server that offers `h2` in ALPN (so a client that offers h2 breaks the upgrade), an mTLS server, and a CONNECT tunnel proxy that records the tunnel target. Any change to `tls.rs`, `proxy.rs` or the pool variants must keep it green, and a new TLS or proxy setting gets a case there too
|
||||
|
||||
# references
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ litellm-http-types.workspace = true
|
|||
hyper-util.workspace = true
|
||||
reqwest.workspace = true
|
||||
rustls.workspace = true
|
||||
rustls-native-certs.workspace = true
|
||||
tokio-tungstenite.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
|
@ -33,5 +32,6 @@ webpki-roots.workspace = true
|
|||
futures-util.workspace = true
|
||||
rcgen = "0.14.10"
|
||||
tempfile.workspace = true
|
||||
tokio-rustls = { version = "0.26", default-features = false, features = ["ring"] }
|
||||
rstest.workspace = true
|
||||
tokio.workspace = true
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ pub enum ClientVariant {
|
|||
NoRedirect,
|
||||
Media,
|
||||
UnpinnedMedia,
|
||||
WebSocket,
|
||||
}
|
||||
|
||||
const CLIENT_TTL: Duration = Duration::from_secs(3600);
|
||||
|
|
@ -73,6 +74,10 @@ impl HttpClientPool {
|
|||
client_certificate: None,
|
||||
..config.clone()
|
||||
},
|
||||
ClientVariant::WebSocket => HttpClientConfig {
|
||||
http2: false,
|
||||
..config.clone()
|
||||
},
|
||||
ClientVariant::Provider | ClientVariant::NoRedirect => config.clone(),
|
||||
};
|
||||
let key = (effective, variant);
|
||||
|
|
@ -105,7 +110,7 @@ impl HttpClientPool {
|
|||
) -> reqwest::ClientBuilder {
|
||||
match variant {
|
||||
ClientVariant::Provider => builder,
|
||||
ClientVariant::NoRedirect | ClientVariant::UnpinnedMedia => {
|
||||
ClientVariant::NoRedirect | ClientVariant::UnpinnedMedia | ClientVariant::WebSocket => {
|
||||
builder.redirect(reqwest::redirect::Policy::none())
|
||||
}
|
||||
ClientVariant::Media => builder
|
||||
|
|
|
|||
|
|
@ -1,62 +1,86 @@
|
|||
use std::{
|
||||
io,
|
||||
sync::{Arc, OnceLock},
|
||||
use http::{
|
||||
HeaderMap, HeaderValue, StatusCode,
|
||||
header::{CONNECTION, SEC_WEBSOCKET_ACCEPT, SEC_WEBSOCKET_KEY, SEC_WEBSOCKET_VERSION, UPGRADE},
|
||||
};
|
||||
|
||||
use rustls::{ClientConfig, RootCertStore};
|
||||
use tokio::net::TcpStream;
|
||||
use reqwest::Url;
|
||||
use tokio_tungstenite::{
|
||||
Connector, MaybeTlsStream, WebSocketStream, connect_async_tls_with_config,
|
||||
tungstenite::{client::IntoClientRequest, error::TlsError, handshake::client::Response},
|
||||
WebSocketStream,
|
||||
tungstenite::{
|
||||
handshake::{client::generate_key, derive_accept_key},
|
||||
protocol::Role,
|
||||
},
|
||||
};
|
||||
|
||||
pub type UpstreamWebSocket = WebSocketStream<MaybeTlsStream<TcpStream>>;
|
||||
use crate::{client::Client, transport::Error};
|
||||
|
||||
static TLS_CONFIG: OnceLock<Arc<ClientConfig>> = OnceLock::new();
|
||||
|
||||
fn build_tls_config() -> Result<ClientConfig, Box<tokio_tungstenite::tungstenite::Error>> {
|
||||
let native = rustls_native_certs::load_native_certs();
|
||||
let mut store = RootCertStore::empty();
|
||||
let (added, _ignored) = store.add_parsable_certificates(native.certs);
|
||||
if added == 0 {
|
||||
return Err(Box::new(tokio_tungstenite::tungstenite::Error::Io(
|
||||
io::Error::other(format!(
|
||||
"no usable native root certificates: {:?}",
|
||||
native.errors
|
||||
)),
|
||||
)));
|
||||
}
|
||||
ClientConfig::builder_with_provider(Arc::new(rustls::crypto::ring::default_provider()))
|
||||
.with_safe_default_protocol_versions()
|
||||
.map(|builder| builder.with_root_certificates(store).with_no_client_auth())
|
||||
.map_err(|error| {
|
||||
Box::new(tokio_tungstenite::tungstenite::Error::Tls(
|
||||
TlsError::Rustls(error),
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn tls_config() -> Result<Arc<ClientConfig>, Box<tokio_tungstenite::tungstenite::Error>> {
|
||||
if let Some(config) = TLS_CONFIG.get() {
|
||||
return Ok(Arc::clone(config));
|
||||
}
|
||||
let built = Arc::new(build_tls_config()?);
|
||||
Ok(Arc::clone(TLS_CONFIG.get_or_init(|| built)))
|
||||
}
|
||||
pub type UpstreamWebSocket = WebSocketStream<reqwest::Upgraded>;
|
||||
|
||||
#[tracing::instrument(name = "litellm.websocket.handshake", level = "debug", skip_all)]
|
||||
pub async fn connect_upstream<R>(
|
||||
request: R,
|
||||
) -> Result<(UpstreamWebSocket, Response), Box<tokio_tungstenite::tungstenite::Error>>
|
||||
where
|
||||
R: IntoClientRequest + Unpin,
|
||||
{
|
||||
let request = request.into_client_request().map_err(Box::new)?;
|
||||
let connector = match request.uri().scheme_str() {
|
||||
Some("wss") => Some(Connector::Rustls(tls_config()?)),
|
||||
_ => None,
|
||||
};
|
||||
connect_async_tls_with_config(request, None, false, connector)
|
||||
pub async fn connect(
|
||||
client: &Client,
|
||||
url: &str,
|
||||
headers: HeaderMap,
|
||||
) -> Result<UpstreamWebSocket, Error> {
|
||||
let key = generate_key();
|
||||
let response = client
|
||||
.get(http_url(url)?)
|
||||
.headers(handshake_headers(headers, &key)?)
|
||||
.send()
|
||||
.await
|
||||
.map_err(Box::new)
|
||||
.map_err(Error::from_reqwest_before_dispatch)?;
|
||||
let status = response.status();
|
||||
if status != StatusCode::SWITCHING_PROTOCOLS {
|
||||
return Err(Error::Http {
|
||||
status: status.as_u16(),
|
||||
body: response.text().await.unwrap_or_default(),
|
||||
});
|
||||
}
|
||||
let expected = derive_accept_key(key.as_bytes());
|
||||
if response
|
||||
.headers()
|
||||
.get(SEC_WEBSOCKET_ACCEPT)
|
||||
.map(HeaderValue::as_bytes)
|
||||
!= Some(expected.as_bytes())
|
||||
{
|
||||
return Err(Error::Network(
|
||||
"WebSocket handshake returned a wrong Sec-WebSocket-Accept".into(),
|
||||
));
|
||||
}
|
||||
let upgraded = response.upgrade().await?;
|
||||
Ok(WebSocketStream::from_raw_socket(upgraded, Role::Client, None).await)
|
||||
}
|
||||
|
||||
fn http_url(url: &str) -> Result<Url, Error> {
|
||||
let mut url = Url::parse(url).map_err(not_dispatched)?;
|
||||
let scheme = match url.scheme() {
|
||||
"ws" | "http" => "http",
|
||||
"wss" | "https" => "https",
|
||||
other => {
|
||||
return Err(not_dispatched(format!(
|
||||
"unsupported WebSocket URL scheme {other:?}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
url.set_scheme(scheme)
|
||||
.map_err(|()| not_dispatched("WebSocket URL cannot be sent over HTTP"))?;
|
||||
Ok(url)
|
||||
}
|
||||
|
||||
fn handshake_headers(headers: HeaderMap, key: &str) -> Result<HeaderMap, Error> {
|
||||
let key = HeaderValue::from_str(key).map_err(not_dispatched)?;
|
||||
Ok([
|
||||
(CONNECTION, HeaderValue::from_static("Upgrade")),
|
||||
(UPGRADE, HeaderValue::from_static("websocket")),
|
||||
(SEC_WEBSOCKET_VERSION, HeaderValue::from_static("13")),
|
||||
(SEC_WEBSOCKET_KEY, key),
|
||||
]
|
||||
.into_iter()
|
||||
.fold(headers, |mut headers, (name, value)| {
|
||||
headers.insert(name, value);
|
||||
headers
|
||||
}))
|
||||
}
|
||||
|
||||
fn not_dispatched(error: impl std::fmt::Display) -> Error {
|
||||
Error::Connect(error.to_string())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,59 +1,375 @@
|
|||
use std::{
|
||||
net::SocketAddr,
|
||||
path::PathBuf,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use litellm_http::websocket::connect_upstream;
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use litellm_http::{
|
||||
ClientIdentity, ClientVariant, EnvironmentProxies, HttpClientConfig, HttpClientPool,
|
||||
HttpSettings, Resolution, Verify, media::PublicDnsResolver, transport::Error,
|
||||
websocket::connect,
|
||||
};
|
||||
use rcgen::{BasicConstraints, CertificateParams, CertifiedIssuer, IsCa, KeyPair};
|
||||
use rstest::rstest;
|
||||
use tokio::net::TcpListener;
|
||||
use rustls::{
|
||||
RootCertStore, ServerConfig,
|
||||
crypto::ring,
|
||||
pki_types::{CertificateDer, PrivateKeyDer},
|
||||
server::WebPkiClientVerifier,
|
||||
};
|
||||
use tokio::{
|
||||
io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt},
|
||||
net::{TcpListener, TcpStream},
|
||||
};
|
||||
use tokio_rustls::TlsAcceptor;
|
||||
use tokio_tungstenite::{
|
||||
accept_hdr_async,
|
||||
tungstenite::{
|
||||
Message,
|
||||
client::IntoClientRequest,
|
||||
handshake::server::{Request, Response},
|
||||
},
|
||||
};
|
||||
|
||||
struct Pki {
|
||||
directory: tempfile::TempDir,
|
||||
ca: CertifiedIssuer<'static, KeyPair>,
|
||||
server_chain: Vec<CertificateDer<'static>>,
|
||||
server_key: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Pki {
|
||||
fn new() -> Self {
|
||||
let mut ca_params = CertificateParams::new(Vec::<String>::new()).unwrap();
|
||||
ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
|
||||
let ca = CertifiedIssuer::self_signed(ca_params, KeyPair::generate().unwrap()).unwrap();
|
||||
let server_key = KeyPair::generate().unwrap();
|
||||
let server = CertificateParams::new(vec!["localhost".into()])
|
||||
.unwrap()
|
||||
.signed_by(&server_key, &ca)
|
||||
.unwrap();
|
||||
let client_key = KeyPair::generate().unwrap();
|
||||
let client = CertificateParams::new(vec!["client".into()])
|
||||
.unwrap()
|
||||
.signed_by(&client_key, &ca)
|
||||
.unwrap();
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
std::fs::write(directory.path().join("ca.pem"), ca.pem()).unwrap();
|
||||
std::fs::write(
|
||||
directory.path().join("client.pem"),
|
||||
format!("{}{}", client.pem(), client_key.serialize_pem()),
|
||||
)
|
||||
.unwrap();
|
||||
Self {
|
||||
directory,
|
||||
server_chain: vec![server.der().clone()],
|
||||
server_key: server_key.serialize_der(),
|
||||
ca,
|
||||
}
|
||||
}
|
||||
|
||||
fn path(&self, name: &str) -> PathBuf {
|
||||
self.directory.path().join(name)
|
||||
}
|
||||
|
||||
fn acceptor(&self, require_client_certificate: bool) -> TlsAcceptor {
|
||||
let provider = Arc::new(ring::default_provider());
|
||||
let builder = ServerConfig::builder_with_provider(Arc::clone(&provider))
|
||||
.with_safe_default_protocol_versions()
|
||||
.unwrap();
|
||||
let builder = if require_client_certificate {
|
||||
let mut roots = RootCertStore::empty();
|
||||
roots.add(self.ca.der().clone()).unwrap();
|
||||
builder.with_client_cert_verifier(
|
||||
WebPkiClientVerifier::builder_with_provider(Arc::new(roots), provider)
|
||||
.build()
|
||||
.unwrap(),
|
||||
)
|
||||
} else {
|
||||
builder.with_no_client_auth()
|
||||
};
|
||||
let mut config = builder
|
||||
.with_single_cert(
|
||||
self.server_chain.clone(),
|
||||
PrivateKeyDer::try_from(self.server_key.clone()).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()];
|
||||
TlsAcceptor::from(Arc::new(config))
|
||||
}
|
||||
}
|
||||
|
||||
fn config(verify: Verify) -> HttpClientConfig {
|
||||
HttpClientConfig {
|
||||
verify,
|
||||
http2: true,
|
||||
..Resolution::from(&HttpSettings::default()).config
|
||||
}
|
||||
}
|
||||
|
||||
fn proxied_through(proxy: SocketAddr) -> EnvironmentProxies {
|
||||
EnvironmentProxies::from_environment(&move |name: &str| {
|
||||
(name == "HTTPS_PROXY").then(|| format!("http://{proxy}"))
|
||||
})
|
||||
}
|
||||
|
||||
async fn open(
|
||||
config: &HttpClientConfig,
|
||||
url: &str,
|
||||
headers: HeaderMap,
|
||||
) -> Result<litellm_http::websocket::UpstreamWebSocket, Error> {
|
||||
let client = HttpClientPool::new(Arc::new(PublicDnsResolver))
|
||||
.client(config, ClientVariant::WebSocket)
|
||||
.unwrap();
|
||||
connect(&client, url, headers).await
|
||||
}
|
||||
|
||||
async fn echo_once<S: AsyncRead + AsyncWrite + Unpin>(
|
||||
stream: S,
|
||||
seen: Arc<Mutex<Option<Request>>>,
|
||||
) {
|
||||
#[expect(
|
||||
clippy::result_large_err,
|
||||
reason = "tungstenite requires an unboxed handshake error response"
|
||||
)]
|
||||
let Ok(mut socket) = accept_hdr_async(stream, |request: &Request, response: Response| {
|
||||
*seen.lock().unwrap() = Some(request.clone());
|
||||
Ok(response)
|
||||
})
|
||||
.await
|
||||
else {
|
||||
return;
|
||||
};
|
||||
if let Some(Ok(message)) = socket.next().await {
|
||||
let _ = socket.send(message).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn serve_tls(
|
||||
pki: &Pki,
|
||||
require_client_certificate: bool,
|
||||
) -> (SocketAddr, Arc<Mutex<Option<Request>>>) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let acceptor = pki.acceptor(require_client_certificate);
|
||||
let seen = Arc::new(Mutex::new(None));
|
||||
let recorded = Arc::clone(&seen);
|
||||
tokio::spawn(async move {
|
||||
while let Ok((stream, _)) = listener.accept().await {
|
||||
if let Ok(stream) = acceptor.accept(stream).await {
|
||||
echo_once(stream, Arc::clone(&recorded)).await;
|
||||
}
|
||||
}
|
||||
});
|
||||
(address, seen)
|
||||
}
|
||||
|
||||
async fn read_head(stream: &mut TcpStream) -> String {
|
||||
let mut head = Vec::new();
|
||||
while !head.ends_with(b"\r\n\r\n") {
|
||||
let mut byte = [0u8; 1];
|
||||
if stream.read(&mut byte).await.unwrap() == 0 {
|
||||
break;
|
||||
}
|
||||
head.push(byte[0]);
|
||||
}
|
||||
String::from_utf8(head).unwrap()
|
||||
}
|
||||
|
||||
async fn serve_tunnel_proxy(target: SocketAddr) -> (SocketAddr, Arc<Mutex<Vec<String>>>) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let requests = Arc::new(Mutex::new(Vec::new()));
|
||||
let seen = Arc::clone(&requests);
|
||||
tokio::spawn(async move {
|
||||
while let Ok((mut client, _)) = listener.accept().await {
|
||||
let head = read_head(&mut client).await;
|
||||
seen.lock()
|
||||
.unwrap()
|
||||
.push(head.lines().next().unwrap_or_default().to_owned());
|
||||
let mut upstream = TcpStream::connect(target).await.unwrap();
|
||||
client
|
||||
.write_all(b"HTTP/1.1 200 Connection established\r\n\r\n")
|
||||
.await
|
||||
.unwrap();
|
||||
let _ = tokio::io::copy_bidirectional(&mut client, &mut upstream).await;
|
||||
}
|
||||
});
|
||||
(address, requests)
|
||||
}
|
||||
|
||||
async fn serve_raw(response: &'static str) -> SocketAddr {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
read_head(&mut stream).await;
|
||||
stream.write_all(response.as_bytes()).await.unwrap();
|
||||
let _ = stream.read(&mut [0u8; 1]).await;
|
||||
});
|
||||
address
|
||||
}
|
||||
|
||||
async fn round_trip(socket: &mut litellm_http::websocket::UpstreamWebSocket) {
|
||||
socket.send(Message::Text("ping".into())).await.unwrap();
|
||||
assert_eq!(
|
||||
socket.next().await.unwrap().unwrap(),
|
||||
Message::Text("ping".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::without_query("/responses")]
|
||||
#[case::with_query("/responses?model=test-model")]
|
||||
#[tokio::test]
|
||||
async fn connects_with_caller_headers_and_exchanges_frames(#[case] path: &'static str) {
|
||||
async fn plain_websocket_sends_caller_headers_and_path(#[case] path: &'static str) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let seen = Arc::new(Mutex::new(None));
|
||||
let recorded = Arc::clone(&seen);
|
||||
tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
#[expect(
|
||||
clippy::result_large_err,
|
||||
reason = "tungstenite requires an unboxed handshake error response"
|
||||
)]
|
||||
let mut socket = accept_hdr_async(stream, move |request: &Request, response: Response| {
|
||||
assert_eq!(request.uri().path_and_query().unwrap().as_str(), path);
|
||||
assert_eq!(request.headers()["authorization"], "Bearer test-key");
|
||||
Ok(response)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
socket.next().await.unwrap().unwrap(),
|
||||
Message::Text("request".into())
|
||||
);
|
||||
socket.send(Message::Text("response".into())).await.unwrap();
|
||||
socket.close(None).await.unwrap();
|
||||
echo_once(stream, recorded).await;
|
||||
});
|
||||
let mut request = format!("ws://{address}{path}")
|
||||
.into_client_request()
|
||||
.unwrap();
|
||||
request
|
||||
.headers_mut()
|
||||
.insert("authorization", "Bearer test-key".parse().unwrap());
|
||||
let (mut socket, response) = connect_upstream(request).await.unwrap();
|
||||
assert_eq!(response.status(), 101);
|
||||
socket.send(Message::Text("request".into())).await.unwrap();
|
||||
assert_eq!(
|
||||
socket.next().await.unwrap().unwrap(),
|
||||
Message::Text("response".into())
|
||||
);
|
||||
assert!(matches!(
|
||||
socket.next().await.unwrap().unwrap(),
|
||||
Message::Close(_)
|
||||
));
|
||||
server.await.unwrap();
|
||||
let headers = HeaderMap::from_iter([(
|
||||
http::header::AUTHORIZATION,
|
||||
HeaderValue::from_static("Bearer test-key"),
|
||||
)]);
|
||||
|
||||
let mut socket = open(
|
||||
&config(Verify::BuiltInRoots),
|
||||
&format!("ws://{address}{path}"),
|
||||
headers,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
round_trip(&mut socket).await;
|
||||
let request = seen.lock().unwrap().take().unwrap();
|
||||
assert_eq!(request.uri().path_and_query().unwrap().as_str(), path);
|
||||
assert_eq!(request.headers()["authorization"], "Bearer test-key");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::ca_bundle_trusts_the_private_ca(|pki: &Pki| Verify::CaBundle(pki.path("ca.pem")), true)]
|
||||
#[case::ssl_verify_off_skips_verification(|_: &Pki| Verify::Disabled, true)]
|
||||
#[case::built_in_roots_reject_the_private_ca(|_: &Pki| Verify::BuiltInRoots, false)]
|
||||
#[tokio::test]
|
||||
async fn wss_honors_ssl_verify(#[case] verify: fn(&Pki) -> Verify, #[case] trusted: bool) {
|
||||
let pki = Pki::new();
|
||||
let (address, _) = serve_tls(&pki, false).await;
|
||||
|
||||
let result = open(
|
||||
&config(verify(&pki)),
|
||||
&format!("wss://localhost:{}/responses", address.port()),
|
||||
HeaderMap::new(),
|
||||
)
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(mut socket) if trusted => round_trip(&mut socket).await,
|
||||
Err(Error::Connect(_)) if !trusted => {}
|
||||
other => panic!("unexpected handshake outcome: {:?}", other.map(drop)),
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::with_certificate(true)]
|
||||
#[case::without_certificate(false)]
|
||||
#[tokio::test]
|
||||
async fn wss_presents_the_configured_client_certificate(#[case] configured: bool) {
|
||||
let pki = Pki::new();
|
||||
let (address, _) = serve_tls(&pki, true).await;
|
||||
let config = HttpClientConfig {
|
||||
client_certificate: configured.then(|| ClientIdentity::Pem(pki.path("client.pem"))),
|
||||
..config(Verify::CaBundle(pki.path("ca.pem")))
|
||||
};
|
||||
|
||||
let result = open(
|
||||
&config,
|
||||
&format!("wss://localhost:{}/responses", address.port()),
|
||||
HeaderMap::new(),
|
||||
)
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(mut socket) if configured => round_trip(&mut socket).await,
|
||||
Err(_) if !configured => {}
|
||||
other => panic!("unexpected handshake outcome: {:?}", other.map(drop)),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn wss_tunnels_through_the_https_proxy() {
|
||||
let pki = Pki::new();
|
||||
let (target, _) = serve_tls(&pki, false).await;
|
||||
let (proxy, requests) = serve_tunnel_proxy(target).await;
|
||||
let config = HttpClientConfig {
|
||||
proxies: proxied_through(proxy),
|
||||
..config(Verify::CaBundle(pki.path("ca.pem")))
|
||||
};
|
||||
|
||||
let mut socket = open(
|
||||
&config,
|
||||
&format!("wss://localhost:{}/responses", target.port()),
|
||||
HeaderMap::new(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
round_trip(&mut socket).await;
|
||||
assert_eq!(
|
||||
*requests.lock().unwrap(),
|
||||
[format!("CONNECT localhost:{} HTTP/1.1", target.port())]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refused_upgrade_reports_the_status_and_body() {
|
||||
let address =
|
||||
serve_raw("HTTP/1.1 403 Forbidden\r\nContent-Length: 6\r\nConnection: close\r\n\r\ndenied")
|
||||
.await;
|
||||
|
||||
let result = open(
|
||||
&config(Verify::BuiltInRoots),
|
||||
&format!("ws://{address}/responses"),
|
||||
HeaderMap::new(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
result.map(drop),
|
||||
Err(Error::Http {
|
||||
status: 403,
|
||||
body: "denied".into()
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn upgrade_with_a_wrong_accept_key_is_rejected() {
|
||||
let address = serve_raw(
|
||||
"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Accept: wrong\r\n\r\n",
|
||||
)
|
||||
.await;
|
||||
|
||||
let result = open(
|
||||
&config(Verify::BuiltInRoots),
|
||||
&format!("ws://{address}/responses"),
|
||||
HeaderMap::new(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(matches!(result, Err(Error::Network(_))));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::non_websocket_scheme("ftp://localhost/responses")]
|
||||
#[case::unparseable("not a url")]
|
||||
#[tokio::test]
|
||||
async fn invalid_url_is_rejected_before_dispatch(#[case] url: &str) {
|
||||
let result = open(&config(Verify::BuiltInRoots), url, HeaderMap::new()).await;
|
||||
|
||||
assert!(matches!(result, Err(Error::Connect(_))));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ tokio-tungstenite.workspace = true
|
|||
tracing.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
litellm-http = { workspace = true, features = ["test-support"] }
|
||||
litellm-cache.workspace = true
|
||||
litellm-cache-memory.workspace = true
|
||||
litellm-host-native.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,12 +1,14 @@
|
|||
use std::{collections::HashMap, sync::Arc, time::Duration};
|
||||
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use litellm_http::websocket::{UpstreamWebSocket, connect_upstream};
|
||||
use litellm_http::{
|
||||
Client,
|
||||
websocket::{self, UpstreamWebSocket},
|
||||
};
|
||||
use tokio::sync::Mutex;
|
||||
use tokio_tungstenite::tungstenite::{
|
||||
Message,
|
||||
client::IntoClientRequest,
|
||||
http::{HeaderName, HeaderValue},
|
||||
http::{HeaderMap, HeaderName, HeaderValue},
|
||||
};
|
||||
|
||||
use super::Error;
|
||||
|
|
@ -24,42 +26,33 @@ impl ResponsesWebSocketConnection {
|
|||
fields(outcome)
|
||||
)]
|
||||
pub async fn connect_url(
|
||||
client: &Client,
|
||||
url: &str,
|
||||
headers: &HashMap<String, String>,
|
||||
timeout: Option<Duration>,
|
||||
) -> Result<Self, Error> {
|
||||
litellm_inference::diagnostic::operation("litellm.websocket.connect_url", async {
|
||||
let mut request = url.into_client_request().map_err(|error| {
|
||||
Error::Transport(litellm_http::transport::Error::Network(error.to_string()))
|
||||
})?;
|
||||
for (name, value) in headers {
|
||||
let header_name = name
|
||||
.parse::<HeaderName>()
|
||||
.map_err(|error| Error::InvalidRequest(error.to_string().into()))?;
|
||||
let header_value = HeaderValue::from_str(value)
|
||||
.map_err(|error| Error::InvalidRequest(error.to_string().into()))?;
|
||||
request.headers_mut().insert(header_name, header_value);
|
||||
}
|
||||
let connect = connect_upstream(request);
|
||||
let result = match timeout {
|
||||
let headers = headers
|
||||
.iter()
|
||||
.map(|(name, value)| {
|
||||
Ok((
|
||||
name.parse::<HeaderName>()
|
||||
.map_err(|error| Error::InvalidRequest(error.to_string().into()))?,
|
||||
HeaderValue::from_str(value)
|
||||
.map_err(|error| Error::InvalidRequest(error.to_string().into()))?,
|
||||
))
|
||||
})
|
||||
.collect::<Result<HeaderMap, Error>>()?;
|
||||
let connect = websocket::connect(client, url, headers);
|
||||
let socket = match timeout {
|
||||
Some(timeout) => tokio::time::timeout(timeout, connect).await.map_err(|_| {
|
||||
Error::Transport(litellm_http::transport::Error::Network(
|
||||
"Responses WebSocket connection timed out".into(),
|
||||
))
|
||||
})?,
|
||||
None => connect.await,
|
||||
};
|
||||
let (socket, _) = result.map_err(|error| match *error {
|
||||
tokio_tungstenite::tungstenite::Error::Http(response) => {
|
||||
Error::Transport(litellm_http::transport::Error::Http {
|
||||
status: response.status().as_u16(),
|
||||
body: String::new(),
|
||||
})
|
||||
}
|
||||
other => {
|
||||
Error::Transport(litellm_http::transport::Error::Network(other.to_string()))
|
||||
}
|
||||
})?;
|
||||
}
|
||||
.map_err(Error::Transport)?;
|
||||
Ok(Self {
|
||||
socket: Arc::new(Mutex::new(Some(socket))),
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,20 +1,20 @@
|
|||
use futures_util::TryStreamExt;
|
||||
use rstest::{fixture, rstest};
|
||||
use serde_json::json;
|
||||
use std::sync::Arc;
|
||||
use wiremock::ResponseTemplate;
|
||||
|
||||
use litellm_host::{call::HostedCompletion, lifecycle::CallEvent};
|
||||
use litellm_host::{
|
||||
interceptors::{ExecutionFacts, ResultSource},
|
||||
lifecycle::ExecutionEvent,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
|
||||
use futures_util::TryStreamExt;
|
||||
use litellm_host::{call::HostedCompletion, lifecycle::CallEvent};
|
||||
use litellm_inference_responses::{
|
||||
Error,
|
||||
route::Responses,
|
||||
types::{ResponsesCall, ResponsesOutput},
|
||||
};
|
||||
use litellm_inference_testing::{RecordingSecrets, no_secrets};
|
||||
use rstest::{fixture, rstest};
|
||||
use serde_json::json;
|
||||
use wiremock::ResponseTemplate;
|
||||
|
||||
mod support;
|
||||
use support::*;
|
||||
|
|
@ -538,6 +538,7 @@ async fn websocket_operations_trace_outcomes_without_capturing_frames_or_credent
|
|||
.logger()
|
||||
.instrument(async {
|
||||
let connection = ResponsesWebSocketConnection::connect_url(
|
||||
&litellm_http::Client::plain_for_test(),
|
||||
&format!("ws://{address}/responses"),
|
||||
&std::collections::HashMap::from([(
|
||||
"authorization".into(),
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
use litellm_inference_responses::websocket::ResponsesWebSocketConnection as RustResponsesWebSocketConnection;
|
||||
use pyo3::prelude::*;
|
||||
use serde_json::Value;
|
||||
|
||||
use litellm_http::ClientVariant;
|
||||
use litellm_inference_responses::websocket::ResponsesWebSocketConnection as RustResponsesWebSocketConnection;
|
||||
|
||||
use crate::{
|
||||
errors::route_error_to_pyerr,
|
||||
marshal::{marshal_headers, optional_timeout},
|
||||
|
|
@ -25,10 +27,12 @@ impl ResponsesWebSocketConnection {
|
|||
) -> PyResult<Bound<'py, PyAny>> {
|
||||
let headers = marshal_headers(headers)?;
|
||||
let timeout = optional_timeout(timeout_seconds);
|
||||
let client = crate::http::host_client(py, ClientVariant::WebSocket)?;
|
||||
crate::execution::run_async_value(py, async move {
|
||||
let inner = RustResponsesWebSocketConnection::connect_url(&url, &headers, timeout)
|
||||
.await
|
||||
.map_err(route_error_to_pyerr)?;
|
||||
let inner =
|
||||
RustResponsesWebSocketConnection::connect_url(&client, &url, &headers, timeout)
|
||||
.await
|
||||
.map_err(route_error_to_pyerr)?;
|
||||
Ok(ResponsesWebSocketConnection { inner })
|
||||
})
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue