refactor(core): centralize HTTP request helpers

This commit is contained in:
Yujong Lee 2026-09-09 19:31:37 -07:00
parent e11be9de21
commit 89080c7af8
7 changed files with 366 additions and 4 deletions

View file

@ -1458,6 +1458,7 @@ dependencies = [
"tokio",
"tracing",
"tracing-subscriber",
"url",
]
[[package]]

View file

@ -39,6 +39,7 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"]
tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] }
futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] }
base64 = "0.22"
url = "2.5.8"
[profile.release]
opt-level = 3

View file

@ -15,6 +15,7 @@ thiserror.workspace = true
tracing.workspace = true
tracing-subscriber = { workspace = true, optional = true }
sha2.workspace = true
url.workspace = true
aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true }
aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"], optional = true }
aws-sdk-sts = { version = "1.108.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true }

View file

@ -36,6 +36,34 @@ pub enum Error {
Unsupported(&'static str),
}
#[derive(Clone, Debug, ThisError, PartialEq, Eq)]
pub enum TransportError {
#[error("upstream request failed with status {status}: {body}")]
Http { status: u16, body: String },
#[error("upstream network error: {0}")]
Network(String),
#[error("could not reach the provider: {0}")]
Connect(String),
}
impl TransportError {
pub fn from_reqwest_before_dispatch(error: reqwest::Error) -> Self {
let before_dispatch = !error.is_timeout() && (error.is_connect() || error.is_builder());
let message = error.without_url().to_string();
if before_dispatch {
Self::Connect(message)
} else {
Self::Network(message)
}
}
}
impl From<reqwest::Error> for TransportError {
fn from(error: reqwest::Error) -> Self {
Self::Network(error.without_url().to_string())
}
}
pub fn json_type_name(value: &serde_json::Value) -> &'static str {
match value {
serde_json::Value::Null => "null",
@ -46,3 +74,53 @@ pub fn json_type_name(value: &serde_json::Value) -> &'static str {
serde_json::Value::Object(_) => "object",
}
}
#[cfg(test)]
mod transport_tests {
use super::*;
#[tokio::test]
async fn transport_errors_remove_urls_and_keep_dispatch_context() {
let error = reqwest::Client::builder()
.no_proxy()
.build()
.expect("client")
.get("http://localhost:invalid/private?api_key=secret")
.send()
.await
.expect_err("invalid port");
let error = TransportError::from_reqwest_before_dispatch(error);
assert!(matches!(error, TransportError::Connect(_)));
assert!(!error.to_string().contains("secret"));
assert!(!error.to_string().contains("private"));
}
#[tokio::test]
async fn request_timeout_is_not_safe_to_retry_as_a_connect_failure() {
use std::time::Duration;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("bind");
let address = listener.local_addr().expect("address");
let request = reqwest::Client::builder()
.no_proxy()
.build()
.expect("client")
.get(format!("http://{address}"))
.timeout(Duration::from_millis(200))
.send();
let (response, accepted) = tokio::join!(
request,
tokio::time::timeout(Duration::from_secs(2), listener.accept())
);
let _connection = accepted
.expect("accept deadline")
.expect("accepted connection");
let error = response.expect_err("server does not respond");
assert!(error.is_timeout());
assert!(matches!(
TransportError::from_reqwest_before_dispatch(error),
TransportError::Network(_)
));
}
}

View file

@ -1,10 +1,43 @@
//! Header and upstream-body helpers shared by every route module.
use serde_json::{Map, Value};
use crate::constants::UPSTREAM_ERROR_BODY_MAX_CHARS;
use crate::error::{Error, json_type_name};
#[allow(
dead_code,
reason = "used by the OCR architecture in the next stacked PR"
)]
pub(crate) enum HeaderPolicy<'a> {
All,
Only(&'a [&'a str]),
Except(&'a [&'a str]),
}
#[allow(
dead_code,
reason = "used by the OCR architecture in the next stacked PR"
)]
pub(crate) fn with_headers(
builder: reqwest::RequestBuilder,
headers: &[(String, String)],
policy: HeaderPolicy<'_>,
) -> reqwest::RequestBuilder {
headers
.iter()
.filter(|(name, _)| match policy {
HeaderPolicy::All => true,
HeaderPolicy::Only(names) => names
.iter()
.any(|allowed| name.eq_ignore_ascii_case(allowed)),
HeaderPolicy::Except(names) => !names
.iter()
.any(|excluded| name.eq_ignore_ascii_case(excluded)),
})
.fold(builder, |builder, (name, value)| {
builder.header(name, value)
})
}
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
pub async fn http_request(
request: reqwest::RequestBuilder,
@ -12,8 +45,6 @@ pub async fn http_request(
request.send().await
}
/// Bound an upstream error body before it crosses a host boundary, so provider
/// bodies stay data-minimized.
pub fn truncate_error_body(body: &str) -> String {
if body.chars().count() <= UPSTREAM_ERROR_BODY_MAX_CHARS {
return body.to_string();
@ -61,11 +92,77 @@ pub fn has_bearer_auth(headers: &[(String, String)]) -> bool {
})
}
#[allow(
dead_code,
reason = "used by the OCR architecture in the next stacked PR"
)]
pub(crate) fn deserialize_optional_param<'de, D, T>(
deserializer: D,
) -> Result<Option<Option<T>>, D::Error>
where
D: serde::Deserializer<'de>,
T: serde::Deserialize<'de>,
{
<Option<T> as serde::Deserialize>::deserialize(deserializer).map(Some)
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[rstest::rstest]
#[case(HeaderPolicy::All, true, true)]
#[case(HeaderPolicy::Only(&["authorization"]), true, false)]
#[case(HeaderPolicy::Except(&["authorization"]), false, true)]
fn forwarding_policy_preserves_matching_headers_and_duplicates(
#[case] policy: HeaderPolicy<'_>,
#[case] auth: bool,
#[case] trace: bool,
) {
let request = with_headers(
reqwest::Client::new().get("https://example.com"),
&[
("AuThOrIzAtIoN".into(), "Bearer token".into()),
("X-Trace".into(), "first".into()),
("x-trace".into(), "second".into()),
],
policy,
)
.build()
.unwrap();
assert_eq!(request.headers().contains_key("authorization"), auth);
let traces: Vec<_> = request.headers().get_all("x-trace").iter().collect();
if trace {
assert_eq!(traces, ["first", "second"]);
} else {
assert!(traces.is_empty());
}
}
#[test]
fn multipart_policy_leaves_content_headers_to_reqwest() {
let request = with_headers(
reqwest::Client::new()
.post("https://example.com")
.multipart(reqwest::multipart::Form::new().text("file", "abc")),
&[
("Content-Type".into(), "application/json".into()),
("CONTENT-LENGTH".into(), "0".into()),
],
HeaderPolicy::Except(&["content-type", "content-length"]),
)
.build()
.unwrap();
assert!(
request.headers()["content-type"]
.to_str()
.unwrap()
.starts_with("multipart/form-data; boundary=")
);
assert_ne!(request.headers()["content-length"], "0");
}
#[test]
fn truncate_leaves_short_bodies_untouched() {
assert_eq!(truncate_error_body("short"), "short");

View file

@ -14,5 +14,6 @@ pub mod realtime;
pub mod responses;
pub mod router;
pub mod routing_utils;
mod url_utils;
pub use error::Error;

View file

@ -0,0 +1,183 @@
#![allow(
dead_code,
reason = "used by the OCR architecture in the next stacked PR"
)]
use std::marker::PhantomData;
use thiserror::Error;
use url::Url;
#[derive(Debug, Error)]
pub(crate) enum ApiUrlError {
#[error("invalid URL: {0}")]
Parse(#[from] url::ParseError),
#[error("URL cannot be used as a base")]
CannotBeBase,
#[error("unsupported URL scheme: {0}")]
Scheme(String),
}
pub(crate) struct Base;
pub(crate) struct Complete;
pub(crate) struct ApiUrl<State> {
url: Url,
state: PhantomData<State>,
}
impl ApiUrl<Base> {
pub(crate) fn parse(value: &str) -> Result<Self, ApiUrlError> {
Ok(Self {
url: Url::parse(value.trim())?,
state: PhantomData,
})
}
pub(crate) fn parse_with_default_scheme(
value: &str,
default_scheme: &str,
) -> Result<Self, ApiUrlError> {
let value = value.trim();
if value.contains("://") {
return Self::parse(value);
}
Self::parse(&format!("{default_scheme}://{value}"))
}
pub(crate) fn with_scheme(mut self, scheme: &str) -> Result<Self, ApiUrlError> {
self.url
.set_scheme(scheme)
.map_err(|()| ApiUrlError::Scheme(scheme.to_string()))?;
Ok(self)
}
pub(crate) fn scheme(&self) -> &str {
self.url.scheme()
}
pub(crate) fn truncate_path_after(mut self, segment: &str) -> Result<Self, ApiUrlError> {
let segments: Vec<String> = self
.url
.path_segments()
.ok_or(ApiUrlError::CannotBeBase)?
.filter(|value| !value.is_empty())
.map(str::to_string)
.collect();
let Some(index) = segments.iter().position(|value| value == segment) else {
return Ok(self);
};
self.url
.path_segments_mut()
.map_err(|()| ApiUrlError::CannotBeBase)?
.clear()
.extend(segments[..=index].iter().map(String::as_str));
Ok(self)
}
pub(crate) fn complete_path(self, target: &[&str]) -> Result<ApiUrl<Complete>, ApiUrlError> {
self.complete_path_with_aliases(target, &[])
}
pub(crate) fn complete_path_with_aliases(
mut self,
target: &[&str],
aliases: &[&[&str]],
) -> Result<ApiUrl<Complete>, ApiUrlError> {
let existing: Vec<String> = self
.url
.path_segments()
.ok_or(ApiUrlError::CannotBeBase)?
.filter(|segment| !segment.is_empty())
.map(str::to_string)
.collect();
let ends_with = |suffix: &[&str]| {
existing.len() >= suffix.len()
&& existing[existing.len() - suffix.len()..]
.iter()
.map(String::as_str)
.eq(suffix.iter().copied())
};
let already_complete = aliases.iter().any(|suffix| ends_with(suffix));
let overlap = if already_complete {
target.len()
} else {
(0..=existing.len().min(target.len()))
.rev()
.find(|&length| {
existing[existing.len() - length..]
.iter()
.map(String::as_str)
.eq(target[..length].iter().copied())
})
.unwrap_or(0)
};
self.url
.path_segments_mut()
.map_err(|()| ApiUrlError::CannotBeBase)?
.pop_if_empty()
.extend(target[overlap..].iter().copied());
Ok(ApiUrl {
url: self.url,
state: PhantomData,
})
}
}
impl ApiUrl<Complete> {
pub(crate) fn has_query_key(&self, key: &str) -> bool {
self.url.query_pairs().any(|(name, _)| name == key)
}
pub(crate) fn append_query_pair(mut self, key: &str, value: &str) -> Self {
self.url.query_pairs_mut().append_pair(key, value);
self
}
pub(crate) fn append_query_pairs<'a>(
mut self,
pairs: impl IntoIterator<Item = (&'a str, &'a str)>,
) -> Self {
self.url.query_pairs_mut().extend_pairs(pairs);
self
}
pub(crate) fn into_string(self) -> String {
self.url.into()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn completion_appends_only_the_missing_path_suffix() {
for (base, expected) in [
("https://example.test", "https://example.test/v1/ocr"),
("https://example.test/v1", "https://example.test/v1/ocr"),
("https://example.test/v1/ocr", "https://example.test/v1/ocr"),
] {
let actual = ApiUrl::parse(base)
.and_then(|url| url.complete_path(&["v1", "ocr"]))
.map(|url| url.into_string())
.expect("url builds");
assert_eq!(actual, expected);
}
}
#[test]
fn completion_places_paths_before_queries_and_encodes_query_values() {
let actual = ApiUrl::parse("https://example.test/v1?tenant=a")
.and_then(|url| url.complete_path(&["v1", "ocr"]))
.map(|url| {
url.append_query_pair("model", "name with spaces")
.into_string()
})
.expect("url builds");
assert_eq!(
actual,
"https://example.test/v1/ocr?tenant=a&model=name+with+spaces"
);
}
}