mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
refactor(core): centralize HTTP request helpers
This commit is contained in:
parent
e11be9de21
commit
89080c7af8
7 changed files with 366 additions and 4 deletions
1
litellm-rust/Cargo.lock
generated
1
litellm-rust/Cargo.lock
generated
|
|
@ -1458,6 +1458,7 @@ dependencies = [
|
|||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"]
|
|||
tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] }
|
||||
futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] }
|
||||
base64 = "0.22"
|
||||
url = "2.5.8"
|
||||
|
||||
[profile.release]
|
||||
opt-level = 3
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ thiserror.workspace = true
|
|||
tracing.workspace = true
|
||||
tracing-subscriber = { workspace = true, optional = true }
|
||||
sha2.workspace = true
|
||||
url.workspace = true
|
||||
aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true }
|
||||
aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"], optional = true }
|
||||
aws-sdk-sts = { version = "1.108.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true }
|
||||
|
|
|
|||
|
|
@ -36,6 +36,34 @@ pub enum Error {
|
|||
Unsupported(&'static str),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, ThisError, PartialEq, Eq)]
|
||||
pub enum TransportError {
|
||||
#[error("upstream request failed with status {status}: {body}")]
|
||||
Http { status: u16, body: String },
|
||||
#[error("upstream network error: {0}")]
|
||||
Network(String),
|
||||
#[error("could not reach the provider: {0}")]
|
||||
Connect(String),
|
||||
}
|
||||
|
||||
impl TransportError {
|
||||
pub fn from_reqwest_before_dispatch(error: reqwest::Error) -> Self {
|
||||
let before_dispatch = !error.is_timeout() && (error.is_connect() || error.is_builder());
|
||||
let message = error.without_url().to_string();
|
||||
if before_dispatch {
|
||||
Self::Connect(message)
|
||||
} else {
|
||||
Self::Network(message)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<reqwest::Error> for TransportError {
|
||||
fn from(error: reqwest::Error) -> Self {
|
||||
Self::Network(error.without_url().to_string())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn json_type_name(value: &serde_json::Value) -> &'static str {
|
||||
match value {
|
||||
serde_json::Value::Null => "null",
|
||||
|
|
@ -46,3 +74,53 @@ pub fn json_type_name(value: &serde_json::Value) -> &'static str {
|
|||
serde_json::Value::Object(_) => "object",
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod transport_tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn transport_errors_remove_urls_and_keep_dispatch_context() {
|
||||
let error = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("client")
|
||||
.get("http://localhost:invalid/private?api_key=secret")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("invalid port");
|
||||
let error = TransportError::from_reqwest_before_dispatch(error);
|
||||
assert!(matches!(error, TransportError::Connect(_)));
|
||||
assert!(!error.to_string().contains("secret"));
|
||||
assert!(!error.to_string().contains("private"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn request_timeout_is_not_safe_to_retry_as_a_connect_failure() {
|
||||
use std::time::Duration;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind");
|
||||
let address = listener.local_addr().expect("address");
|
||||
let request = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("client")
|
||||
.get(format!("http://{address}"))
|
||||
.timeout(Duration::from_millis(200))
|
||||
.send();
|
||||
let (response, accepted) = tokio::join!(
|
||||
request,
|
||||
tokio::time::timeout(Duration::from_secs(2), listener.accept())
|
||||
);
|
||||
let _connection = accepted
|
||||
.expect("accept deadline")
|
||||
.expect("accepted connection");
|
||||
let error = response.expect_err("server does not respond");
|
||||
assert!(error.is_timeout());
|
||||
assert!(matches!(
|
||||
TransportError::from_reqwest_before_dispatch(error),
|
||||
TransportError::Network(_)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,43 @@
|
|||
//! Header and upstream-body helpers shared by every route module.
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::constants::UPSTREAM_ERROR_BODY_MAX_CHARS;
|
||||
use crate::error::{Error, json_type_name};
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "used by the OCR architecture in the next stacked PR"
|
||||
)]
|
||||
pub(crate) enum HeaderPolicy<'a> {
|
||||
All,
|
||||
Only(&'a [&'a str]),
|
||||
Except(&'a [&'a str]),
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "used by the OCR architecture in the next stacked PR"
|
||||
)]
|
||||
pub(crate) fn with_headers(
|
||||
builder: reqwest::RequestBuilder,
|
||||
headers: &[(String, String)],
|
||||
policy: HeaderPolicy<'_>,
|
||||
) -> reqwest::RequestBuilder {
|
||||
headers
|
||||
.iter()
|
||||
.filter(|(name, _)| match policy {
|
||||
HeaderPolicy::All => true,
|
||||
HeaderPolicy::Only(names) => names
|
||||
.iter()
|
||||
.any(|allowed| name.eq_ignore_ascii_case(allowed)),
|
||||
HeaderPolicy::Except(names) => !names
|
||||
.iter()
|
||||
.any(|excluded| name.eq_ignore_ascii_case(excluded)),
|
||||
})
|
||||
.fold(builder, |builder, (name, value)| {
|
||||
builder.header(name, value)
|
||||
})
|
||||
}
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
pub async fn http_request(
|
||||
request: reqwest::RequestBuilder,
|
||||
|
|
@ -12,8 +45,6 @@ pub async fn http_request(
|
|||
request.send().await
|
||||
}
|
||||
|
||||
/// Bound an upstream error body before it crosses a host boundary, so provider
|
||||
/// bodies stay data-minimized.
|
||||
pub fn truncate_error_body(body: &str) -> String {
|
||||
if body.chars().count() <= UPSTREAM_ERROR_BODY_MAX_CHARS {
|
||||
return body.to_string();
|
||||
|
|
@ -61,11 +92,77 @@ pub fn has_bearer_auth(headers: &[(String, String)]) -> bool {
|
|||
})
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "used by the OCR architecture in the next stacked PR"
|
||||
)]
|
||||
pub(crate) fn deserialize_optional_param<'de, D, T>(
|
||||
deserializer: D,
|
||||
) -> Result<Option<Option<T>>, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
T: serde::Deserialize<'de>,
|
||||
{
|
||||
<Option<T> as serde::Deserialize>::deserialize(deserializer).map(Some)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case(HeaderPolicy::All, true, true)]
|
||||
#[case(HeaderPolicy::Only(&["authorization"]), true, false)]
|
||||
#[case(HeaderPolicy::Except(&["authorization"]), false, true)]
|
||||
fn forwarding_policy_preserves_matching_headers_and_duplicates(
|
||||
#[case] policy: HeaderPolicy<'_>,
|
||||
#[case] auth: bool,
|
||||
#[case] trace: bool,
|
||||
) {
|
||||
let request = with_headers(
|
||||
reqwest::Client::new().get("https://example.com"),
|
||||
&[
|
||||
("AuThOrIzAtIoN".into(), "Bearer token".into()),
|
||||
("X-Trace".into(), "first".into()),
|
||||
("x-trace".into(), "second".into()),
|
||||
],
|
||||
policy,
|
||||
)
|
||||
.build()
|
||||
.unwrap();
|
||||
assert_eq!(request.headers().contains_key("authorization"), auth);
|
||||
let traces: Vec<_> = request.headers().get_all("x-trace").iter().collect();
|
||||
if trace {
|
||||
assert_eq!(traces, ["first", "second"]);
|
||||
} else {
|
||||
assert!(traces.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_policy_leaves_content_headers_to_reqwest() {
|
||||
let request = with_headers(
|
||||
reqwest::Client::new()
|
||||
.post("https://example.com")
|
||||
.multipart(reqwest::multipart::Form::new().text("file", "abc")),
|
||||
&[
|
||||
("Content-Type".into(), "application/json".into()),
|
||||
("CONTENT-LENGTH".into(), "0".into()),
|
||||
],
|
||||
HeaderPolicy::Except(&["content-type", "content-length"]),
|
||||
)
|
||||
.build()
|
||||
.unwrap();
|
||||
assert!(
|
||||
request.headers()["content-type"]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.starts_with("multipart/form-data; boundary=")
|
||||
);
|
||||
assert_ne!(request.headers()["content-length"], "0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncate_leaves_short_bodies_untouched() {
|
||||
assert_eq!(truncate_error_body("short"), "short");
|
||||
|
|
|
|||
|
|
@ -14,5 +14,6 @@ pub mod realtime;
|
|||
pub mod responses;
|
||||
pub mod router;
|
||||
pub mod routing_utils;
|
||||
mod url_utils;
|
||||
|
||||
pub use error::Error;
|
||||
|
|
|
|||
183
litellm-rust/crates/core/src/url_utils.rs
Normal file
183
litellm-rust/crates/core/src/url_utils.rs
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
#![allow(
|
||||
dead_code,
|
||||
reason = "used by the OCR architecture in the next stacked PR"
|
||||
)]
|
||||
|
||||
use std::marker::PhantomData;
|
||||
|
||||
use thiserror::Error;
|
||||
use url::Url;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub(crate) enum ApiUrlError {
|
||||
#[error("invalid URL: {0}")]
|
||||
Parse(#[from] url::ParseError),
|
||||
#[error("URL cannot be used as a base")]
|
||||
CannotBeBase,
|
||||
#[error("unsupported URL scheme: {0}")]
|
||||
Scheme(String),
|
||||
}
|
||||
|
||||
pub(crate) struct Base;
|
||||
pub(crate) struct Complete;
|
||||
|
||||
pub(crate) struct ApiUrl<State> {
|
||||
url: Url,
|
||||
state: PhantomData<State>,
|
||||
}
|
||||
|
||||
impl ApiUrl<Base> {
|
||||
pub(crate) fn parse(value: &str) -> Result<Self, ApiUrlError> {
|
||||
Ok(Self {
|
||||
url: Url::parse(value.trim())?,
|
||||
state: PhantomData,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn parse_with_default_scheme(
|
||||
value: &str,
|
||||
default_scheme: &str,
|
||||
) -> Result<Self, ApiUrlError> {
|
||||
let value = value.trim();
|
||||
if value.contains("://") {
|
||||
return Self::parse(value);
|
||||
}
|
||||
Self::parse(&format!("{default_scheme}://{value}"))
|
||||
}
|
||||
|
||||
pub(crate) fn with_scheme(mut self, scheme: &str) -> Result<Self, ApiUrlError> {
|
||||
self.url
|
||||
.set_scheme(scheme)
|
||||
.map_err(|()| ApiUrlError::Scheme(scheme.to_string()))?;
|
||||
Ok(self)
|
||||
}
|
||||
|
||||
pub(crate) fn scheme(&self) -> &str {
|
||||
self.url.scheme()
|
||||
}
|
||||
|
||||
pub(crate) fn truncate_path_after(mut self, segment: &str) -> Result<Self, ApiUrlError> {
|
||||
let segments: Vec<String> = self
|
||||
.url
|
||||
.path_segments()
|
||||
.ok_or(ApiUrlError::CannotBeBase)?
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_string)
|
||||
.collect();
|
||||
let Some(index) = segments.iter().position(|value| value == segment) else {
|
||||
return Ok(self);
|
||||
};
|
||||
self.url
|
||||
.path_segments_mut()
|
||||
.map_err(|()| ApiUrlError::CannotBeBase)?
|
||||
.clear()
|
||||
.extend(segments[..=index].iter().map(String::as_str));
|
||||
Ok(self)
|
||||
}
|
||||
|
||||
pub(crate) fn complete_path(self, target: &[&str]) -> Result<ApiUrl<Complete>, ApiUrlError> {
|
||||
self.complete_path_with_aliases(target, &[])
|
||||
}
|
||||
|
||||
pub(crate) fn complete_path_with_aliases(
|
||||
mut self,
|
||||
target: &[&str],
|
||||
aliases: &[&[&str]],
|
||||
) -> Result<ApiUrl<Complete>, ApiUrlError> {
|
||||
let existing: Vec<String> = self
|
||||
.url
|
||||
.path_segments()
|
||||
.ok_or(ApiUrlError::CannotBeBase)?
|
||||
.filter(|segment| !segment.is_empty())
|
||||
.map(str::to_string)
|
||||
.collect();
|
||||
let ends_with = |suffix: &[&str]| {
|
||||
existing.len() >= suffix.len()
|
||||
&& existing[existing.len() - suffix.len()..]
|
||||
.iter()
|
||||
.map(String::as_str)
|
||||
.eq(suffix.iter().copied())
|
||||
};
|
||||
let already_complete = aliases.iter().any(|suffix| ends_with(suffix));
|
||||
let overlap = if already_complete {
|
||||
target.len()
|
||||
} else {
|
||||
(0..=existing.len().min(target.len()))
|
||||
.rev()
|
||||
.find(|&length| {
|
||||
existing[existing.len() - length..]
|
||||
.iter()
|
||||
.map(String::as_str)
|
||||
.eq(target[..length].iter().copied())
|
||||
})
|
||||
.unwrap_or(0)
|
||||
};
|
||||
self.url
|
||||
.path_segments_mut()
|
||||
.map_err(|()| ApiUrlError::CannotBeBase)?
|
||||
.pop_if_empty()
|
||||
.extend(target[overlap..].iter().copied());
|
||||
Ok(ApiUrl {
|
||||
url: self.url,
|
||||
state: PhantomData,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl ApiUrl<Complete> {
|
||||
pub(crate) fn has_query_key(&self, key: &str) -> bool {
|
||||
self.url.query_pairs().any(|(name, _)| name == key)
|
||||
}
|
||||
|
||||
pub(crate) fn append_query_pair(mut self, key: &str, value: &str) -> Self {
|
||||
self.url.query_pairs_mut().append_pair(key, value);
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn append_query_pairs<'a>(
|
||||
mut self,
|
||||
pairs: impl IntoIterator<Item = (&'a str, &'a str)>,
|
||||
) -> Self {
|
||||
self.url.query_pairs_mut().extend_pairs(pairs);
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn into_string(self) -> String {
|
||||
self.url.into()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn completion_appends_only_the_missing_path_suffix() {
|
||||
for (base, expected) in [
|
||||
("https://example.test", "https://example.test/v1/ocr"),
|
||||
("https://example.test/v1", "https://example.test/v1/ocr"),
|
||||
("https://example.test/v1/ocr", "https://example.test/v1/ocr"),
|
||||
] {
|
||||
let actual = ApiUrl::parse(base)
|
||||
.and_then(|url| url.complete_path(&["v1", "ocr"]))
|
||||
.map(|url| url.into_string())
|
||||
.expect("url builds");
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completion_places_paths_before_queries_and_encodes_query_values() {
|
||||
let actual = ApiUrl::parse("https://example.test/v1?tenant=a")
|
||||
.and_then(|url| url.complete_path(&["v1", "ocr"]))
|
||||
.map(|url| {
|
||||
url.append_query_pair("model", "name with spaces")
|
||||
.into_string()
|
||||
})
|
||||
.expect("url builds");
|
||||
assert_eq!(
|
||||
actual,
|
||||
"https://example.test/v1/ocr?tenant=a&model=name+with+spaces"
|
||||
);
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue