mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-21 00:21:49 +00:00
Merge pull request #41969 from BerriAI/litellm_rust_settings_layers
refactor(rust): centralize layered settings resolution
This commit is contained in:
yujonglee
GitHub
commit
362be56bb0
72 changed files with 1416 additions and 515 deletions
5
litellm-rust/Cargo.lock
generated
5
litellm-rust/Cargo.lock
generated
|
|
@ -1976,6 +1976,7 @@ dependencies = [
|
|||
"azure_identity",
|
||||
"litellm-auth",
|
||||
"moka",
|
||||
"rstest",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"strum",
|
||||
|
|
@ -2137,11 +2138,14 @@ version = "0.1.0"
|
|||
dependencies = [
|
||||
"http 1.4.2",
|
||||
"hyper-util",
|
||||
"litellm-core-utils",
|
||||
"reqwest 0.12.28",
|
||||
"rstest",
|
||||
"rustls 0.23.42",
|
||||
"serde_json",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"veil",
|
||||
"webpki-roots",
|
||||
]
|
||||
|
||||
|
|
@ -2187,6 +2191,7 @@ dependencies = [
|
|||
"litellm-auth-gcp",
|
||||
"litellm-callbacks-legacy",
|
||||
"litellm-core",
|
||||
"litellm-core-utils",
|
||||
"litellm-host-python",
|
||||
"litellm-http",
|
||||
"litellm-llms",
|
||||
|
|
|
|||
|
|
@ -18,4 +18,5 @@ azure_core = "1.0.0"
|
|||
azure_identity = { version = "1.0.0", features = ["tokio"] }
|
||||
|
||||
[dev-dependencies]
|
||||
rstest.workspace = true
|
||||
tokio.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,11 +1,10 @@
|
|||
use serde_json::{Map, Value};
|
||||
use std::collections::BTreeMap;
|
||||
use strum::EnumString;
|
||||
|
||||
use litellm_auth::Error;
|
||||
use litellm_auth::{
|
||||
CredentialResolverHandle, InputSource, SecretValue, Sourced, TokenProviderHandle,
|
||||
CredentialResolverHandle, Error, InputSource, SecretValue, Sourced, TokenProviderHandle,
|
||||
};
|
||||
use serde_json::{Map, Value};
|
||||
use strum::EnumString;
|
||||
|
||||
pub const DEFAULT_AZURE_SCOPE: &str = "https://cognitiveservices.azure.com/.default";
|
||||
|
||||
|
|
@ -52,6 +51,16 @@ pub struct AzureAuthInputs {
|
|||
}
|
||||
|
||||
impl AzureAuthInputs {
|
||||
pub fn or_configured_token_refresh(self, enabled: bool) -> Self {
|
||||
if *self.enable_azure_ad_token_refresh.value() || !enabled {
|
||||
return self;
|
||||
}
|
||||
Self {
|
||||
enable_azure_ad_token_refresh: Sourced::new(true, InputSource::Deployment),
|
||||
..self
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn from_optional_params(params: &Map<String, Value>) -> Result<Self, Error> {
|
||||
Self::from_sourced_optional_params(params, &BTreeMap::new())
|
||||
|
|
@ -115,12 +124,12 @@ fn source_for(sources: &BTreeMap<String, InputSource>, name: &str) -> InputSourc
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::json;
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use super::{AzureAuthInputs, AzureCredentialType, ConfigValue};
|
||||
use litellm_auth::{InputSource, Sourced};
|
||||
use serde_json::json;
|
||||
|
||||
use super::{AzureAuthInputs, AzureCredentialType, ConfigValue};
|
||||
|
||||
#[test]
|
||||
fn selector_parsing_is_exact() {
|
||||
|
|
@ -189,4 +198,29 @@ mod tests {
|
|||
assert!(!debug.contains("token-value"));
|
||||
assert!(!debug.contains("secret-value"));
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::global_turns_refresh_on(json!({}), true, true, InputSource::Deployment)]
|
||||
#[case::global_overrides_a_call_false_like_python(json!({"enable_azure_ad_token_refresh": false}), true, true, InputSource::Deployment)]
|
||||
#[case::call_true_survives_a_global_false(json!({"enable_azure_ad_token_refresh": true}), false, true, InputSource::Request)]
|
||||
#[case::both_off(json!({}), false, false, InputSource::Request)]
|
||||
fn token_refresh_follows_the_configured_global(
|
||||
#[case] params: serde_json::Value,
|
||||
#[case] global: bool,
|
||||
#[case] enabled: bool,
|
||||
#[case] source: InputSource,
|
||||
) {
|
||||
let sources = BTreeMap::from([(
|
||||
"enable_azure_ad_token_refresh".to_string(),
|
||||
InputSource::Request,
|
||||
)]);
|
||||
let inputs =
|
||||
AzureAuthInputs::from_sourced_optional_params(params.as_object().unwrap(), &sources)
|
||||
.unwrap()
|
||||
.or_configured_token_refresh(global);
|
||||
assert_eq!(
|
||||
inputs.enable_azure_ad_token_refresh,
|
||||
Sourced::new(enabled, source)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,17 +1,13 @@
|
|||
use std::collections::BTreeMap;
|
||||
use std::future::Future;
|
||||
use std::path::Path;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::{collections::BTreeMap, future::Future, path::Path, pin::Pin, sync::Arc};
|
||||
|
||||
use gcp_auth::{CustomServiceAccount, TokenProvider};
|
||||
use litellm_auth::{
|
||||
CredentialPlacement, Error, InputSource, SecretValue, Sourced, http::apply_credential,
|
||||
};
|
||||
use moka::future::Cache;
|
||||
use serde_json::{Map, Value};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use litellm_auth::http::apply_credential;
|
||||
use litellm_auth::{CredentialPlacement, Error, InputSource, SecretValue, Sourced};
|
||||
|
||||
const CLOUD_PLATFORM_SCOPE: &str = "https://www.googleapis.com/auth/cloud-platform";
|
||||
const GOOGLE_OAUTH_TOKEN_ENDPOINT: &str = "https://oauth2.googleapis.com/token";
|
||||
const GOOGLE_APPLICATION_CREDENTIALS_ENV: &str = "GOOGLE_APPLICATION_CREDENTIALS";
|
||||
|
|
@ -45,6 +41,16 @@ impl VertexConfig {
|
|||
})
|
||||
}
|
||||
|
||||
pub fn or_configured(self, project_id: Option<&str>, location: Option<&str>) -> Self {
|
||||
let configured =
|
||||
|value: Option<&str>| value.filter(|value| !value.is_empty()).map(str::to_string);
|
||||
Self {
|
||||
project_id: self.project_id.or_else(|| configured(project_id)),
|
||||
location: self.location.or_else(|| configured(location)),
|
||||
..self
|
||||
}
|
||||
}
|
||||
|
||||
pub fn project_id(&self) -> Option<&str> {
|
||||
self.project_id.as_deref()
|
||||
}
|
||||
|
|
@ -571,4 +577,29 @@ mod tests {
|
|||
assert_eq!(loads.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn configured_defaults_sit_between_call_params_and_the_environment() {
|
||||
let env = |name: &str| Some(format!("env-{name}"));
|
||||
let from_config =
|
||||
VertexConfig::default().or_configured(Some("global-project"), Some("global-location"));
|
||||
assert_eq!(
|
||||
get_vertex_ai_project(&from_config, &env).as_deref(),
|
||||
Some("global-project")
|
||||
);
|
||||
assert_eq!(
|
||||
get_vertex_ai_location(&from_config, &env).as_deref(),
|
||||
Some("global-location")
|
||||
);
|
||||
let from_call =
|
||||
config(json!({"vertex_project":"call-project","vertex_location":"call-location"}))
|
||||
.or_configured(Some("global-project"), Some("global-location"));
|
||||
assert_eq!(from_call.project_id(), Some("call-project"));
|
||||
assert_eq!(from_call.location(), Some("call-location"));
|
||||
let empty_global = VertexConfig::default().or_configured(Some(""), None);
|
||||
assert_eq!(
|
||||
get_vertex_ai_project(&empty_global, &env).as_deref(),
|
||||
Some("env-VERTEXAI_PROJECT")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,4 +6,5 @@ pub mod params;
|
|||
pub mod prompt_templates;
|
||||
pub mod secret_redaction;
|
||||
pub mod serde_compat;
|
||||
pub mod settings;
|
||||
pub mod url_utils;
|
||||
|
|
|
|||
144
litellm-rust/crates/core-utils/src/settings.rs
Normal file
144
litellm-rust/crates/core-utils/src/settings.rs
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
use std::str::FromStr;
|
||||
|
||||
pub trait Lookup {
|
||||
fn get(&self, name: &str) -> Option<String>;
|
||||
|
||||
fn truthy(&self, name: &str) -> Option<String> {
|
||||
self.get(name).filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
fn enabled(&self, name: &str) -> Option<bool> {
|
||||
self.get(name)
|
||||
.is_some_and(|value| value.trim().eq_ignore_ascii_case("true"))
|
||||
.then_some(true)
|
||||
}
|
||||
|
||||
fn parsed<T: FromStr>(&self, name: &str) -> Option<T>
|
||||
where
|
||||
Self: Sized,
|
||||
{
|
||||
self.get(name).and_then(|value| value.trim().parse().ok())
|
||||
}
|
||||
}
|
||||
|
||||
impl<F: Fn(&str) -> Option<String>> Lookup for F {
|
||||
fn get(&self, name: &str) -> Option<String> {
|
||||
self(name)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ProcessEnvironment;
|
||||
|
||||
impl Lookup for ProcessEnvironment {
|
||||
fn get(&self, name: &str) -> Option<String> {
|
||||
std::env::var(name).ok()
|
||||
}
|
||||
}
|
||||
|
||||
pub trait Layer: Default {
|
||||
fn or(self, lower: Self) -> Self;
|
||||
}
|
||||
|
||||
pub fn merge<L: Layer>(highest_precedence_first: impl IntoIterator<Item = L>) -> L {
|
||||
highest_precedence_first
|
||||
.into_iter()
|
||||
.reduce(L::or)
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn env_of(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
|
||||
move |name| {
|
||||
values
|
||||
.iter()
|
||||
.find(|(key, _)| *key == name)
|
||||
.map(|(_, value)| value.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_keeps_a_present_empty_value_like_os_getenv_with_a_fallback() {
|
||||
let env = env_of(&[("EMPTY", "")]);
|
||||
assert_eq!(env.get("EMPTY"), Some(String::new()));
|
||||
assert_eq!(env.get("ABSENT"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truthy_drops_an_empty_value_like_a_python_or_chain() {
|
||||
let env = env_of(&[("EMPTY", ""), ("SET", "value")]);
|
||||
assert_eq!(env.truthy("EMPTY"), None);
|
||||
assert_eq!(env.truthy("SET").as_deref(), Some("value"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enabled_only_switches_on_for_true_and_never_forces_off() {
|
||||
let env = env_of(&[
|
||||
("LOWER", "true"),
|
||||
("PADDED", " True "),
|
||||
("OFF", "false"),
|
||||
("ONE", "1"),
|
||||
]);
|
||||
assert_eq!(env.enabled("LOWER"), Some(true));
|
||||
assert_eq!(env.enabled("PADDED"), Some(true));
|
||||
assert_eq!(env.enabled("OFF"), None);
|
||||
assert_eq!(env.enabled("ONE"), None);
|
||||
assert_eq!(env.enabled("ABSENT"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parsed_trims_and_skips_values_that_do_not_parse() {
|
||||
let env = env_of(&[("PADDED", " 45 "), ("WORD", "soon"), ("FRACTION", "0.5")]);
|
||||
assert_eq!(env.parsed::<u32>("PADDED"), Some(45));
|
||||
assert_eq!(env.parsed::<u32>("WORD"), None);
|
||||
assert_eq!(env.parsed::<f64>("FRACTION"), Some(0.5));
|
||||
assert_eq!(env.parsed::<u32>("ABSENT"), None);
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, PartialEq)]
|
||||
struct Pair {
|
||||
first: Option<u8>,
|
||||
second: Option<u8>,
|
||||
}
|
||||
|
||||
impl Layer for Pair {
|
||||
fn or(self, lower: Self) -> Self {
|
||||
Self {
|
||||
first: self.first.or(lower.first),
|
||||
second: self.second.or(lower.second),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_takes_each_field_from_the_highest_layer_that_sets_it() {
|
||||
let merged = merge([
|
||||
Pair {
|
||||
first: Some(1),
|
||||
second: None,
|
||||
},
|
||||
Pair {
|
||||
first: Some(2),
|
||||
second: Some(2),
|
||||
},
|
||||
Pair {
|
||||
first: Some(3),
|
||||
second: Some(3),
|
||||
},
|
||||
]);
|
||||
assert_eq!(
|
||||
merged,
|
||||
Pair {
|
||||
first: Some(1),
|
||||
second: Some(2),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merging_no_layers_yields_the_empty_layer() {
|
||||
assert_eq!(merge(Vec::<Pair>::new()), Pair::default());
|
||||
}
|
||||
}
|
||||
|
|
@ -5,10 +5,11 @@ litellm-core is the LiteLLM SDK in Rust — it makes the LLM call. Each top-leve
|
|||
Each crate mirrors one top-level Python package, so a Rust path reads as its Python path with the crate name in place of the package directory. Dependencies only point down:
|
||||
|
||||
- `litellm-types` mirrors `litellm/types/`: pure serde data, no I/O
|
||||
- `litellm-core-utils` mirrors `litellm/litellm_core_utils/`: pure helpers (provider resolution, prompt factory, call arguments), no network I/O
|
||||
- `litellm-llms` mirrors `litellm/llms/`: `base_llm/<api>/transformation.rs`, `<provider>/<api>/transformation.rs`, and `custom_httpx/` (HTTP helpers and the OCR request handler)
|
||||
- `litellm-core-utils` mirrors `litellm/litellm_core_utils/`: pure helpers (provider resolution, prompt factory, call arguments, settings lookup and layer merge), no network I/O
|
||||
- `litellm-http` is Rust-only and route-neutral: settings resolution, the pooled `reqwest` clients, TLS, proxies, the SSRF-safe media fetcher, request and header helpers, and transport errors. Python's `litellm/llms/custom_httpx/` is split by responsibility instead of mirrored: its transport half lives here, its OCR handler in `litellm-llms`
|
||||
- `litellm-llms` mirrors `litellm/llms/`: `base_llm/<api>/transformation.rs`, `<provider>/<api>/transformation.rs`, and `base_llm/ocr/handler.rs` (the OCR request handler)
|
||||
- `litellm-core` mirrors the route packages (`litellm/ocr/`, `litellm/messages/`, ...): entrypoints, route request types, provider dispatch, the route machine, and hooks
|
||||
|
||||
A route module owns the call entrypoint, route request types (`*Request<'a>`), credential fallback, provider dispatch, and the handler glue that runs a provider config. Provider code never imports from core; when it needs the caller's hooks mid-call it goes through `litellm_llms::custom_httpx::llm_http_handler::CallHooks`, which each route implements over its host. Import every item from its canonical path. Never re-export another crate's items or give an item a second public path; the only re-export allowed is a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`). Handlers belong in core or llms, never in a host crate
|
||||
A route module owns the call entrypoint, route request types (`*Request<'a>`), credential fallback, provider dispatch, and the handler glue that runs a provider config. Provider code never imports from core; when it needs the caller's hooks mid-call it goes through `litellm_llms::base_llm::ocr::handler::CallHooks`, which each route implements over its host. Import every item from its canonical path. Never re-export another crate's items or give an item a second public path; the only re-export allowed is a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`). Handlers belong in core or llms, never in a host crate
|
||||
|
||||
Not here: serving HTTP (axum routes, extractors), config file reading, rollout state, databases, or callback execution of any kind. Core runs each route as a machine that yields host operations and call events; which integrations consume those events is the host's business.
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ futures-util.workspace = true
|
|||
base64.workspace = true
|
||||
litellm-auth.workspace = true
|
||||
litellm-auth-aws.workspace = true
|
||||
litellm-http.workspace = true
|
||||
litellm-llms.workspace = true
|
||||
moka.workspace = true
|
||||
mime_guess = "2.0.5"
|
||||
|
|
@ -36,7 +37,6 @@ veil.workspace = true
|
|||
|
||||
[dev-dependencies]
|
||||
litellm-auth-gcp.workspace = true
|
||||
litellm-http.workspace = true
|
||||
litellm-llms = { workspace = true, features = ["test-support"] }
|
||||
rstest.workspace = true
|
||||
rstest_reuse.workspace = true
|
||||
|
|
|
|||
|
|
@ -20,9 +20,9 @@ pub enum Error {
|
|||
#[error(transparent)]
|
||||
Auth(#[from] litellm_auth::Error),
|
||||
#[error(transparent)]
|
||||
Transport(#[from] litellm_llms::custom_httpx::transport::Error),
|
||||
Transport(#[from] litellm_http::transport::Error),
|
||||
#[error(transparent)]
|
||||
Headers(#[from] litellm_llms::custom_httpx::http_handler::HeaderError),
|
||||
Headers(#[from] litellm_http::request::HeaderError),
|
||||
#[error(transparent)]
|
||||
Aws(#[from] litellm_auth_aws::Error),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use litellm_llms::custom_httpx::http_handler::{http_request, truncate_error_body};
|
||||
use litellm_http::request::{http_request, truncate_error_body};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::{Error, client::http_client};
|
||||
|
|
@ -18,23 +18,17 @@ pub async fn execute_audio_transcription_provider_call(
|
|||
request_builder = request_builder.timeout(duration);
|
||||
}
|
||||
let response = http_request(request_builder).await.map_err(|error| {
|
||||
Error::Transport(litellm_llms::custom_httpx::transport::Error::Network(
|
||||
error.to_string(),
|
||||
))
|
||||
Error::Transport(litellm_http::transport::Error::Network(error.to_string()))
|
||||
})?;
|
||||
let status = response.status();
|
||||
let text = response.text().await.map_err(|error| {
|
||||
Error::Transport(litellm_llms::custom_httpx::transport::Error::Network(
|
||||
error.to_string(),
|
||||
))
|
||||
Error::Transport(litellm_http::transport::Error::Network(error.to_string()))
|
||||
})?;
|
||||
if !status.is_success() {
|
||||
return Err(Error::Transport(
|
||||
litellm_llms::custom_httpx::transport::Error::Http {
|
||||
status: status.as_u16(),
|
||||
body: truncate_error_body(&text),
|
||||
},
|
||||
));
|
||||
return Err(Error::Transport(litellm_http::transport::Error::Http {
|
||||
status: status.as_u16(),
|
||||
body: truncate_error_body(&text),
|
||||
}));
|
||||
}
|
||||
let response_json = serde_json::from_str(&text)
|
||||
.map_err(|error| Error::InvalidResponse(format!("invalid audio response JSON: {error}")))?;
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider};
|
||||
use litellm_http::request::{has_header, string_headers};
|
||||
use litellm_llms::{
|
||||
base_llm::audio_transcription::transformation::{
|
||||
AudioTranscriptionAuth, BaseAudioTranscriptionConfig,
|
||||
},
|
||||
bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG,
|
||||
custom_httpx::http_handler::{has_header, string_headers},
|
||||
};
|
||||
|
||||
use super::Error;
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
use litellm_http::request::string_headers as shared_string_headers;
|
||||
use litellm_llms::{
|
||||
anthropic::chat::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG,
|
||||
base_llm::chat::transformation::BaseConfig,
|
||||
bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG,
|
||||
custom_httpx::http_handler::string_headers as shared_string_headers,
|
||||
};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
|
|
|
|||
|
|
@ -20,9 +20,9 @@ pub enum Error {
|
|||
#[error(transparent)]
|
||||
Auth(#[from] litellm_auth::Error),
|
||||
#[error(transparent)]
|
||||
Transport(#[from] litellm_llms::custom_httpx::transport::Error),
|
||||
Transport(#[from] litellm_http::transport::Error),
|
||||
#[error(transparent)]
|
||||
Headers(#[from] litellm_llms::custom_httpx::http_handler::HeaderError),
|
||||
Headers(#[from] litellm_http::request::HeaderError),
|
||||
#[error(transparent)]
|
||||
Aws(#[from] litellm_auth_aws::Error),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
use litellm_llms::{
|
||||
base_llm::chat::transformation::{ChatCompletionsAuth, ProviderChatResponseData},
|
||||
custom_httpx::http_handler::{http_request, truncate_error_body},
|
||||
};
|
||||
use litellm_http::request::{http_request, truncate_error_body};
|
||||
use litellm_llms::base_llm::chat::transformation::{ChatCompletionsAuth, ProviderChatResponseData};
|
||||
use litellm_types::utils::ChatCompletionsResponse;
|
||||
use serde_json::Value;
|
||||
|
||||
|
|
@ -34,30 +32,22 @@ pub(super) async fn execute_chat_completions_provider_call(
|
|||
// so the host can still serve it. Everything else here, a timeout
|
||||
// above all, may have reached the provider and been answered.
|
||||
if err.is_connect() || err.is_builder() {
|
||||
Error::Transport(litellm_llms::custom_httpx::transport::Error::Connect(
|
||||
err.to_string(),
|
||||
))
|
||||
Error::Transport(litellm_http::transport::Error::Connect(err.to_string()))
|
||||
} else {
|
||||
Error::Transport(litellm_llms::custom_httpx::transport::Error::Network(
|
||||
err.to_string(),
|
||||
))
|
||||
Error::Transport(litellm_http::transport::Error::Network(err.to_string()))
|
||||
}
|
||||
})?;
|
||||
|
||||
let status = response.status();
|
||||
let text = response.text().await.map_err(|err| {
|
||||
Error::Transport(litellm_llms::custom_httpx::transport::Error::Network(
|
||||
err.to_string(),
|
||||
))
|
||||
Error::Transport(litellm_http::transport::Error::Network(err.to_string()))
|
||||
})?;
|
||||
|
||||
if !status.is_success() {
|
||||
return Err(Error::Transport(
|
||||
litellm_llms::custom_httpx::transport::Error::Http {
|
||||
status: status.as_u16(),
|
||||
body: truncate_error_body(&text),
|
||||
},
|
||||
));
|
||||
return Err(Error::Transport(litellm_http::transport::Error::Http {
|
||||
status: status.as_u16(),
|
||||
body: truncate_error_body(&text),
|
||||
}));
|
||||
}
|
||||
|
||||
let body: Value = serde_json::from_str(&text).map_err(|err| {
|
||||
|
|
@ -82,9 +72,7 @@ pub(super) async fn execute_chat_completions_provider_call(
|
|||
pub(super) fn as_response_error(err: Error) -> Error {
|
||||
match err {
|
||||
already @ (Error::InvalidResponse(_)
|
||||
| Error::Transport(litellm_llms::custom_httpx::transport::Error::Http {
|
||||
..
|
||||
})) => already,
|
||||
| Error::Transport(litellm_http::transport::Error::Http { .. })) => already,
|
||||
other => Error::InvalidResponse(other.to_string()),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider};
|
||||
use litellm_llms::{
|
||||
base_llm::chat::transformation::{BaseConfig, ChatCompletionsAuth},
|
||||
custom_httpx::http_handler::has_header,
|
||||
};
|
||||
use litellm_http::request::has_header;
|
||||
use litellm_llms::base_llm::chat::transformation::{BaseConfig, ChatCompletionsAuth};
|
||||
use litellm_types::llms::openai::ChatMessage;
|
||||
use serde_json::Value;
|
||||
|
||||
|
|
|
|||
|
|
@ -265,7 +265,7 @@ fn rejects_non_string_extra_headers() {
|
|||
call.extra_headers = Some(Map::from_iter([("x-trace".to_string(), json!(7))]));
|
||||
assert_eq!(
|
||||
decline(call),
|
||||
Error::Headers(litellm_llms::custom_httpx::http_handler::HeaderError {
|
||||
Error::Headers(litellm_http::request::HeaderError {
|
||||
context: "chat completions",
|
||||
name: "x-trace".to_string(),
|
||||
actual: "number",
|
||||
|
|
@ -771,10 +771,7 @@ mod round_trip {
|
|||
assert!(
|
||||
matches!(
|
||||
err,
|
||||
Error::Transport(litellm_llms::custom_httpx::transport::Error::Http {
|
||||
status: 429,
|
||||
..
|
||||
})
|
||||
Error::Transport(litellm_http::transport::Error::Http { status: 429, .. })
|
||||
),
|
||||
"expected a 429, got {err:?}"
|
||||
);
|
||||
|
|
@ -801,7 +798,7 @@ mod round_trip {
|
|||
assert!(
|
||||
matches!(
|
||||
err,
|
||||
Error::Transport(litellm_llms::custom_httpx::transport::Error::Connect(_))
|
||||
Error::Transport(litellm_http::transport::Error::Connect(_))
|
||||
),
|
||||
"expected a pre-send connect failure, got {err:?}"
|
||||
);
|
||||
|
|
@ -825,16 +822,11 @@ mod round_trip {
|
|||
}
|
||||
// An upstream status is already unambiguous, so it survives intact.
|
||||
assert!(matches!(
|
||||
as_response_error(Error::Transport(
|
||||
litellm_llms::custom_httpx::transport::Error::Http {
|
||||
status: 500,
|
||||
body: "boom".to_string()
|
||||
}
|
||||
)),
|
||||
Error::Transport(litellm_llms::custom_httpx::transport::Error::Http {
|
||||
as_response_error(Error::Transport(litellm_http::transport::Error::Http {
|
||||
status: 500,
|
||||
..
|
||||
})
|
||||
body: "boom".to_string()
|
||||
})),
|
||||
Error::Transport(litellm_http::transport::Error::Http { status: 500, .. })
|
||||
));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,9 @@
|
|||
pub(super) use litellm_llms::custom_httpx::http_handler::{
|
||||
has_bearer_auth, has_header, truncate_error_body,
|
||||
};
|
||||
use litellm_http::request::string_headers as shared_string_headers;
|
||||
pub(super) use litellm_http::request::{has_bearer_auth, has_header, truncate_error_body};
|
||||
use litellm_llms::{
|
||||
anthropic::experimental_pass_through::messages::transformation::ANTHROPIC_MESSAGES_CONFIG,
|
||||
azure_ai::anthropic::messages_transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG,
|
||||
base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig,
|
||||
custom_httpx::http_handler::string_headers as shared_string_headers,
|
||||
};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
|
|
|
|||
|
|
@ -15,9 +15,9 @@ pub enum Error {
|
|||
#[error(transparent)]
|
||||
Auth(#[from] litellm_auth::Error),
|
||||
#[error(transparent)]
|
||||
Transport(#[from] litellm_llms::custom_httpx::transport::Error),
|
||||
Transport(#[from] litellm_http::transport::Error),
|
||||
#[error(transparent)]
|
||||
Headers(#[from] litellm_llms::custom_httpx::http_handler::HeaderError),
|
||||
Headers(#[from] litellm_http::request::HeaderError),
|
||||
}
|
||||
|
||||
impl From<LlmError> for Error {
|
||||
|
|
|
|||
|
|
@ -1,9 +1,7 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use litellm_llms::{
|
||||
base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig,
|
||||
custom_httpx::{http_handler::http_request, transport::Error as TransportError},
|
||||
};
|
||||
use litellm_http::{request::http_request, transport::Error as TransportError};
|
||||
use litellm_llms::base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig;
|
||||
use litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse;
|
||||
use serde_json::Value;
|
||||
|
||||
|
|
|
|||
|
|
@ -82,7 +82,7 @@ fn string_headers_rejects_non_string_values() {
|
|||
let err = string_headers(Some(headers)).expect_err("non-string header rejected");
|
||||
assert_eq!(
|
||||
err,
|
||||
Error::Headers(litellm_llms::custom_httpx::http_handler::HeaderError {
|
||||
Error::Headers(litellm_http::request::HeaderError {
|
||||
context: "messages",
|
||||
name: "x-count".to_string(),
|
||||
actual: "number",
|
||||
|
|
@ -432,7 +432,7 @@ async fn messages_maps_provider_error_status_to_http_error() {
|
|||
|
||||
assert!(matches!(
|
||||
err,
|
||||
Error::Transport(litellm_llms::custom_httpx::transport::Error::Http { status: 401, .. })
|
||||
Error::Transport(litellm_http::transport::Error::Http { status: 401, .. })
|
||||
));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
use litellm_llms::{
|
||||
base_llm::ocr::{error::Error, transformation::LiteLLMOcrResponse},
|
||||
custom_httpx::llm_http_handler::OcrClient,
|
||||
use litellm_llms::base_llm::ocr::{
|
||||
error::Error, handler::OcrClient, transformation::LiteLLMOcrResponse,
|
||||
};
|
||||
|
||||
use crate::ocr::{
|
||||
|
|
|
|||
|
|
@ -1,12 +1,10 @@
|
|||
use futures_util::future::BoxFuture;
|
||||
use litellm_auth::SecretValue;
|
||||
use litellm_host::event::{MachineEvent, RawResponse, RequestContext, WireRequest};
|
||||
use litellm_llms::{
|
||||
base_llm::ocr::{
|
||||
error::Error,
|
||||
transformation::{LiteLLMOcrResponse, PreparedOcrRequest},
|
||||
},
|
||||
custom_httpx::llm_http_handler::{CallHooks, OcrClient},
|
||||
use litellm_llms::base_llm::ocr::{
|
||||
error::Error,
|
||||
handler::{CallHooks, OcrClient},
|
||||
transformation::{LiteLLMOcrResponse, PreparedOcrRequest},
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
|
|
@ -24,7 +22,7 @@ pub(crate) async fn perform_ocr_request(
|
|||
) -> Result<LiteLLMOcrResponse, Error> {
|
||||
request.response_format()?;
|
||||
let config = request.config;
|
||||
let request = prepare_request(request, caller_document);
|
||||
let request = prepare_request(request, caller_document, client);
|
||||
let hooks = OcrCallHooks::new(host.clone(), &request, config);
|
||||
config.ocr(client, &request, &hooks).await
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
use litellm_auth::{InputSource, SecretValue, Sourced};
|
||||
use litellm_llms::base_llm::ocr::transformation::{
|
||||
OcrConnection, OcrCredentialInputs, PreparedOcrRequest, credential_env,
|
||||
use litellm_llms::base_llm::ocr::{
|
||||
handler::OcrClient,
|
||||
transformation::{OcrConnection, OcrCredentialInputs, PreparedOcrRequest},
|
||||
};
|
||||
|
||||
use super::provider_config::OcrProvider;
|
||||
|
|
@ -9,26 +10,31 @@ use crate::ocr::types::{LiteLLMOcrRequest, ResolvedOcrRequest};
|
|||
pub(crate) fn prepare_request(
|
||||
request: ResolvedOcrRequest,
|
||||
caller_document: bool,
|
||||
client: &OcrClient,
|
||||
) -> PreparedOcrRequest {
|
||||
let credentials = request.credentials.clone();
|
||||
let api_base_env = match request.config.provider() {
|
||||
OcrProvider::Mistral => Some("MISTRAL_API_BASE"),
|
||||
OcrProvider::AzureAi => Some("AZURE_AI_API_BASE"),
|
||||
OcrProvider::Cohere | OcrProvider::Reducto | OcrProvider::VertexAi => None,
|
||||
let (preferred_api_key_env, api_base_env) = match request.config.provider() {
|
||||
OcrProvider::Mistral => (
|
||||
Some("MISTRAL_AZURE_API_KEY"),
|
||||
Some("MISTRAL_AZURE_API_BASE"),
|
||||
),
|
||||
OcrProvider::AzureAi => (None, Some("AZURE_AI_API_BASE")),
|
||||
OcrProvider::Cohere | OcrProvider::Reducto | OcrProvider::VertexAi => (None, None),
|
||||
};
|
||||
let secret = |name: &str| client.secrets().truthy(name);
|
||||
let dynamic_api_key = credentials.dynamic_api_key.or_else(|| {
|
||||
credentials.api_key.clone().or_else(|| {
|
||||
request
|
||||
.config
|
||||
.get_api_key_env_var()
|
||||
.and_then(credential_env)
|
||||
preferred_api_key_env
|
||||
.into_iter()
|
||||
.chain(request.config.get_api_key_env_var())
|
||||
.find_map(secret)
|
||||
.map(|value| Sourced::new(SecretValue::new(value), InputSource::Environment))
|
||||
})
|
||||
});
|
||||
let dynamic_api_base = credentials.dynamic_api_base.or_else(|| {
|
||||
credentials.api_base.clone().or_else(|| {
|
||||
api_base_env
|
||||
.and_then(credential_env)
|
||||
.and_then(secret)
|
||||
.map(|value| Sourced::new(value, InputSource::Environment))
|
||||
})
|
||||
});
|
||||
|
|
@ -51,7 +57,12 @@ pub(crate) fn prepare_request(
|
|||
PreparedOcrRequest {
|
||||
model,
|
||||
document,
|
||||
connection: OcrConnection::new(resolved, transport),
|
||||
connection: OcrConnection::new(
|
||||
resolved,
|
||||
transport,
|
||||
client.settings().clone(),
|
||||
client.secrets().clone(),
|
||||
),
|
||||
caller_document,
|
||||
optional_params,
|
||||
input_sources,
|
||||
|
|
@ -61,7 +72,11 @@ pub(crate) fn prepare_request(
|
|||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn prepare_request_for_test(request: ResolvedOcrRequest) -> PreparedOcrRequest {
|
||||
prepare_request(request, true)
|
||||
prepare_request(
|
||||
request,
|
||||
true,
|
||||
&OcrClient::for_test(reqwest::Client::new(), reqwest::Client::new()),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
|
|
@ -7,13 +7,13 @@ use litellm_llms::{
|
|||
},
|
||||
base_llm::ocr::{
|
||||
error::Error,
|
||||
handler::{self, CallHooks, OcrClient},
|
||||
transformation::{
|
||||
BaseOcrConfig, LiteLLMOcrResponse, OcrCredentialInputs, OcrDocument,
|
||||
PreparedOcrRequest, ResolvedOcrCredentials,
|
||||
},
|
||||
},
|
||||
cohere::ocr::transformation::CohereParseConfig,
|
||||
custom_httpx::llm_http_handler::{self, CallHooks, OcrClient},
|
||||
mistral::ocr::transformation::MistralOcrConfig,
|
||||
reducto::ocr::transformation::{ReductoParseLegacyConfig, ReductoParseV3Config},
|
||||
vertex_ai::ocr::{
|
||||
|
|
@ -116,7 +116,7 @@ impl OcrConfigKind {
|
|||
request: &PreparedOcrRequest,
|
||||
hooks: &dyn CallHooks<Error>,
|
||||
) -> Result<LiteLLMOcrResponse, Error> {
|
||||
with_config!(self, config => llm_http_handler::ocr(&config, client, request, hooks).await)
|
||||
with_config!(self, config => handler::ocr(&config, client, request, hooks).await)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,9 +6,8 @@ use litellm_host::{
|
|||
machine::{HostChannel, HostTokenProvider, MachineFault, RouteMachine, TokenRoute},
|
||||
route::Route,
|
||||
};
|
||||
use litellm_llms::{
|
||||
base_llm::ocr::{error::Error, transformation::LiteLLMOcrResponse},
|
||||
custom_httpx::llm_http_handler::OcrClient,
|
||||
use litellm_llms::base_llm::ocr::{
|
||||
error::Error, handler::OcrClient, transformation::LiteLLMOcrResponse,
|
||||
};
|
||||
|
||||
use super::handler::perform_ocr_request;
|
||||
|
|
|
|||
|
|
@ -277,7 +277,7 @@ mod tests {
|
|||
vec![("x-a".to_string(), "1".to_string())]
|
||||
);
|
||||
assert_eq!(request.transport.extra_headers_source, InputSource::Request);
|
||||
assert_eq!(request.transport.timeout, Duration::from_secs(7));
|
||||
assert_eq!(request.transport.timeout, Some(Duration::from_secs(7)));
|
||||
assert_eq!(request.input_sources.len(), 2);
|
||||
|
||||
let defaulted = LiteLLMOcrRequest::from_inputs(
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ pub enum Error {
|
|||
#[error(transparent)]
|
||||
Auth(#[from] litellm_auth::Error),
|
||||
#[error(transparent)]
|
||||
Transport(#[from] litellm_llms::custom_httpx::transport::Error),
|
||||
Transport(#[from] litellm_http::transport::Error),
|
||||
#[error(transparent)]
|
||||
Headers(#[from] litellm_llms::custom_httpx::http_handler::HeaderError),
|
||||
Headers(#[from] litellm_http::request::HeaderError),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -95,9 +95,7 @@ impl ResponsesWebSocketConnection {
|
|||
timeout: Option<Duration>,
|
||||
) -> Result<Self, Error> {
|
||||
let mut request = url.into_client_request().map_err(|error| {
|
||||
Error::Transport(litellm_llms::custom_httpx::transport::Error::Network(
|
||||
error.to_string(),
|
||||
))
|
||||
Error::Transport(litellm_http::transport::Error::Network(error.to_string()))
|
||||
})?;
|
||||
for (name, value) in headers {
|
||||
let header_name = name
|
||||
|
|
@ -110,7 +108,7 @@ impl ResponsesWebSocketConnection {
|
|||
let connect = connect_upstream(request);
|
||||
let result = match timeout {
|
||||
Some(timeout) => tokio::time::timeout(timeout, connect).await.map_err(|_| {
|
||||
Error::Transport(litellm_llms::custom_httpx::transport::Error::Network(
|
||||
Error::Transport(litellm_http::transport::Error::Network(
|
||||
"Responses WebSocket connection timed out".into(),
|
||||
))
|
||||
})?,
|
||||
|
|
@ -118,14 +116,12 @@ impl ResponsesWebSocketConnection {
|
|||
};
|
||||
let (socket, _) = result.map_err(|error| match *error {
|
||||
tokio_tungstenite::tungstenite::Error::Http(response) => {
|
||||
Error::Transport(litellm_llms::custom_httpx::transport::Error::Http {
|
||||
Error::Transport(litellm_http::transport::Error::Http {
|
||||
status: response.status().as_u16(),
|
||||
body: String::new(),
|
||||
})
|
||||
}
|
||||
other => Error::Transport(litellm_llms::custom_httpx::transport::Error::Network(
|
||||
other.to_string(),
|
||||
)),
|
||||
other => Error::Transport(litellm_http::transport::Error::Network(other.to_string())),
|
||||
})?;
|
||||
Ok(Self {
|
||||
socket: Arc::new(Mutex::new(Some(socket))),
|
||||
|
|
@ -135,16 +131,12 @@ impl ResponsesWebSocketConnection {
|
|||
pub async fn send_text(&self, text: String) -> Result<(), Error> {
|
||||
let mut socket = self.socket.lock().await;
|
||||
let Some(socket) = socket.as_mut() else {
|
||||
return Err(Error::Transport(
|
||||
litellm_llms::custom_httpx::transport::Error::Network(
|
||||
"Responses WebSocket is closed".into(),
|
||||
),
|
||||
));
|
||||
return Err(Error::Transport(litellm_http::transport::Error::Network(
|
||||
"Responses WebSocket is closed".into(),
|
||||
)));
|
||||
};
|
||||
socket.send(Message::Text(text)).await.map_err(|error| {
|
||||
Error::Transport(litellm_llms::custom_httpx::transport::Error::Network(
|
||||
error.to_string(),
|
||||
))
|
||||
Error::Transport(litellm_http::transport::Error::Network(error.to_string()))
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -160,9 +152,9 @@ impl ResponsesWebSocketConnection {
|
|||
.map_err(|error| Error::InvalidResponse(error.to_string())),
|
||||
Some(Ok(Message::Close(_))) | None => Ok(None),
|
||||
Some(Ok(_)) => Ok(None),
|
||||
Some(Err(error)) => Err(Error::Transport(
|
||||
litellm_llms::custom_httpx::transport::Error::Network(error.to_string()),
|
||||
)),
|
||||
Some(Err(error)) => Err(Error::Transport(litellm_http::transport::Error::Network(
|
||||
error.to_string(),
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -170,9 +162,7 @@ impl ResponsesWebSocketConnection {
|
|||
let mut socket = self.socket.lock().await;
|
||||
if let Some(socket) = socket.as_mut() {
|
||||
socket.close(None).await.map_err(|error| {
|
||||
Error::Transport(litellm_llms::custom_httpx::transport::Error::Network(
|
||||
error.to_string(),
|
||||
))
|
||||
Error::Transport(litellm_http::transport::Error::Network(error.to_string()))
|
||||
})?;
|
||||
}
|
||||
*socket = None;
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
use litellm_host::event::{CallEvent, MachineEvent};
|
||||
use litellm_llms::base_llm::ocr::error::Error;
|
||||
use litellm_llms::base_llm::ocr::{error::Error, settings::OcrSettings};
|
||||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::{
|
||||
test_support::{MockResponse, mock_server, perform_ocr, perform_ocr_with, wire_request},
|
||||
test_support::{
|
||||
MockResponse, mock_server, ocr_client, perform_ocr, perform_ocr_with, wire_request,
|
||||
},
|
||||
wire::{OcrWireRequest, decode_request},
|
||||
};
|
||||
use crate::ocr::route::LocalOcrHost;
|
||||
|
|
@ -200,6 +202,42 @@ async fn immediate_response_normalizes_pages_and_preserves_native() {
|
|||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn client_settings_choose_the_api_version_and_the_inch_to_pixel_dpi() {
|
||||
let (base, seen, server) = mock_server(vec![MockResponse::json(json!({
|
||||
"status":"succeeded",
|
||||
"analyzeResult":{"pages":[{"pageNumber":1,"width":8.5,"height":11,"unit":"inch"}]}
|
||||
}))])
|
||||
.await;
|
||||
let client = ocr_client().with_settings(OcrSettings {
|
||||
document_intelligence_api_version: "2099-01-01".into(),
|
||||
document_intelligence_dpi: 72,
|
||||
..OcrSettings::default()
|
||||
});
|
||||
|
||||
let result = crate::ocr::client::perform(
|
||||
&client,
|
||||
wire_request("azure_ai/doc-intelligence/prebuilt-read", &base, json!({})),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
server.await.unwrap();
|
||||
|
||||
let target = seen.lock().unwrap()[0]
|
||||
.split_whitespace()
|
||||
.nth(1)
|
||||
.unwrap()
|
||||
.to_string();
|
||||
assert_eq!(
|
||||
query_value(&format!("{base}{target}"), "api-version").as_deref(),
|
||||
Some("2099-01-01")
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(&result.pages[0].dimensions).unwrap(),
|
||||
json!({"width":612,"height":792,"dpi":72})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accepted_response_polls_to_success_with_only_credentials() {
|
||||
let operation = json!({"status":"succeeded","analyzeResult":{"pages":[]}});
|
||||
|
|
@ -425,13 +463,19 @@ async fn polling_deadline_bounds_retry_delay() {
|
|||
},
|
||||
])
|
||||
.await;
|
||||
let mut request = wire_request("azure_ai/doc-intelligence/prebuilt-read", &base, json!({}));
|
||||
request.transport.poll_timeout = std::time::Duration::from_millis(100);
|
||||
let request = wire_request("azure_ai/doc-intelligence/prebuilt-read", &base, json!({}));
|
||||
let client = ocr_client().with_settings(OcrSettings {
|
||||
poll_timeout: std::time::Duration::from_millis(100),
|
||||
..OcrSettings::default()
|
||||
});
|
||||
|
||||
let error = tokio::time::timeout(std::time::Duration::from_secs(1), perform_ocr(request))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap_err();
|
||||
let error = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(1),
|
||||
crate::ocr::client::perform(&client, request),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap_err();
|
||||
server.await.unwrap();
|
||||
assert!(error.to_string().contains("timed out"));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,16 +6,15 @@ use litellm_host::{
|
|||
host::{Host, HostOp, HostResult},
|
||||
machine::{HostFailure, Machine, MachineStep},
|
||||
};
|
||||
use litellm_http::{HttpClientPool, HttpSettings, Resolution};
|
||||
use litellm_llms::{
|
||||
base_llm::ocr::{
|
||||
error::Error as OcrError,
|
||||
transformation::{LiteLLMOcrResponse, OCR_RESPONSE_MAX_BYTES, OcrTransportConfig},
|
||||
},
|
||||
custom_httpx::{
|
||||
llm_http_handler::OcrClient,
|
||||
media::{PublicDnsResolver, UrlPolicy},
|
||||
},
|
||||
use litellm_http::{
|
||||
HttpClientPool, HttpSettings, Resolution,
|
||||
media::{PublicDnsResolver, UrlPolicy},
|
||||
};
|
||||
use litellm_llms::base_llm::ocr::{
|
||||
error::Error as OcrError,
|
||||
handler::OcrClient,
|
||||
settings::OcrSettings,
|
||||
transformation::{LiteLLMOcrResponse, OCR_RESPONSE_MAX_BYTES, OcrTransportConfig},
|
||||
};
|
||||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
|
|
@ -175,6 +174,43 @@ async fn facade_retains_native_response_when_requested() {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::plain_key(&[("MISTRAL_API_KEY", "plain")], "plain")]
|
||||
#[case::azure_key_wins(&[("MISTRAL_AZURE_API_KEY", "azure"), ("MISTRAL_API_KEY", "plain")], "azure")]
|
||||
#[case::empty_azure_key_falls_through(&[("MISTRAL_AZURE_API_KEY", ""), ("MISTRAL_API_KEY", "plain")], "plain")]
|
||||
#[tokio::test]
|
||||
async fn mistral_env_fallbacks_follow_python_through_the_injected_secret_source(
|
||||
#[case] secrets: &'static [(&'static str, &'static str)],
|
||||
#[case] expected_key: &str,
|
||||
) {
|
||||
let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await;
|
||||
let secret_base = base.clone();
|
||||
let client = ocr_client().with_secrets(Arc::new(move |name: &str| match name {
|
||||
"MISTRAL_AZURE_API_BASE" => Some(secret_base.clone()),
|
||||
"MISTRAL_API_BASE" => Some("http://127.0.0.1:9/never-read".into()),
|
||||
_ => secrets
|
||||
.iter()
|
||||
.find(|(key, _)| *key == name)
|
||||
.map(|(_, value)| value.to_string()),
|
||||
}));
|
||||
let request = decode_request(OcrWireRequest {
|
||||
model: "mistral/model".into(),
|
||||
document: json!({"type":"document_url","document_url":"data:application/pdf;base64,YWJj"}),
|
||||
api_key: None,
|
||||
api_base: None,
|
||||
custom_llm_provider: None,
|
||||
extra_headers: None,
|
||||
optional_params: Default::default(),
|
||||
input_sources: Default::default(),
|
||||
timeout_seconds: Some(2.0),
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
crate::ocr::client::perform(&client, request).await.unwrap();
|
||||
server.await.unwrap();
|
||||
assert!(seen.lock().unwrap()[0].contains(&format!("authorization: Bearer {expected_key}")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ocr_client_uses_the_injected_http_pool_configuration() {
|
||||
let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await;
|
||||
|
|
@ -187,6 +223,8 @@ async fn ocr_client_uses_the_injected_http_pool_configuration() {
|
|||
&Resolution::from(&settings).config,
|
||||
UrlPolicy::default(),
|
||||
VertexAuth::default(),
|
||||
OcrSettings::default(),
|
||||
Arc::new(litellm_core_utils::settings::ProcessEnvironment),
|
||||
)
|
||||
.unwrap();
|
||||
crate::ocr::client::perform(&client, wire_request("mistral/model", &base, json!({})))
|
||||
|
|
@ -624,7 +662,7 @@ async fn read_bounded_response(response: Vec<u8>, limit: usize) -> Result<bytes:
|
|||
.unwrap();
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(2),
|
||||
litellm_llms::custom_httpx::llm_http_handler::read_response_bytes(response, limit),
|
||||
litellm_llms::base_llm::ocr::handler::read_response_bytes(response, limit),
|
||||
)
|
||||
.await;
|
||||
server.abort();
|
||||
|
|
@ -676,10 +714,7 @@ async fn oversized_error_retains_http_status_and_bounded_diagnostics_without_dra
|
|||
.await
|
||||
.unwrap_err();
|
||||
match error {
|
||||
OcrError::Transport(litellm_llms::custom_httpx::transport::Error::Http {
|
||||
status,
|
||||
body,
|
||||
}) => {
|
||||
OcrError::Transport(litellm_http::transport::Error::Http { status, body }) => {
|
||||
assert_eq!(status, 429);
|
||||
assert_eq!(body, prefix);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,9 +2,10 @@ use std::sync::{Arc, Mutex};
|
|||
|
||||
use futures_util::future::BoxFuture;
|
||||
use litellm_host::event::WireRequest;
|
||||
use litellm_llms::{
|
||||
base_llm::ocr::{error::Error, transformation::LiteLLMOcrResponse},
|
||||
custom_httpx::llm_http_handler::{CallHooks, OcrClient},
|
||||
use litellm_llms::base_llm::ocr::{
|
||||
error::Error,
|
||||
handler::{CallHooks, OcrClient},
|
||||
transformation::LiteLLMOcrResponse,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use tokio::{
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
use litellm_auth::InputSource;
|
||||
use litellm_llms::base_llm::ocr::transformation::OcrResponseFormat;
|
||||
use litellm_llms::base_llm::ocr::{settings::OcrSettings, transformation::OcrResponseFormat};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::test_support::{MockResponse, mock_server, perform_ocr, wire_request};
|
||||
use super::test_support::{MockResponse, mock_server, ocr_client, perform_ocr, wire_request};
|
||||
|
||||
fn request_body(request: &str) -> Value {
|
||||
serde_json::from_str(request.split_once("\r\n\r\n").unwrap().1).unwrap()
|
||||
|
|
@ -48,6 +48,27 @@ async fn facade_executes_vertex_mistral_with_resolved_project_and_location() {
|
|||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn configured_project_and_location_apply_when_the_call_sets_neither() {
|
||||
let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await;
|
||||
let client = ocr_client().with_settings(OcrSettings {
|
||||
vertex_project: Some("configured-project".into()),
|
||||
vertex_location: Some("europe-west4".into()),
|
||||
..OcrSettings::default()
|
||||
});
|
||||
|
||||
crate::ocr::client::perform(
|
||||
&client,
|
||||
wire_request("vertex_ai/mistral-ocr-maas", &base, json!({})),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
server.await.unwrap();
|
||||
assert!(seen.lock().unwrap()[0].starts_with(
|
||||
"POST /v1/projects/configured-project/locations/europe-west4/publishers/mistralai/models/mistral-ocr-maas:rawPredict "
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn supplied_authorization_is_forwarded_without_a_static_token() {
|
||||
let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await;
|
||||
|
|
|
|||
|
|
@ -5,12 +5,19 @@ edition.workspace = true
|
|||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[features]
|
||||
test-support = []
|
||||
|
||||
[dependencies]
|
||||
http.workspace = true
|
||||
litellm-core-utils.workspace = true
|
||||
hyper-util.workspace = true
|
||||
reqwest.workspace = true
|
||||
rustls.workspace = true
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
tokio.workspace = true
|
||||
veil.workspace = true
|
||||
webpki-roots.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ use std::{
|
|||
|
||||
use crate::{
|
||||
error::Error,
|
||||
proxy::EnvironmentProxies,
|
||||
settings::{HttpSettings, SslVerify, TcpKeepalive},
|
||||
tls::{CipherSelection, KeyExchangeGroup, Tls12CipherSuite, Unsupported},
|
||||
};
|
||||
|
|
@ -26,7 +27,7 @@ pub struct HttpClientConfig {
|
|||
pub force_ipv4: bool,
|
||||
pub http2: bool,
|
||||
pub user_agent: Option<String>,
|
||||
pub trust_proxy_env: bool,
|
||||
pub proxies: EnvironmentProxies,
|
||||
pub connect_timeout: Duration,
|
||||
pub tcp_keepalive: Option<TcpKeepalive>,
|
||||
pub pool_idle_timeout: Duration,
|
||||
|
|
@ -72,7 +73,11 @@ impl From<&HttpSettings> for Resolution {
|
|||
force_ipv4: settings.force_ipv4,
|
||||
http2: settings.http2,
|
||||
user_agent: settings.user_agent.clone(),
|
||||
trust_proxy_env: settings.trust_proxy_env,
|
||||
proxies: if settings.trust_proxy_env {
|
||||
settings.proxies.clone()
|
||||
} else {
|
||||
EnvironmentProxies::default()
|
||||
},
|
||||
connect_timeout: settings.connect_timeout,
|
||||
tcp_keepalive: settings.tcp_keepalive,
|
||||
pool_idle_timeout: settings.pool_idle_timeout,
|
||||
|
|
@ -111,11 +116,11 @@ impl TryFrom<&HttpClientConfig> for reqwest::ClientBuilder {
|
|||
Some(agent) => with_protocol.user_agent(agent),
|
||||
None => with_protocol,
|
||||
};
|
||||
Ok(if config.trust_proxy_env {
|
||||
with_agent
|
||||
} else {
|
||||
with_agent.no_proxy()
|
||||
})
|
||||
Ok(config
|
||||
.proxies
|
||||
.reqwest_proxies()
|
||||
.into_iter()
|
||||
.fold(with_agent.no_proxy(), reqwest::ClientBuilder::proxy))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -227,6 +232,25 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
fn proxies() -> EnvironmentProxies {
|
||||
EnvironmentProxies::from_environment(&|name: &str| {
|
||||
(name == "HTTPS_PROXY").then(|| "http://proxy.corp:3128".to_string())
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proxies_are_dropped_when_the_transport_does_not_trust_the_environment() {
|
||||
let settings = HttpSettings {
|
||||
trust_proxy_env: false,
|
||||
proxies: proxies(),
|
||||
..HttpSettings::default()
|
||||
};
|
||||
assert_eq!(
|
||||
Resolution::from(&settings).config.proxies,
|
||||
EnvironmentProxies::default()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn connection_settings_carry_over_unchanged() {
|
||||
let keepalive = TcpKeepalive {
|
||||
|
|
@ -240,6 +264,7 @@ mod tests {
|
|||
http2: true,
|
||||
user_agent: Some("litellm/1.0".into()),
|
||||
trust_proxy_env: true,
|
||||
proxies: proxies(),
|
||||
connect_timeout: Duration::from_secs(7),
|
||||
tcp_keepalive: Some(keepalive),
|
||||
pool_idle_timeout: Duration::from_secs(45),
|
||||
|
|
@ -256,7 +281,7 @@ mod tests {
|
|||
force_ipv4: true,
|
||||
http2: true,
|
||||
user_agent: Some("litellm/1.0".into()),
|
||||
trust_proxy_env: true,
|
||||
proxies: proxies(),
|
||||
connect_timeout: Duration::from_secs(7),
|
||||
tcp_keepalive: Some(keepalive),
|
||||
pool_idle_timeout: Duration::from_secs(45),
|
||||
|
|
|
|||
|
|
@ -1,9 +1,12 @@
|
|||
mod config;
|
||||
mod error;
|
||||
pub mod media;
|
||||
mod pool;
|
||||
mod proxy;
|
||||
pub mod request;
|
||||
mod settings;
|
||||
mod tls;
|
||||
pub mod transport;
|
||||
|
||||
pub use config::{HttpClientConfig, Resolution, Verify};
|
||||
pub use error::Error;
|
||||
|
|
|
|||
|
|
@ -7,12 +7,13 @@ use std::{
|
|||
time::Duration,
|
||||
};
|
||||
|
||||
use litellm_http::{ClientVariant, EnvironmentProxies, HttpClientConfig, HttpClientPool};
|
||||
use reqwest::{
|
||||
Url,
|
||||
dns::{Addrs, Name, Resolve, Resolving},
|
||||
};
|
||||
|
||||
use crate::{ClientVariant, HttpClientConfig, HttpClientPool};
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
#[error("media URL rejected by network policy")]
|
||||
|
|
@ -32,7 +33,7 @@ pub enum Error {
|
|||
#[error("media download timed out")]
|
||||
Timeout,
|
||||
#[error("{0}")]
|
||||
Transport(#[from] crate::custom_httpx::transport::Error),
|
||||
Transport(#[from] crate::transport::Error),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
|
|
@ -101,13 +102,8 @@ impl MediaFetcher {
|
|||
pool: &HttpClientPool,
|
||||
config: &HttpClientConfig,
|
||||
url_policy: UrlPolicy,
|
||||
) -> Result<Self, litellm_http::Error> {
|
||||
let uses_proxy: ProxyMatch = if config.trust_proxy_env {
|
||||
let proxies = EnvironmentProxies::from_environment();
|
||||
Arc::new(move |url| proxies.apply_to(url))
|
||||
} else {
|
||||
Arc::new(|_| false)
|
||||
};
|
||||
) -> Result<Self, crate::Error> {
|
||||
let uses_proxy: ProxyMatch = Arc::new(config.proxies.matcher());
|
||||
Self::with_resolution(
|
||||
pool,
|
||||
config,
|
||||
|
|
@ -123,7 +119,7 @@ impl MediaFetcher {
|
|||
url_policy: UrlPolicy,
|
||||
address_resolver: Arc<dyn AddressResolver>,
|
||||
uses_proxy: ProxyMatch,
|
||||
) -> Result<Self, litellm_http::Error> {
|
||||
) -> Result<Self, crate::Error> {
|
||||
Ok(Self {
|
||||
pinned: pool.client(config, ClientVariant::Media)?,
|
||||
unpinned: pool.client(config, ClientVariant::UnpinnedMedia)?,
|
||||
|
|
@ -168,7 +164,7 @@ impl MediaFetcher {
|
|||
.get(url.clone())
|
||||
.send()
|
||||
.await
|
||||
.map_err(crate::custom_httpx::transport::Error::from)?;
|
||||
.map_err(crate::transport::Error::from)?;
|
||||
if response.status().is_redirection() {
|
||||
if redirects_followed == policy.max_redirects {
|
||||
return Err(Error::TooManyRedirects);
|
||||
|
|
@ -199,7 +195,7 @@ impl MediaFetcher {
|
|||
while let Some(chunk) = response
|
||||
.chunk()
|
||||
.await
|
||||
.map_err(crate::custom_httpx::transport::Error::from)?
|
||||
.map_err(crate::transport::Error::from)?
|
||||
{
|
||||
enforce_download_size(bytes.len() as u64 + chunk.len() as u64, policy.max_bytes)?;
|
||||
bytes.extend_from_slice(&chunk);
|
||||
|
|
@ -249,7 +245,7 @@ impl MediaFetcher {
|
|||
.address_resolver
|
||||
.resolve(host, port)
|
||||
.await
|
||||
.map_err(|error| crate::custom_httpx::transport::Error::Network(error.to_string()))?;
|
||||
.map_err(|error| crate::transport::Error::Network(error.to_string()))?;
|
||||
validate_addresses(&addresses)
|
||||
}
|
||||
}
|
||||
|
|
@ -350,13 +346,13 @@ impl Resolve for PublicDnsResolver {
|
|||
mod tests {
|
||||
use std::collections::HashSet;
|
||||
|
||||
use litellm_http::{HttpSettings, Resolution};
|
||||
use tokio::{
|
||||
io::{AsyncReadExt, AsyncWriteExt},
|
||||
net::TcpListener,
|
||||
};
|
||||
|
||||
use super::*;
|
||||
use crate::{HttpSettings, Resolution};
|
||||
|
||||
async fn serve(response: &'static [u8]) -> (Url, tokio::task::JoinHandle<()>) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0")
|
||||
|
|
@ -443,10 +439,7 @@ mod tests {
|
|||
url_policy: UrlPolicy,
|
||||
uses_proxy: bool,
|
||||
) -> MediaFetcher {
|
||||
let direct = HttpClientConfig {
|
||||
trust_proxy_env: false,
|
||||
..Resolution::from(&HttpSettings::default()).config
|
||||
};
|
||||
let direct = Resolution::from(&HttpSettings::default()).config;
|
||||
MediaFetcher::with_resolution(
|
||||
&HttpClientPool::new(Arc::new(LoopbackDnsResolver(pinned_address))),
|
||||
&direct,
|
||||
|
|
@ -6,7 +6,7 @@ use std::{
|
|||
|
||||
use reqwest::dns::Resolve;
|
||||
|
||||
use crate::{config::HttpClientConfig, error::Error};
|
||||
use crate::{config::HttpClientConfig, error::Error, proxy::EnvironmentProxies};
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
pub enum ClientVariant {
|
||||
|
|
@ -52,7 +52,7 @@ impl HttpClientPool {
|
|||
let effective = match variant {
|
||||
ClientVariant::Media => HttpClientConfig {
|
||||
client_certificate: None,
|
||||
trust_proxy_env: false,
|
||||
proxies: EnvironmentProxies::default(),
|
||||
..config.clone()
|
||||
},
|
||||
ClientVariant::UnpinnedMedia => HttpClientConfig {
|
||||
|
|
@ -138,6 +138,13 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
fn proxied_through(proxy: &str) -> EnvironmentProxies {
|
||||
let proxy = proxy.to_owned();
|
||||
EnvironmentProxies::from_environment(&move |name: &str| {
|
||||
(name == "HTTP_PROXY").then(|| proxy.clone())
|
||||
})
|
||||
}
|
||||
|
||||
async fn serve(
|
||||
status_line: &'static str,
|
||||
) -> (SocketAddr, Arc<AtomicUsize>, Arc<Mutex<Vec<String>>>) {
|
||||
|
|
@ -202,6 +209,50 @@ mod tests {
|
|||
assert_eq!(connections.load(Ordering::SeqCst), 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn provider_clients_route_through_the_resolved_proxy_not_the_process_environment() {
|
||||
let (proxy, connections, requests) = serve("HTTP/1.1 204 No Content").await;
|
||||
let config = HttpClientConfig {
|
||||
proxies: proxied_through(&format!("http://user:secret@{proxy}")),
|
||||
..config("a")
|
||||
};
|
||||
let response = get(
|
||||
&pool(),
|
||||
&config,
|
||||
ClientVariant::Provider,
|
||||
"http://upstream.invalid/v1/ocr",
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 204);
|
||||
assert_eq!(connections.load(Ordering::SeqCst), 1);
|
||||
let request = requests.lock().unwrap().concat();
|
||||
assert!(request.starts_with("GET http://upstream.invalid/v1/ocr HTTP/1.1"));
|
||||
assert!(request.contains("proxy-authorization: Basic dXNlcjpzZWNyZXQ="));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn no_proxy_hosts_bypass_the_resolved_proxy() {
|
||||
let (upstream, _, _) = serve("HTTP/1.1 204 No Content").await;
|
||||
let (proxy, proxy_connections, _) = serve("HTTP/1.1 502 Bad Gateway").await;
|
||||
let config = HttpClientConfig {
|
||||
proxies: EnvironmentProxies::from_environment(&move |name: &str| match name {
|
||||
"HTTP_PROXY" => Some(format!("http://{proxy}")),
|
||||
"NO_PROXY" => Some("127.0.0.1".into()),
|
||||
_ => None,
|
||||
}),
|
||||
..config("a")
|
||||
};
|
||||
let response = get(
|
||||
&pool(),
|
||||
&config,
|
||||
ClientVariant::Provider,
|
||||
&format!("http://{upstream}/v1/ocr"),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 204);
|
||||
assert_eq!(proxy_connections.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn expired_clients_are_rebuilt() {
|
||||
let (address, connections, _) = serve("HTTP/1.1 204 No Content").await;
|
||||
|
|
@ -220,9 +271,12 @@ mod tests {
|
|||
let (address, connections, _) = serve("HTTP/1.1 204 No Content").await;
|
||||
let pool = HttpClientPool::new(Arc::new(FixedResolver(address)));
|
||||
let url = format!("http://media.invalid:{}/doc", address.port());
|
||||
for trust_proxy_env in [true, false] {
|
||||
for proxies in [
|
||||
proxied_through("http://proxy.invalid:3128"),
|
||||
EnvironmentProxies::default(),
|
||||
] {
|
||||
let config = HttpClientConfig {
|
||||
trust_proxy_env,
|
||||
proxies,
|
||||
..config("a")
|
||||
};
|
||||
get(&pool, &config, ClientVariant::Media, &url).await;
|
||||
|
|
|
|||
|
|
@ -1,15 +1,164 @@
|
|||
use hyper_util::client::proxy::matcher::Matcher;
|
||||
use litellm_core_utils::settings::Lookup;
|
||||
use veil::Redact;
|
||||
|
||||
pub struct EnvironmentProxies(Matcher);
|
||||
#[derive(Clone, Redact, Default, PartialEq, Eq, Hash)]
|
||||
pub struct EnvironmentProxies {
|
||||
#[redact]
|
||||
all: String,
|
||||
#[redact]
|
||||
http: String,
|
||||
#[redact]
|
||||
https: String,
|
||||
no: String,
|
||||
}
|
||||
|
||||
impl EnvironmentProxies {
|
||||
pub fn from_environment() -> Self {
|
||||
Self(Matcher::from_system())
|
||||
pub fn from_environment(env: &impl Lookup) -> Self {
|
||||
Self::resolve(env, cfg!(windows))
|
||||
}
|
||||
|
||||
pub fn apply_to(&self, url: &reqwest::Url) -> bool {
|
||||
url.as_str()
|
||||
.parse::<http::Uri>()
|
||||
.is_ok_and(|uri| self.0.intercept(&uri).is_some())
|
||||
fn resolve(env: &impl Lookup, names_ignore_case: bool) -> Self {
|
||||
let lowercase_first = |upper: Option<&str>, lower: &str| {
|
||||
env.get(lower)
|
||||
.or_else(|| upper.and_then(|name| env.truthy(name)))
|
||||
.unwrap_or_default()
|
||||
};
|
||||
let is_cgi = env.get("REQUEST_METHOD").is_some();
|
||||
Self {
|
||||
all: lowercase_first(Some("ALL_PROXY"), "all_proxy"),
|
||||
http: if is_cgi && names_ignore_case {
|
||||
String::new()
|
||||
} else {
|
||||
lowercase_first((!is_cgi).then_some("HTTP_PROXY"), "http_proxy")
|
||||
},
|
||||
https: lowercase_first(Some("HTTPS_PROXY"), "https_proxy"),
|
||||
no: lowercase_first(Some("NO_PROXY"), "no_proxy"),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn matcher(&self) -> impl Fn(&reqwest::Url) -> bool + Send + Sync + use<> {
|
||||
let matcher = Matcher::builder()
|
||||
.all(self.all.clone())
|
||||
.http(self.http.clone())
|
||||
.https(self.https.clone())
|
||||
.no(self.no.clone())
|
||||
.build();
|
||||
move |url| {
|
||||
url.as_str()
|
||||
.parse::<http::Uri>()
|
||||
.is_ok_and(|uri| matcher.intercept(&uri).is_some())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn reqwest_proxies(&self) -> Vec<reqwest::Proxy> {
|
||||
let no_proxy = reqwest::NoProxy::from_string(&self.no);
|
||||
[
|
||||
reqwest::Proxy::http(self.http.as_str()),
|
||||
reqwest::Proxy::https(self.https.as_str()),
|
||||
reqwest::Proxy::all(self.all.as_str()),
|
||||
]
|
||||
.into_iter()
|
||||
.filter_map(Result::ok)
|
||||
.map(|proxy| proxy.no_proxy(no_proxy.clone()))
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn env_of(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
|
||||
move |name| {
|
||||
values
|
||||
.iter()
|
||||
.find(|(key, _)| *key == name)
|
||||
.map(|(_, value)| value.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
fn url(value: &str) -> reqwest::Url {
|
||||
reqwest::Url::parse(value).unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::http_only(&[("HTTP_PROXY", "http://proxy:3128")], "http://api.test/", true)]
|
||||
#[case::http_proxy_skips_https(&[("HTTP_PROXY", "http://proxy:3128")], "https://api.test/", false)]
|
||||
#[case::all_covers_https(&[("ALL_PROXY", "http://proxy:3128")], "https://api.test/", true)]
|
||||
#[case::lowercase(&[("https_proxy", "http://proxy:3128")], "https://api.test/", true)]
|
||||
#[case::no_proxy_bypass(&[("HTTPS_PROXY", "http://proxy:3128"), ("NO_PROXY", "api.test")], "https://api.test/", false)]
|
||||
fn proxies_follow_the_injected_environment(
|
||||
#[case] env: &'static [(&'static str, &'static str)],
|
||||
#[case] target: &str,
|
||||
#[case] expected: bool,
|
||||
) {
|
||||
let proxies = EnvironmentProxies::from_environment(&env_of(env));
|
||||
assert_eq!(proxies.matcher()(&url(target)), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::lowercase_wins(&[("HTTPS_PROXY", "http://upper:3128"), ("https_proxy", "http://lower:3128")], &[("https_proxy", "http://lower:3128")])]
|
||||
#[case::empty_uppercase_falls_through_to_lowercase(&[("HTTPS_PROXY", ""), ("https_proxy", "http://lower:3128")], &[("https_proxy", "http://lower:3128")])]
|
||||
#[case::empty_uppercase_alone_is_unset(&[("HTTPS_PROXY", "")], &[])]
|
||||
#[case::empty_lowercase_clears_the_uppercase_value(&[("HTTPS_PROXY", "http://upper:3128"), ("https_proxy", "")], &[])]
|
||||
#[case::lowercase_no_proxy_wins(&[("NO_PROXY", "upper.test"), ("no_proxy", "lower.test")], &[("no_proxy", "lower.test")])]
|
||||
#[case::cgi_forgets_the_client_settable_http_proxy(&[("REQUEST_METHOD", "GET"), ("HTTP_PROXY", "http://attacker:3128")], &[])]
|
||||
#[case::cgi_keeps_lowercase_http_proxy(&[("REQUEST_METHOD", "GET"), ("HTTP_PROXY", "http://attacker:3128"), ("http_proxy", "http://lower:3128")], &[("http_proxy", "http://lower:3128")])]
|
||||
#[case::cgi_keeps_every_other_variable(&[("REQUEST_METHOD", "GET"), ("HTTPS_PROXY", "http://proxy:3128"), ("ALL_PROXY", "http://all:3128"), ("NO_PROXY", "internal.test")], &[("HTTPS_PROXY", "http://proxy:3128"), ("ALL_PROXY", "http://all:3128"), ("NO_PROXY", "internal.test")])]
|
||||
fn variables_resolve_like_urllib_getproxies_environment(
|
||||
#[case] env: &'static [(&'static str, &'static str)],
|
||||
#[case] equivalent: &'static [(&'static str, &'static str)],
|
||||
) {
|
||||
assert_eq!(
|
||||
EnvironmentProxies::from_environment(&env_of(env)),
|
||||
EnvironmentProxies::from_environment(&env_of(equivalent))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cgi_drops_http_proxy_entirely_where_variable_names_ignore_case() {
|
||||
let windows_env = |name: &str| match name.to_ascii_uppercase().as_str() {
|
||||
"REQUEST_METHOD" => Some("GET".to_string()),
|
||||
"HTTP_PROXY" => Some("http://attacker:3128".to_string()),
|
||||
"HTTPS_PROXY" => Some("http://proxy:3128".to_string()),
|
||||
_ => None,
|
||||
};
|
||||
let proxies = EnvironmentProxies::resolve(&windows_env, true);
|
||||
assert!(!proxies.matcher()(&url("http://api.test/")));
|
||||
assert!(proxies.matcher()(&url("https://api.test/")));
|
||||
assert!(EnvironmentProxies::resolve(&windows_env, false).matcher()(
|
||||
&url("http://api.test/")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_cgi_request_still_proxies_https_through_the_configured_proxy() {
|
||||
let proxies = EnvironmentProxies::from_environment(&env_of(&[
|
||||
("REQUEST_METHOD", "GET"),
|
||||
("HTTPS_PROXY", "http://proxy:3128"),
|
||||
]));
|
||||
assert!(proxies.matcher()(&url("https://api.test/")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn debug_output_hides_proxy_credentials_but_shows_which_variables_are_set() {
|
||||
let proxies = EnvironmentProxies::from_environment(&env_of(&[
|
||||
("HTTPS_PROXY", "http://operator:hunter2@proxy.corp:3128"),
|
||||
("NO_PROXY", "internal.test"),
|
||||
]));
|
||||
let debug = format!("{proxies:?}");
|
||||
assert!(!debug.contains("hunter2") && !debug.contains("operator"));
|
||||
assert!(debug.contains("internal.test"));
|
||||
assert_ne!(debug, format!("{:?}", EnvironmentProxies::default()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_environment_proxies_nothing() {
|
||||
let proxies = EnvironmentProxies::from_environment(&env_of(&[]));
|
||||
assert_eq!(proxies, EnvironmentProxies::default());
|
||||
assert!(proxies.reqwest_proxies().is_empty());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,20 +13,12 @@ use serde_json::{Map, Value};
|
|||
/// before truncation, so provider bodies are bounded and data-minimized.
|
||||
const UPSTREAM_ERROR_BODY_MAX_CHARS: usize = 256;
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "used by the OCR architecture in the next stacked PR"
|
||||
)]
|
||||
pub enum HeaderPolicy<'a> {
|
||||
All,
|
||||
Only(&'a [&'a str]),
|
||||
Except(&'a [&'a str]),
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "used by the OCR architecture in the next stacked PR"
|
||||
)]
|
||||
pub fn with_headers(
|
||||
builder: reqwest::RequestBuilder,
|
||||
headers: &[(String, String)],
|
||||
|
|
@ -107,18 +99,6 @@ pub fn has_bearer_auth(headers: &[(String, String)]) -> bool {
|
|||
})
|
||||
}
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "used by the OCR architecture in the next stacked PR"
|
||||
)]
|
||||
pub fn deserialize_optional_param<'de, D, T>(deserializer: D) -> Result<Option<Option<T>>, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
T: serde::Deserialize<'de>,
|
||||
{
|
||||
<Option<T> as serde::Deserialize>::deserialize(deserializer).map(Some)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::json;
|
||||
|
|
@ -3,6 +3,10 @@ use std::{
|
|||
time::Duration,
|
||||
};
|
||||
|
||||
use litellm_core_utils::settings::{Layer, Lookup, merge};
|
||||
|
||||
use crate::proxy::EnvironmentProxies;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
|
||||
pub enum SslVerify {
|
||||
Enabled,
|
||||
|
|
@ -42,41 +46,41 @@ pub struct HttpSettingsLayer {
|
|||
pub user_agent: Option<String>,
|
||||
pub tcp_keepalive: Option<TcpKeepalive>,
|
||||
pub pool_idle_timeout: Option<Duration>,
|
||||
pub proxies: Option<EnvironmentProxies>,
|
||||
}
|
||||
|
||||
impl HttpSettingsLayer {
|
||||
pub fn from_environment(env: &(dyn Fn(&str) -> Option<String> + Sync)) -> Self {
|
||||
let enabled = |name: &str| {
|
||||
env(name)
|
||||
.is_some_and(|value| value.trim().eq_ignore_ascii_case("true"))
|
||||
.then_some(true)
|
||||
};
|
||||
let number = |name: &str| env(name).and_then(|value| value.trim().parse::<u32>().ok());
|
||||
pub fn from_environment(env: &impl Lookup) -> Self {
|
||||
let seconds = |name: &str, default: u32| {
|
||||
Duration::from_secs(u64::from(number(name).unwrap_or(default)))
|
||||
Duration::from_secs(u64::from(env.parsed::<u32>(name).unwrap_or(default)))
|
||||
};
|
||||
Self {
|
||||
ssl_verify: env("SSL_VERIFY").map(|value| SslVerify::parse(&value)),
|
||||
ssl_cert_file: env("SSL_CERT_FILE").map(PathBuf::from),
|
||||
ssl_certificate: env("SSL_CERTIFICATE").map(PathBuf::from),
|
||||
ssl_security_level: env("SSL_SECURITY_LEVEL"),
|
||||
ssl_ecdh_curve: env("SSL_ECDH_CURVE"),
|
||||
ssl_verify: env.get("SSL_VERIFY").map(|value| SslVerify::parse(&value)),
|
||||
ssl_cert_file: env.truthy("SSL_CERT_FILE").map(PathBuf::from),
|
||||
ssl_certificate: env.get("SSL_CERTIFICATE").map(PathBuf::from),
|
||||
ssl_security_level: env.get("SSL_SECURITY_LEVEL"),
|
||||
ssl_ecdh_curve: env.get("SSL_ECDH_CURVE"),
|
||||
force_ipv4: None,
|
||||
http2: enabled("LITELLM_HTTP2"),
|
||||
aiohttp_trust_env: enabled("AIOHTTP_TRUST_ENV"),
|
||||
disable_aiohttp_trust_env: enabled("DISABLE_AIOHTTP_TRUST_ENV"),
|
||||
disable_aiohttp_transport: enabled("DISABLE_AIOHTTP_TRANSPORT"),
|
||||
user_agent: env("LITELLM_USER_AGENT"),
|
||||
tcp_keepalive: enabled("AIOHTTP_SO_KEEPALIVE").map(|_| TcpKeepalive {
|
||||
http2: env.enabled("LITELLM_HTTP2"),
|
||||
aiohttp_trust_env: env.enabled("AIOHTTP_TRUST_ENV"),
|
||||
disable_aiohttp_trust_env: env.enabled("DISABLE_AIOHTTP_TRUST_ENV"),
|
||||
disable_aiohttp_transport: env.enabled("DISABLE_AIOHTTP_TRANSPORT"),
|
||||
user_agent: env.get("LITELLM_USER_AGENT"),
|
||||
tcp_keepalive: env.enabled("AIOHTTP_SO_KEEPALIVE").map(|_| TcpKeepalive {
|
||||
idle: seconds("AIOHTTP_TCP_KEEPIDLE", 60),
|
||||
interval: seconds("AIOHTTP_TCP_KEEPINTVL", 30),
|
||||
retries: number("AIOHTTP_TCP_KEEPCNT").unwrap_or(5),
|
||||
retries: env.parsed("AIOHTTP_TCP_KEEPCNT").unwrap_or(5),
|
||||
}),
|
||||
pool_idle_timeout: number("AIOHTTP_KEEPALIVE_TIMEOUT")
|
||||
pool_idle_timeout: env
|
||||
.parsed::<u32>("AIOHTTP_KEEPALIVE_TIMEOUT")
|
||||
.map(|timeout| Duration::from_secs(u64::from(timeout))),
|
||||
proxies: Some(EnvironmentProxies::from_environment(env))
|
||||
.filter(|proxies| *proxies != EnvironmentProxies::default()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Layer for HttpSettingsLayer {
|
||||
fn or(self, lower: Self) -> Self {
|
||||
Self {
|
||||
ssl_verify: self.ssl_verify.or(lower.ssl_verify),
|
||||
|
|
@ -96,6 +100,7 @@ impl HttpSettingsLayer {
|
|||
user_agent: self.user_agent.or(lower.user_agent),
|
||||
tcp_keepalive: self.tcp_keepalive.or(lower.tcp_keepalive),
|
||||
pool_idle_timeout: self.pool_idle_timeout.or(lower.pool_idle_timeout),
|
||||
proxies: self.proxies.or(lower.proxies),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -111,6 +116,7 @@ pub struct HttpSettings {
|
|||
pub http2: bool,
|
||||
pub user_agent: Option<String>,
|
||||
pub trust_proxy_env: bool,
|
||||
pub proxies: EnvironmentProxies,
|
||||
pub connect_timeout: Duration,
|
||||
pub tcp_keepalive: Option<TcpKeepalive>,
|
||||
pub pool_idle_timeout: Duration,
|
||||
|
|
@ -128,6 +134,7 @@ impl Default for HttpSettings {
|
|||
http2: false,
|
||||
user_agent: None,
|
||||
trust_proxy_env: true,
|
||||
proxies: EnvironmentProxies::default(),
|
||||
connect_timeout: Duration::from_secs(10),
|
||||
tcp_keepalive: None,
|
||||
pool_idle_timeout: Duration::from_secs(120),
|
||||
|
|
@ -139,10 +146,7 @@ impl HttpSettings {
|
|||
pub fn from_layers(
|
||||
highest_precedence_first: impl IntoIterator<Item = HttpSettingsLayer>,
|
||||
) -> Self {
|
||||
let merged = highest_precedence_first
|
||||
.into_iter()
|
||||
.reduce(HttpSettingsLayer::or)
|
||||
.unwrap_or_default();
|
||||
let merged = merge(highest_precedence_first);
|
||||
let defaults = Self::default();
|
||||
let http2 = merged.http2.unwrap_or(defaults.http2);
|
||||
Self {
|
||||
|
|
@ -164,6 +168,7 @@ impl HttpSettings {
|
|||
pool_idle_timeout: merged
|
||||
.pool_idle_timeout
|
||||
.unwrap_or(defaults.pool_idle_timeout),
|
||||
proxies: merged.proxies.unwrap_or_default(),
|
||||
..defaults
|
||||
}
|
||||
}
|
||||
|
|
@ -190,9 +195,7 @@ mod tests {
|
|||
None
|
||||
}
|
||||
|
||||
fn env_of(
|
||||
values: &'static [(&'static str, &'static str)],
|
||||
) -> impl Fn(&str) -> Option<String> + Sync {
|
||||
fn env_of(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
|
||||
move |name| {
|
||||
values
|
||||
.iter()
|
||||
|
|
|
|||
|
|
@ -46,11 +46,8 @@ mod tests {
|
|||
.send()
|
||||
.await
|
||||
.expect_err("invalid port");
|
||||
let error = crate::custom_httpx::transport::Error::from_reqwest_before_dispatch(error);
|
||||
assert!(matches!(
|
||||
error,
|
||||
crate::custom_httpx::transport::Error::Connect(_)
|
||||
));
|
||||
let error = crate::transport::Error::from_reqwest_before_dispatch(error);
|
||||
assert!(matches!(error, crate::transport::Error::Connect(_)));
|
||||
assert!(!error.to_string().contains("secret"));
|
||||
assert!(!error.to_string().contains("private"));
|
||||
}
|
||||
|
|
@ -76,7 +73,7 @@ mod tests {
|
|||
.await
|
||||
.expect_err("nothing listens on the port");
|
||||
let root_cause = root_cause(&error).expect("reqwest reports a cause");
|
||||
let message = crate::custom_httpx::transport::Error::from(error).to_string();
|
||||
let message = crate::transport::Error::from(error).to_string();
|
||||
assert!(message.contains(&root_cause), "{message}");
|
||||
assert!(!message.contains("secret"));
|
||||
}
|
||||
|
|
@ -105,8 +102,8 @@ mod tests {
|
|||
let error = response.expect_err("server does not respond");
|
||||
assert!(error.is_timeout());
|
||||
assert!(matches!(
|
||||
crate::custom_httpx::transport::Error::from_reqwest_before_dispatch(error),
|
||||
crate::custom_httpx::transport::Error::Network(_)
|
||||
crate::transport::Error::from_reqwest_before_dispatch(error),
|
||||
crate::transport::Error::Network(_)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
litellm-llms mirrors `litellm/llms/`: base config traits, provider transformations, and the `custom_httpx` handlers. See `../core/AGENTS.md` for how the crates layer.
|
||||
litellm-llms mirrors `litellm/llms/`: base config traits, provider transformations, and the OCR request handler in `base_llm/ocr/handler.rs`. Transport code (clients, media fetching, header helpers, transport errors) lives in `litellm-http`. See `../core/AGENTS.md` for how the crates layer.
|
||||
|
||||
## Python/Rust transformation pairs
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ license.workspace = true
|
|||
repository.workspace = true
|
||||
|
||||
[features]
|
||||
test-support = []
|
||||
test-support = ["litellm-http/test-support"]
|
||||
|
||||
[dependencies]
|
||||
litellm-types.workspace = true
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ use crate::{
|
|||
base_llm::ocr::{
|
||||
document::{inline_remote_document, validate_inline_document},
|
||||
error::Error,
|
||||
handler::OcrClient,
|
||||
transformation::{
|
||||
BaseOcrConfig, LiteLLMOcrResponse, OcrDocument, OcrRequestContext, OcrResponseFormat,
|
||||
PreparedOcrRequest,
|
||||
|
|
@ -13,7 +14,6 @@ use crate::{
|
|||
cohere::ocr::transformation::{
|
||||
CohereOptions, CohereParseConfig, CohereRequest, validate_document,
|
||||
},
|
||||
custom_httpx::llm_http_handler::OcrClient,
|
||||
};
|
||||
|
||||
#[derive(Default)]
|
||||
|
|
@ -53,7 +53,7 @@ impl BaseOcrConfig for AzureAICohereParseConfig {
|
|||
) -> Result<String, Error> {
|
||||
let base = super::transformation::AzureAiOcrConfig::resolve_api_base(
|
||||
request.connection.api_base.as_deref(),
|
||||
&crate::base_llm::ocr::transformation::credential_env,
|
||||
&|name: &str| request.connection.secret(name),
|
||||
)?;
|
||||
self.get_complete_url(&base)
|
||||
}
|
||||
|
|
@ -108,7 +108,7 @@ impl BaseOcrConfig for AzureAICohereParseConfig {
|
|||
}
|
||||
|
||||
fn validate_request_body(&self, body: &Value) -> Result<(), Error> {
|
||||
let document = crate::custom_httpx::llm_http_handler::body_document(body)?;
|
||||
let document = crate::base_llm::ocr::handler::body_document(body)?;
|
||||
validate_document(&document)?;
|
||||
validate_inline_document(&document)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,21 @@ use std::sync::OnceLock;
|
|||
use litellm_auth::{InputSource, Sourced};
|
||||
use litellm_auth_azure::{AzureAuthInputs, AzureAuthService};
|
||||
|
||||
use crate::base_llm::ocr::{error::Error, transformation::OcrConnection};
|
||||
use crate::base_llm::ocr::{
|
||||
error::Error,
|
||||
transformation::{OcrConnection, PreparedOcrRequest},
|
||||
};
|
||||
|
||||
pub(crate) fn azure_auth_inputs(request: &PreparedOcrRequest) -> Result<AzureAuthInputs, Error> {
|
||||
Ok(AzureAuthInputs {
|
||||
azure_ad_token_provider: request.azure_ad_token_provider.clone(),
|
||||
..AzureAuthInputs::from_sourced_optional_params(
|
||||
&request.optional_params,
|
||||
&request.input_sources,
|
||||
)?
|
||||
}
|
||||
.or_configured_token_refresh(request.connection.settings.enable_azure_ad_token_refresh))
|
||||
}
|
||||
|
||||
pub(super) async fn resolve_entra(
|
||||
config: &AzureAuthInputs,
|
||||
|
|
|
|||
|
|
@ -14,24 +14,20 @@ use serde_json::{Map, Value};
|
|||
use serde_with::serde_as;
|
||||
use tokio::time::Instant;
|
||||
|
||||
use crate::{
|
||||
base_llm::ocr::{
|
||||
document::InlineDocument,
|
||||
error::Error,
|
||||
transformation::{
|
||||
BaseOcrConfig, DecodedOcrResponse, LiteLLMOcrResponse, OCR_INLINE_MAX_BYTES,
|
||||
OCR_POLL_RETRY_SECS, OcrConnection, OcrCredentialInputs, OcrDocument, OcrPage,
|
||||
OcrPageDimensions, OcrResponseContext, OcrResponseFormat, OcrUsageInfo,
|
||||
PreparedOcrRequest, ResolvedOcrCredentials, credential_env,
|
||||
decode_and_normalize_response, decode_response,
|
||||
},
|
||||
use crate::base_llm::ocr::{
|
||||
document::InlineDocument,
|
||||
error::Error,
|
||||
handler::{CallHooks, OcrClient, read_json_response},
|
||||
settings::OcrSettings,
|
||||
transformation::{
|
||||
BaseOcrConfig, DecodedOcrResponse, LiteLLMOcrResponse, OCR_INLINE_MAX_BYTES,
|
||||
OCR_POLL_RETRY_SECS, OcrConnection, OcrCredentialInputs, OcrDocument, OcrPage,
|
||||
OcrPageDimensions, OcrResponseContext, OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest,
|
||||
ResolvedOcrCredentials, decode_and_normalize_response, decode_response,
|
||||
},
|
||||
custom_httpx::llm_http_handler::{CallHooks, OcrClient, read_json_response},
|
||||
};
|
||||
|
||||
const AZURE_DI_API_VERSION: &str = "2024-11-30";
|
||||
const AZURE_DI_SUBSCRIPTION_HEADER: &str = "Ocp-Apim-Subscription-Key";
|
||||
const AZURE_DI_DEFAULT_DPI: i64 = 96;
|
||||
const AZURE_DI_DEFAULT_WIDTH: f64 = 8.5;
|
||||
const AZURE_DI_DEFAULT_HEIGHT: f64 = 11.0;
|
||||
|
||||
|
|
@ -178,15 +174,11 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig {
|
|||
request: &PreparedOcrRequest,
|
||||
_client: &OcrClient,
|
||||
) -> Result<Self::Environment, Error> {
|
||||
let config = AzureAuthInputs {
|
||||
azure_ad_token_provider: request.azure_ad_token_provider.clone(),
|
||||
..AzureAuthInputs::from_sourced_optional_params(
|
||||
&request.optional_params,
|
||||
&request.input_sources,
|
||||
)?
|
||||
};
|
||||
self.resolve_headers(&request.connection, &config, &credential_env)
|
||||
.await
|
||||
let config = crate::azure_ai::ocr::common_utils::azure_auth_inputs(request)?;
|
||||
self.resolve_headers(&request.connection, &config, &|name: &str| {
|
||||
request.connection.secret(name)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
fn get_complete_url(
|
||||
|
|
@ -196,9 +188,17 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig {
|
|||
_environment: &Self::Environment,
|
||||
) -> Result<String, Error> {
|
||||
let endpoint = nonblank(request.connection.api_base.clone())
|
||||
.or_else(|| nonblank(credential_env(AZURE_DI_ENDPOINT_ENV)))
|
||||
.or_else(|| nonblank(request.connection.secret(AZURE_DI_ENDPOINT_ENV)))
|
||||
.ok_or_else(|| Error::Auth(litellm_auth::Error::ProviderAuthentication("Missing Azure Document Intelligence API Base - Set AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT or pass api_base".into())))?;
|
||||
self.build_ocr_url(&endpoint, &request.model, optional_params)
|
||||
self.build_ocr_url(
|
||||
&endpoint,
|
||||
&request.model,
|
||||
optional_params,
|
||||
&request
|
||||
.connection
|
||||
.settings
|
||||
.document_intelligence_api_version,
|
||||
)
|
||||
}
|
||||
|
||||
fn transform_ocr_request(
|
||||
|
|
@ -217,12 +217,13 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig {
|
|||
raw_response: &[u8],
|
||||
request_format: OcrResponseFormat,
|
||||
) -> Result<LiteLLMOcrResponse, Error> {
|
||||
decode_and_normalize_response(
|
||||
model,
|
||||
raw_response,
|
||||
request_format,
|
||||
transform_completed_response,
|
||||
)
|
||||
decode_and_normalize_response(model, raw_response, request_format, |model, response| {
|
||||
transform_completed_response(
|
||||
model,
|
||||
response,
|
||||
OcrSettings::default().document_intelligence_dpi,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
async fn async_transform_ocr_response(
|
||||
|
|
@ -243,7 +244,11 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig {
|
|||
.await?;
|
||||
Ok(LiteLLMOcrResponse {
|
||||
provider_native_response: decoded.native,
|
||||
..transform_completed_response(model, decoded.data)?
|
||||
..transform_completed_response(
|
||||
model,
|
||||
decoded.data,
|
||||
context.connection.settings.document_intelligence_dpi,
|
||||
)?
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -356,6 +361,7 @@ fn build_request(document: OcrDocument) -> Result<DocumentIntelligenceRequest, E
|
|||
fn transform_completed_response(
|
||||
model: &str,
|
||||
response: AzureDocumentIntelligenceOperation,
|
||||
dpi: i64,
|
||||
) -> Result<LiteLLMOcrResponse, Error> {
|
||||
if response.status != Some(OperationStatus::Succeeded) {
|
||||
return Err(Error::OperationStatus(
|
||||
|
|
@ -369,7 +375,7 @@ fn transform_completed_response(
|
|||
let pages = result
|
||||
.pages
|
||||
.into_iter()
|
||||
.map(transform_azure_page)
|
||||
.map(|page| transform_azure_page(page, dpi))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let pages_processed = i64::try_from(pages.len()).map_err(|_| Error::NumericRange("pages"))?;
|
||||
Ok(LiteLLMOcrResponse {
|
||||
|
|
@ -384,7 +390,7 @@ fn transform_completed_response(
|
|||
})
|
||||
}
|
||||
|
||||
fn transform_azure_page(page: AzureDocumentIntelligencePage) -> Result<OcrPage, Error> {
|
||||
fn transform_azure_page(page: AzureDocumentIntelligencePage, dpi: i64) -> Result<OcrPage, Error> {
|
||||
let index = page
|
||||
.page_number
|
||||
.unwrap_or(1)
|
||||
|
|
@ -394,6 +400,7 @@ fn transform_azure_page(page: AzureDocumentIntelligencePage) -> Result<OcrPage,
|
|||
page.width.unwrap_or(AZURE_DI_DEFAULT_WIDTH),
|
||||
page.height.unwrap_or(AZURE_DI_DEFAULT_HEIGHT),
|
||||
page.unit.as_deref().unwrap_or("inch"),
|
||||
dpi,
|
||||
)?;
|
||||
let markdown = page
|
||||
.lines
|
||||
|
|
@ -409,16 +416,17 @@ fn transform_azure_page(page: AzureDocumentIntelligencePage) -> Result<OcrPage,
|
|||
})
|
||||
}
|
||||
|
||||
fn convert_dimensions(width: f64, height: f64, unit: &str) -> Result<OcrPageDimensions, Error> {
|
||||
let scale = if unit == "inch" {
|
||||
AZURE_DI_DEFAULT_DPI as f64
|
||||
} else {
|
||||
1.0
|
||||
};
|
||||
fn convert_dimensions(
|
||||
width: f64,
|
||||
height: f64,
|
||||
unit: &str,
|
||||
dpi: i64,
|
||||
) -> Result<OcrPageDimensions, Error> {
|
||||
let scale = if unit == "inch" { dpi as f64 } else { 1.0 };
|
||||
Ok(OcrPageDimensions {
|
||||
width: Some(pixel_dimension(width, scale, "page.width")?),
|
||||
height: Some(pixel_dimension(height, scale, "page.height")?),
|
||||
dpi: Some(AZURE_DI_DEFAULT_DPI),
|
||||
dpi: Some(dpi),
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -440,7 +448,7 @@ async fn read_operation_response(
|
|||
hooks: &dyn CallHooks<Error>,
|
||||
) -> Result<DecodedOcrResponse<AzureDocumentIntelligenceOperation>, Error> {
|
||||
if response.status() != reqwest::StatusCode::ACCEPTED {
|
||||
let bytes = crate::custom_httpx::llm_http_handler::read_response_bytes(
|
||||
let bytes = crate::base_llm::ocr::handler::read_response_bytes(
|
||||
response,
|
||||
connection.max_response_bytes,
|
||||
)
|
||||
|
|
@ -462,11 +470,9 @@ async fn read_operation_response(
|
|||
{
|
||||
return Err(Error::PollOrigin);
|
||||
}
|
||||
let bytes = crate::custom_httpx::llm_http_handler::read_response_bytes(
|
||||
response,
|
||||
connection.max_response_bytes,
|
||||
)
|
||||
.await?;
|
||||
let bytes =
|
||||
crate::base_llm::ocr::handler::read_response_bytes(response, connection.max_response_bytes)
|
||||
.await?;
|
||||
hooks.response_received(&bytes).await?;
|
||||
poll_operation(http_client, operation, headers, connection, native, hooks).await
|
||||
}
|
||||
|
|
@ -480,7 +486,7 @@ async fn poll_operation(
|
|||
hooks: &dyn CallHooks<Error>,
|
||||
) -> Result<DecodedOcrResponse<AzureDocumentIntelligenceOperation>, Error> {
|
||||
let deadline = Instant::now()
|
||||
.checked_add(connection.poll_timeout)
|
||||
.checked_add(connection.settings.poll_timeout)
|
||||
.ok_or(Error::PollTimeout)?;
|
||||
|
||||
loop {
|
||||
|
|
@ -491,21 +497,19 @@ async fn poll_operation(
|
|||
let builder = http_client
|
||||
.get(url.clone())
|
||||
.timeout(remaining.min(connection.timeout));
|
||||
let builder = crate::custom_httpx::http_handler::with_headers(
|
||||
let builder = litellm_http::request::with_headers(
|
||||
builder,
|
||||
headers,
|
||||
crate::custom_httpx::http_handler::HeaderPolicy::Only(&[
|
||||
litellm_http::request::HeaderPolicy::Only(&[
|
||||
AZURE_DI_SUBSCRIPTION_HEADER,
|
||||
"authorization",
|
||||
]),
|
||||
);
|
||||
let response = tokio::time::timeout_at(
|
||||
deadline,
|
||||
crate::custom_httpx::http_handler::http_request(builder),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| Error::PollTimeout)?
|
||||
.map_err(crate::custom_httpx::transport::Error::from)?;
|
||||
let response =
|
||||
tokio::time::timeout_at(deadline, litellm_http::request::http_request(builder))
|
||||
.await
|
||||
.map_err(|_| Error::PollTimeout)?
|
||||
.map_err(litellm_http::transport::Error::from)?;
|
||||
let retry = response
|
||||
.headers()
|
||||
.get(reqwest::header::RETRY_AFTER)
|
||||
|
|
@ -551,13 +555,14 @@ impl AzureDocumentIntelligenceOcrConfig {
|
|||
endpoint: &str,
|
||||
model: &str,
|
||||
params: &DocumentIntelligenceParams,
|
||||
api_version: &str,
|
||||
) -> Result<String, Error> {
|
||||
let model = format!("{}:analyze", model_id(model)?);
|
||||
ApiUrl::parse(endpoint)
|
||||
.and_then(|url| url.complete_path(&["documentintelligence", "documentModels", &model]))
|
||||
.map(|url| {
|
||||
url.append_query_pairs(
|
||||
[("api-version", AZURE_DI_API_VERSION)]
|
||||
[("api-version", api_version)]
|
||||
.into_iter()
|
||||
.chain(params.pages.iter().map(|pages| ("pages", pages.as_str())))
|
||||
.chain(
|
||||
|
|
@ -580,8 +585,8 @@ impl AzureDocumentIntelligenceOcrConfig {
|
|||
config: &AzureAuthInputs,
|
||||
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
|
||||
) -> Result<Vec<(String, String)>, Error> {
|
||||
if crate::custom_httpx::http_handler::has_header(&connection.extra_headers, "authorization")
|
||||
|| crate::custom_httpx::http_handler::has_header(
|
||||
if litellm_http::request::has_header(&connection.extra_headers, "authorization")
|
||||
|| litellm_http::request::has_header(
|
||||
&connection.extra_headers,
|
||||
AZURE_DI_SUBSCRIPTION_HEADER,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,12 +7,12 @@ use crate::{
|
|||
base_llm::ocr::{
|
||||
document::{inline_remote_document, validate_inline_document},
|
||||
error::Error,
|
||||
handler::OcrClient,
|
||||
transformation::{
|
||||
BaseOcrConfig, LiteLLMOcrResponse, OcrConnection, OcrDocument, OcrRequestContext,
|
||||
OcrResponseFormat, PreparedOcrRequest, credential_env,
|
||||
OcrResponseFormat, PreparedOcrRequest,
|
||||
},
|
||||
},
|
||||
custom_httpx::llm_http_handler::OcrClient,
|
||||
mistral::ocr::transformation::{MistralOcrConfig, MistralOcrRequest},
|
||||
};
|
||||
|
||||
|
|
@ -50,15 +50,11 @@ impl BaseOcrConfig for AzureAiOcrConfig {
|
|||
request: &PreparedOcrRequest,
|
||||
_client: &OcrClient,
|
||||
) -> Result<Self::Environment, Error> {
|
||||
let config = AzureAuthInputs {
|
||||
azure_ad_token_provider: request.azure_ad_token_provider.clone(),
|
||||
..AzureAuthInputs::from_sourced_optional_params(
|
||||
&request.optional_params,
|
||||
&request.input_sources,
|
||||
)?
|
||||
};
|
||||
self.resolve_headers(&request.connection, &config, &credential_env)
|
||||
.await
|
||||
let config = crate::azure_ai::ocr::common_utils::azure_auth_inputs(request)?;
|
||||
self.resolve_headers(&request.connection, &config, &|name: &str| {
|
||||
request.connection.secret(name)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
fn get_complete_url(
|
||||
|
|
@ -67,7 +63,9 @@ impl BaseOcrConfig for AzureAiOcrConfig {
|
|||
_optional_params: &Self::OcrParams,
|
||||
_environment: &Self::Environment,
|
||||
) -> Result<String, Error> {
|
||||
self.build_ocr_url(request.connection.api_base.as_deref(), &credential_env)
|
||||
self.build_ocr_url(request.connection.api_base.as_deref(), &|name: &str| {
|
||||
request.connection.secret(name)
|
||||
})
|
||||
}
|
||||
|
||||
fn transform_ocr_request(
|
||||
|
|
@ -107,7 +105,7 @@ impl BaseOcrConfig for AzureAiOcrConfig {
|
|||
}
|
||||
|
||||
fn validate_request_body(&self, body: &Value) -> Result<(), Error> {
|
||||
validate_inline_document(&crate::custom_httpx::llm_http_handler::body_document(body)?)
|
||||
validate_inline_document(&crate::base_llm::ocr::handler::body_document(body)?)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -134,8 +132,7 @@ impl AzureAiOcrConfig {
|
|||
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
|
||||
) -> Result<Vec<(String, String)>, Error> {
|
||||
Self::resolve_api_base(connection.api_base.as_deref(), env_lookup)?;
|
||||
if crate::custom_httpx::http_handler::has_header(&connection.extra_headers, "authorization")
|
||||
{
|
||||
if litellm_http::request::has_header(&connection.extra_headers, "authorization") {
|
||||
if config.azure_ad_token_provider.is_some() {
|
||||
super::common_utils::resolve_entra(config, env_lookup).await?;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,18 +1,14 @@
|
|||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
use data_url::{DataUrl, DataUrlError, forgiving_base64::DecodeError, mime::Mime};
|
||||
use litellm_http::{
|
||||
media::{DownloadPolicy, Error as MediaError, MediaFetcher},
|
||||
transport::Error as TransportError,
|
||||
};
|
||||
use reqwest::Url;
|
||||
|
||||
use crate::{
|
||||
base_llm::ocr::{
|
||||
error::Error,
|
||||
transformation::{
|
||||
OCR_INLINE_MAX_BYTES, OCR_MAX_FETCH_REDIRECTS, OcrConnection, OcrDocument,
|
||||
},
|
||||
},
|
||||
custom_httpx::{
|
||||
media::{DownloadPolicy, Error as MediaError, MediaFetcher},
|
||||
transport::Error as TransportError,
|
||||
},
|
||||
use crate::base_llm::ocr::{
|
||||
error::Error,
|
||||
transformation::{OCR_INLINE_MAX_BYTES, OCR_MAX_FETCH_REDIRECTS, OcrConnection, OcrDocument},
|
||||
};
|
||||
|
||||
pub struct InlineDocument<'a>(DataUrl<'a>);
|
||||
|
|
@ -72,7 +68,7 @@ pub async fn inline_remote_document(
|
|||
url,
|
||||
DownloadPolicy {
|
||||
timeout: connection.timeout,
|
||||
max_bytes: connection.max_download_bytes,
|
||||
max_bytes: connection.settings.max_download_bytes,
|
||||
max_redirects: OCR_MAX_FETCH_REDIRECTS,
|
||||
},
|
||||
)
|
||||
|
|
@ -196,10 +192,8 @@ mod tests {
|
|||
.redirect(reqwest::redirect::Policy::none())
|
||||
.build()
|
||||
.unwrap();
|
||||
let client = crate::custom_httpx::llm_http_handler::OcrClient::for_test(
|
||||
provider_http,
|
||||
document_http,
|
||||
);
|
||||
let client =
|
||||
crate::base_llm::ocr::handler::OcrClient::for_test(provider_http, document_http);
|
||||
let converted = inline_remote_document(
|
||||
client.document_fetcher(),
|
||||
OcrDocument::ImageUrl {
|
||||
|
|
|
|||
|
|
@ -95,11 +95,11 @@ pub enum Error {
|
|||
#[error(transparent)]
|
||||
Auth(#[from] litellm_auth::Error),
|
||||
#[error(transparent)]
|
||||
Transport(#[from] crate::custom_httpx::transport::Error),
|
||||
Transport(#[from] litellm_http::transport::Error),
|
||||
#[error(transparent)]
|
||||
Params(#[from] litellm_core_utils::params::Error),
|
||||
#[error(transparent)]
|
||||
Headers(#[from] crate::custom_httpx::http_handler::HeaderError),
|
||||
Headers(#[from] litellm_http::request::HeaderError),
|
||||
}
|
||||
|
||||
impl From<litellm_host::machine::MachineFault> for Error {
|
||||
|
|
@ -125,9 +125,7 @@ impl Error {
|
|||
pub fn http_status_code(&self) -> Option<u16> {
|
||||
match self {
|
||||
Self::Provider { status, .. }
|
||||
| Self::Transport(crate::custom_httpx::transport::Error::Http { status, .. }) => {
|
||||
Some(*status)
|
||||
}
|
||||
| Self::Transport(litellm_http::transport::Error::Http { status, .. }) => Some(*status),
|
||||
error if error.is_request() => Some(400),
|
||||
_ => None,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,22 +2,21 @@ use bytes::{Bytes, BytesMut};
|
|||
use futures_util::future::BoxFuture;
|
||||
use litellm_auth_gcp::VertexAuth;
|
||||
use litellm_host::event::WireRequest;
|
||||
use litellm_http::{ClientVariant, HttpClientConfig, HttpClientPool};
|
||||
use litellm_http::{
|
||||
ClientVariant, HttpClientConfig, HttpClientPool,
|
||||
media::{MediaFetcher, UrlPolicy},
|
||||
request::{HeaderPolicy, execute_http_request, with_headers},
|
||||
transport,
|
||||
};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
base_llm::ocr::{
|
||||
error::Error,
|
||||
transformation::{
|
||||
BaseOcrConfig, DecodedOcrResponse, LiteLLMOcrResponse, OcrDocument, OcrResponseContext,
|
||||
PreparedOcrRequest, decode_request_value, decode_response,
|
||||
},
|
||||
},
|
||||
custom_httpx::{
|
||||
http_handler::{HeaderPolicy, execute_http_request, with_headers},
|
||||
media::{MediaFetcher, UrlPolicy},
|
||||
transport,
|
||||
use crate::base_llm::ocr::{
|
||||
error::Error,
|
||||
settings::{OcrSettings, Secrets},
|
||||
transformation::{
|
||||
BaseOcrConfig, DecodedOcrResponse, LiteLLMOcrResponse, OcrDocument, OcrResponseContext,
|
||||
PreparedOcrRequest, decode_request_value, decode_response,
|
||||
},
|
||||
};
|
||||
|
||||
|
|
@ -35,6 +34,8 @@ pub struct OcrClient {
|
|||
polling_http: reqwest::Client,
|
||||
document_fetcher: MediaFetcher,
|
||||
vertex_auth: VertexAuth,
|
||||
settings: OcrSettings,
|
||||
secrets: Secrets,
|
||||
}
|
||||
|
||||
impl OcrClient {
|
||||
|
|
@ -43,12 +44,16 @@ impl OcrClient {
|
|||
config: &HttpClientConfig,
|
||||
url_policy: UrlPolicy,
|
||||
vertex_auth: VertexAuth,
|
||||
settings: OcrSettings,
|
||||
secrets: Secrets,
|
||||
) -> Result<Self, litellm_http::Error> {
|
||||
Ok(Self {
|
||||
provider_http: pool.client(config, ClientVariant::Provider)?,
|
||||
polling_http: pool.client(config, ClientVariant::NoRedirect)?,
|
||||
document_fetcher: MediaFetcher::new(pool, config, url_policy)?,
|
||||
vertex_auth,
|
||||
settings,
|
||||
secrets,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -68,6 +73,14 @@ impl OcrClient {
|
|||
&self.vertex_auth
|
||||
}
|
||||
|
||||
pub fn settings(&self) -> &OcrSettings {
|
||||
&self.settings
|
||||
}
|
||||
|
||||
pub fn secrets(&self) -> &Secrets {
|
||||
&self.secrets
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-support"))]
|
||||
pub fn for_test(provider_http: reqwest::Client, document_http: reqwest::Client) -> Self {
|
||||
Self {
|
||||
|
|
@ -78,8 +91,20 @@ impl OcrClient {
|
|||
.expect("test polling client builds"),
|
||||
document_fetcher: MediaFetcher::for_test(document_http),
|
||||
vertex_auth: VertexAuth::default(),
|
||||
settings: OcrSettings::default(),
|
||||
secrets: std::sync::Arc::new(litellm_core_utils::settings::ProcessEnvironment),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-support"))]
|
||||
pub fn with_settings(self, settings: OcrSettings) -> Self {
|
||||
Self { settings, ..self }
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-support"))]
|
||||
pub fn with_secrets(self, secrets: Secrets) -> Self {
|
||||
Self { secrets, ..self }
|
||||
}
|
||||
}
|
||||
|
||||
/// Rust counterpart of `BaseLLMHTTPHandler.async_ocr`: prepare the provider request,
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
pub mod document;
|
||||
pub mod error;
|
||||
pub mod handler;
|
||||
pub mod settings;
|
||||
pub mod transformation;
|
||||
|
|
|
|||
147
litellm-rust/crates/llms/src/base_llm/ocr/settings.rs
Normal file
147
litellm-rust/crates/llms/src/base_llm/ocr/settings.rs
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use litellm_core_utils::settings::Lookup;
|
||||
|
||||
pub type Secrets = Arc<dyn Lookup + Send + Sync>;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct OcrSettings {
|
||||
pub request_timeout: Duration,
|
||||
pub max_download_bytes: u64,
|
||||
pub poll_timeout: Duration,
|
||||
pub document_intelligence_api_version: String,
|
||||
pub document_intelligence_dpi: i64,
|
||||
pub vertex_project: Option<String>,
|
||||
pub vertex_location: Option<String>,
|
||||
pub enable_azure_ad_token_refresh: bool,
|
||||
}
|
||||
|
||||
impl Default for OcrSettings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
request_timeout: Duration::from_secs(6000),
|
||||
max_download_bytes: megabytes(50.0),
|
||||
poll_timeout: Duration::from_secs(120),
|
||||
document_intelligence_api_version: "2024-11-30".into(),
|
||||
document_intelligence_dpi: 96,
|
||||
vertex_project: None,
|
||||
vertex_location: None,
|
||||
enable_azure_ad_token_refresh: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl OcrSettings {
|
||||
pub fn from_environment(env: &impl Lookup) -> Self {
|
||||
let defaults = Self::default();
|
||||
Self {
|
||||
request_timeout: env
|
||||
.parsed::<f64>("REQUEST_TIMEOUT")
|
||||
.and_then(|seconds| Duration::try_from_secs_f64(seconds).ok())
|
||||
.unwrap_or(defaults.request_timeout),
|
||||
max_download_bytes: env
|
||||
.parsed::<f64>("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB")
|
||||
.filter(|size| size.is_finite())
|
||||
.map_or(defaults.max_download_bytes, megabytes),
|
||||
poll_timeout: env
|
||||
.parsed::<i64>("AZURE_OPERATION_POLLING_TIMEOUT")
|
||||
.map_or(defaults.poll_timeout, |seconds| {
|
||||
Duration::from_secs(seconds.max(0).unsigned_abs())
|
||||
}),
|
||||
document_intelligence_api_version: env
|
||||
.get("AZURE_DOCUMENT_INTELLIGENCE_API_VERSION")
|
||||
.unwrap_or(defaults.document_intelligence_api_version),
|
||||
document_intelligence_dpi: env
|
||||
.parsed("AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI")
|
||||
.unwrap_or(defaults.document_intelligence_dpi),
|
||||
..defaults
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn megabytes(size: f64) -> u64 {
|
||||
(size * 1024.0 * 1024.0) as u64
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn env_of(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
|
||||
move |name| {
|
||||
values
|
||||
.iter()
|
||||
.find(|(key, _)| *key == name)
|
||||
.map(|(_, value)| value.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_environment_keeps_the_python_defaults() {
|
||||
assert_eq!(
|
||||
OcrSettings::from_environment(&env_of(&[])),
|
||||
OcrSettings::default()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_setting_follows_its_environment_variable() {
|
||||
let settings = OcrSettings::from_environment(&env_of(&[
|
||||
("REQUEST_TIMEOUT", "30.5"),
|
||||
("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", "0.5"),
|
||||
("AZURE_OPERATION_POLLING_TIMEOUT", " 600 "),
|
||||
("AZURE_DOCUMENT_INTELLIGENCE_API_VERSION", "2025-01-01"),
|
||||
("AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI", "72"),
|
||||
]));
|
||||
assert_eq!(
|
||||
settings,
|
||||
OcrSettings {
|
||||
request_timeout: Duration::from_millis(30_500),
|
||||
max_download_bytes: 512 * 1024,
|
||||
poll_timeout: Duration::from_secs(600),
|
||||
document_intelligence_api_version: "2025-01-01".into(),
|
||||
document_intelligence_dpi: 72,
|
||||
..OcrSettings::default()
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::zero_disables_downloads("0", 0)]
|
||||
#[case::negative_rejects_every_download("-1", 0)]
|
||||
#[case::fraction_truncates_like_int("0.0000001", 0)]
|
||||
#[case::unparsable_keeps_the_default("big", 50 * 1024 * 1024)]
|
||||
fn download_size_converts_megabytes_like_python(
|
||||
#[case] value: &'static str,
|
||||
#[case] bytes: u64,
|
||||
) {
|
||||
let env =
|
||||
move |name: &str| (name == "MAX_IMAGE_URL_DOWNLOAD_SIZE_MB").then(|| value.to_string());
|
||||
assert_eq!(
|
||||
OcrSettings::from_environment(&env).max_download_bytes,
|
||||
bytes
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_negative_polling_timeout_expires_immediately() {
|
||||
let env =
|
||||
|name: &str| (name == "AZURE_OPERATION_POLLING_TIMEOUT").then(|| "-5".to_string());
|
||||
assert_eq!(
|
||||
OcrSettings::from_environment(&env).poll_timeout,
|
||||
Duration::ZERO
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_api_version_is_sent_as_is_like_python_str_of_getenv() {
|
||||
let env =
|
||||
|name: &str| (name == "AZURE_DOCUMENT_INTELLIGENCE_API_VERSION").then(String::new);
|
||||
assert_eq!(
|
||||
OcrSettings::from_environment(&env).document_intelligence_api_version,
|
||||
""
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,9 +1,10 @@
|
|||
use std::{collections::BTreeMap, future::Future, time::Duration};
|
||||
use std::{collections::BTreeMap, future::Future, sync::Arc, time::Duration};
|
||||
|
||||
use litellm_auth::{InputSource, SecretValue, Sourced, TokenProviderHandle};
|
||||
use litellm_core_utils::{
|
||||
call_arguments::CallArguments,
|
||||
serde_compat::{FiniteF64, LaxI64},
|
||||
settings::ProcessEnvironment,
|
||||
};
|
||||
use serde::{
|
||||
Deserialize, Serialize,
|
||||
|
|
@ -12,19 +13,15 @@ use serde::{
|
|||
use serde_json::{Map, Value};
|
||||
use serde_with::serde_as;
|
||||
|
||||
use crate::{
|
||||
base_llm::ocr::error::Error,
|
||||
custom_httpx::llm_http_handler::{
|
||||
CallHooks, OcrClient, read_response_bytes, transform_request_body,
|
||||
},
|
||||
use crate::base_llm::ocr::{
|
||||
error::Error,
|
||||
handler::{CallHooks, OcrClient, read_response_bytes, transform_request_body},
|
||||
settings::{OcrSettings, Secrets},
|
||||
};
|
||||
|
||||
pub const OCR_RESPONSE_MAX_BYTES: usize = 64 * 1024 * 1024;
|
||||
pub const OCR_HTTP_TIMEOUT_SECS: u64 = 600;
|
||||
pub const OCR_INLINE_MAX_BYTES: usize = 50 * 1024 * 1024;
|
||||
pub const OCR_DOWNLOAD_MAX_BYTES: u64 = 50 * 1024 * 1024;
|
||||
pub const OCR_MAX_FETCH_REDIRECTS: usize = 10;
|
||||
pub const OCR_POLL_TIMEOUT_SECS: u64 = 120;
|
||||
pub const OCR_POLL_RETRY_SECS: u64 = 2;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
|
|
@ -116,10 +113,8 @@ impl OcrCredentialInputs {
|
|||
pub struct OcrTransportConfig {
|
||||
pub extra_headers: Vec<(String, String)>,
|
||||
pub extra_headers_source: InputSource,
|
||||
pub timeout: Duration,
|
||||
pub max_download_bytes: u64,
|
||||
pub timeout: Option<Duration>,
|
||||
pub max_response_bytes: usize,
|
||||
pub poll_timeout: Duration,
|
||||
}
|
||||
|
||||
impl Default for OcrTransportConfig {
|
||||
|
|
@ -127,10 +122,8 @@ impl Default for OcrTransportConfig {
|
|||
Self {
|
||||
extra_headers: Vec::new(),
|
||||
extra_headers_source: InputSource::Deployment,
|
||||
timeout: Duration::from_secs(OCR_HTTP_TIMEOUT_SECS),
|
||||
max_download_bytes: OCR_DOWNLOAD_MAX_BYTES,
|
||||
timeout: None,
|
||||
max_response_bytes: OCR_RESPONSE_MAX_BYTES,
|
||||
poll_timeout: Duration::from_secs(OCR_POLL_TIMEOUT_SECS),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -145,7 +138,7 @@ impl OcrTransportConfig {
|
|||
Self {
|
||||
extra_headers,
|
||||
extra_headers_source,
|
||||
timeout: timeout.unwrap_or(self.timeout),
|
||||
timeout: timeout.or(self.timeout),
|
||||
..self
|
||||
}
|
||||
}
|
||||
|
|
@ -166,13 +159,18 @@ pub struct OcrConnection {
|
|||
pub extra_headers: Vec<(String, String)>,
|
||||
pub extra_headers_source: InputSource,
|
||||
pub timeout: Duration,
|
||||
pub max_download_bytes: u64,
|
||||
pub max_response_bytes: usize,
|
||||
pub poll_timeout: Duration,
|
||||
pub settings: OcrSettings,
|
||||
pub secrets: Secrets,
|
||||
}
|
||||
|
||||
impl OcrConnection {
|
||||
pub fn new(credentials: ResolvedOcrCredentials, transport: OcrTransportConfig) -> Self {
|
||||
pub fn new(
|
||||
credentials: ResolvedOcrCredentials,
|
||||
transport: OcrTransportConfig,
|
||||
settings: OcrSettings,
|
||||
secrets: Secrets,
|
||||
) -> Self {
|
||||
let api_key_source = credentials
|
||||
.api_key
|
||||
.as_ref()
|
||||
|
|
@ -190,12 +188,19 @@ impl OcrConnection {
|
|||
api_base_source,
|
||||
extra_headers: transport.extra_headers,
|
||||
extra_headers_source: transport.extra_headers_source,
|
||||
timeout: transport.timeout,
|
||||
max_download_bytes: transport.max_download_bytes,
|
||||
timeout: transport
|
||||
.timeout
|
||||
.filter(|timeout| !timeout.is_zero())
|
||||
.unwrap_or(settings.request_timeout),
|
||||
max_response_bytes: transport.max_response_bytes,
|
||||
poll_timeout: transport.poll_timeout,
|
||||
settings,
|
||||
secrets,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn secret(&self, name: &str) -> Option<String> {
|
||||
self.secrets.get(name)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for OcrConnection {
|
||||
|
|
@ -203,6 +208,8 @@ impl Default for OcrConnection {
|
|||
Self::new(
|
||||
ResolvedOcrCredentials::default(),
|
||||
OcrTransportConfig::default(),
|
||||
OcrSettings::default(),
|
||||
Arc::new(ProcessEnvironment),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -565,16 +572,38 @@ pub fn decode_and_normalize_response<T: DeserializeOwned>(
|
|||
})
|
||||
}
|
||||
|
||||
pub fn credential_env(name: &str) -> Option<String> {
|
||||
std::env::var(name).ok()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn connection_timeout_falls_back_to_the_request_timeout_setting_like_a_python_or() {
|
||||
let settings = OcrSettings {
|
||||
request_timeout: Duration::from_secs(42),
|
||||
..OcrSettings::default()
|
||||
};
|
||||
let timeout = |call: Option<Duration>| {
|
||||
OcrConnection::new(
|
||||
ResolvedOcrCredentials::default(),
|
||||
OcrTransportConfig {
|
||||
timeout: call,
|
||||
..OcrTransportConfig::default()
|
||||
},
|
||||
settings.clone(),
|
||||
Arc::new(ProcessEnvironment),
|
||||
)
|
||||
.timeout
|
||||
};
|
||||
assert_eq!(timeout(None), Duration::from_secs(42));
|
||||
assert_eq!(timeout(Some(Duration::ZERO)), Duration::from_secs(42));
|
||||
assert_eq!(
|
||||
timeout(Some(Duration::from_secs(5))),
|
||||
Duration::from_secs(5)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalized_response_rejects_invalid_shared_fields() {
|
||||
for fields in [
|
||||
|
|
|
|||
|
|
@ -7,17 +7,15 @@ use serde::{Deserialize, Serialize};
|
|||
use serde_json::{Map, Value};
|
||||
use serde_with::serde_as;
|
||||
|
||||
use crate::{
|
||||
base_llm::ocr::{
|
||||
document::InlineDocument,
|
||||
error::Error,
|
||||
transformation::{
|
||||
BaseOcrConfig, LiteLLMOcrResponse, OCR_INLINE_MAX_BYTES, OcrConnection, OcrDocument,
|
||||
OcrPage, OcrPageImage, OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest,
|
||||
credential_env, decode_and_normalize_response, decode_response_value,
|
||||
},
|
||||
use crate::base_llm::ocr::{
|
||||
document::InlineDocument,
|
||||
error::Error,
|
||||
handler::OcrClient,
|
||||
transformation::{
|
||||
BaseOcrConfig, LiteLLMOcrResponse, OCR_INLINE_MAX_BYTES, OcrConnection, OcrDocument,
|
||||
OcrPage, OcrPageImage, OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest,
|
||||
decode_and_normalize_response, decode_response_value,
|
||||
},
|
||||
custom_httpx::llm_http_handler::OcrClient,
|
||||
};
|
||||
|
||||
const COHERE_PARSE_API_BASE: &str = "https://api.cohere.com";
|
||||
|
|
@ -124,7 +122,9 @@ impl BaseOcrConfig for CohereParseConfig {
|
|||
request: &PreparedOcrRequest,
|
||||
_client: &OcrClient,
|
||||
) -> Result<Self::Environment, Error> {
|
||||
self.resolve_headers(&request.connection, &credential_env)
|
||||
self.resolve_headers(&request.connection, &|name: &str| {
|
||||
request.connection.secret(name)
|
||||
})
|
||||
}
|
||||
|
||||
fn get_complete_url(
|
||||
|
|
@ -163,7 +163,7 @@ impl BaseOcrConfig for CohereParseConfig {
|
|||
}
|
||||
|
||||
fn validate_request_body(&self, body: &Value) -> Result<(), Error> {
|
||||
validate_document(&crate::custom_httpx::llm_http_handler::body_document(body)?)
|
||||
validate_document(&crate::base_llm::ocr::handler::body_document(body)?)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -173,8 +173,7 @@ impl CohereParseConfig {
|
|||
connection: &OcrConnection,
|
||||
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
|
||||
) -> Result<Vec<(String, String)>, Error> {
|
||||
if crate::custom_httpx::http_handler::has_header(&connection.extra_headers, "authorization")
|
||||
{
|
||||
if litellm_http::request::has_header(&connection.extra_headers, "authorization") {
|
||||
return Ok(connection.extra_headers.clone());
|
||||
}
|
||||
let key = connection
|
||||
|
|
|
|||
|
|
@ -1,4 +0,0 @@
|
|||
pub mod http_handler;
|
||||
pub mod llm_http_handler;
|
||||
pub mod media;
|
||||
pub mod transport;
|
||||
|
|
@ -3,7 +3,6 @@ pub mod azure_ai;
|
|||
pub mod base_llm;
|
||||
pub mod bedrock;
|
||||
pub mod cohere;
|
||||
pub mod custom_httpx;
|
||||
pub mod mistral;
|
||||
pub mod openai;
|
||||
pub mod reducto;
|
||||
|
|
|
|||
|
|
@ -2,16 +2,13 @@ use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, ur
|
|||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
base_llm::ocr::{
|
||||
error::Error,
|
||||
transformation::{
|
||||
BaseOcrConfig, LiteLLMOcrResponse, OcrConnection, OcrDocument, OcrPage,
|
||||
OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest, credential_env,
|
||||
decode_and_normalize_response,
|
||||
},
|
||||
use crate::base_llm::ocr::{
|
||||
error::Error,
|
||||
handler::OcrClient,
|
||||
transformation::{
|
||||
BaseOcrConfig, LiteLLMOcrResponse, OcrConnection, OcrDocument, OcrPage, OcrResponseFormat,
|
||||
OcrUsageInfo, PreparedOcrRequest, decode_and_normalize_response,
|
||||
},
|
||||
custom_httpx::llm_http_handler::OcrClient,
|
||||
};
|
||||
|
||||
const MISTRAL_OCR_API_BASE: &str = "https://api.mistral.ai/v1";
|
||||
|
|
@ -87,7 +84,9 @@ impl BaseOcrConfig for MistralOcrConfig {
|
|||
request: &PreparedOcrRequest,
|
||||
_client: &OcrClient,
|
||||
) -> Result<Self::Environment, Error> {
|
||||
self.resolve_headers(&request.connection, &credential_env)
|
||||
self.resolve_headers(&request.connection, &|name: &str| {
|
||||
request.connection.secret(name)
|
||||
})
|
||||
}
|
||||
|
||||
fn get_complete_url(
|
||||
|
|
@ -129,8 +128,7 @@ impl MistralOcrConfig {
|
|||
connection: &OcrConnection,
|
||||
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
|
||||
) -> Result<Vec<(String, String)>, Error> {
|
||||
if crate::custom_httpx::http_handler::has_header(&connection.extra_headers, "authorization")
|
||||
{
|
||||
if litellm_http::request::has_header(&connection.extra_headers, "authorization") {
|
||||
return Ok(connection.extra_headers.clone());
|
||||
}
|
||||
let api_key = connection
|
||||
|
|
|
|||
|
|
@ -8,18 +8,14 @@ use litellm_core_utils::{
|
|||
use serde::{Deserialize, Deserializer, Serialize};
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use crate::{
|
||||
base_llm::ocr::{
|
||||
document::InlineDocument,
|
||||
error::Error,
|
||||
transformation::{
|
||||
BaseOcrConfig, LiteLLMOcrResponse, OCR_INLINE_MAX_BYTES, OcrConnection, OcrDocument,
|
||||
OcrPage, OcrRequestContext, OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest,
|
||||
credential_env, decode_and_normalize_response,
|
||||
},
|
||||
},
|
||||
custom_httpx::llm_http_handler::{
|
||||
CallHooks, OcrClient, build_http_request, guardrail_document,
|
||||
use crate::base_llm::ocr::{
|
||||
document::InlineDocument,
|
||||
error::Error,
|
||||
handler::{CallHooks, OcrClient, build_http_request, guardrail_document},
|
||||
transformation::{
|
||||
BaseOcrConfig, LiteLLMOcrResponse, OCR_INLINE_MAX_BYTES, OcrConnection, OcrDocument,
|
||||
OcrPage, OcrRequestContext, OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest,
|
||||
decode_and_normalize_response,
|
||||
},
|
||||
};
|
||||
|
||||
|
|
@ -114,7 +110,9 @@ impl BaseOcrConfig for ReductoParseV3Config {
|
|||
request: &PreparedOcrRequest,
|
||||
_client: &OcrClient,
|
||||
) -> Result<Self::Environment, Error> {
|
||||
resolve_headers(&request.connection, &credential_env)
|
||||
resolve_headers(&request.connection, &|name: &str| {
|
||||
request.connection.secret(name)
|
||||
})
|
||||
}
|
||||
|
||||
fn get_complete_url(
|
||||
|
|
@ -437,7 +435,7 @@ fn resolve_headers(
|
|||
connection: &OcrConnection,
|
||||
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
|
||||
) -> Result<Vec<(String, String)>, Error> {
|
||||
if crate::custom_httpx::http_handler::has_header(&connection.extra_headers, "authorization") {
|
||||
if litellm_http::request::has_header(&connection.extra_headers, "authorization") {
|
||||
return Ok(connection.extra_headers.clone());
|
||||
}
|
||||
let api_key = connection
|
||||
|
|
@ -515,25 +513,21 @@ async fn upload_bytes_async(
|
|||
)?)
|
||||
.multipart(reqwest::multipart::Form::new().part("file", part))
|
||||
.timeout(connection.timeout);
|
||||
let builder = crate::custom_httpx::http_handler::with_headers(
|
||||
let builder = litellm_http::request::with_headers(
|
||||
builder,
|
||||
headers,
|
||||
crate::custom_httpx::http_handler::HeaderPolicy::Except(&[
|
||||
"content-type",
|
||||
"content-length",
|
||||
]),
|
||||
litellm_http::request::HeaderPolicy::Except(&["content-type", "content-length"]),
|
||||
);
|
||||
let response = crate::custom_httpx::http_handler::http_request(builder)
|
||||
let response = litellm_http::request::http_request(builder)
|
||||
.await
|
||||
.map_err(crate::custom_httpx::transport::Error::from)?;
|
||||
let uploaded =
|
||||
crate::custom_httpx::llm_http_handler::read_json_response::<ReductoUploadResponse>(
|
||||
response,
|
||||
false,
|
||||
connection.max_response_bytes,
|
||||
)
|
||||
.await?
|
||||
.data;
|
||||
.map_err(litellm_http::transport::Error::from)?;
|
||||
let uploaded = crate::base_llm::ocr::handler::read_json_response::<ReductoUploadResponse>(
|
||||
response,
|
||||
false,
|
||||
connection.max_response_bytes,
|
||||
)
|
||||
.await?
|
||||
.data;
|
||||
let file_id = uploaded
|
||||
.file_id
|
||||
.as_deref()
|
||||
|
|
|
|||
|
|
@ -1,6 +1,22 @@
|
|||
use litellm_auth::InputSource;
|
||||
use litellm_auth_gcp::VertexConfig;
|
||||
|
||||
use crate::base_llm::ocr::{error::Error, transformation::OcrConnection};
|
||||
use crate::base_llm::ocr::{
|
||||
error::Error,
|
||||
transformation::{OcrConnection, PreparedOcrRequest},
|
||||
};
|
||||
|
||||
pub(super) fn vertex_config(request: &PreparedOcrRequest) -> Result<VertexConfig, Error> {
|
||||
let settings = &request.connection.settings;
|
||||
Ok(VertexConfig::from_sourced_optional_params(
|
||||
&request.optional_params,
|
||||
&request.input_sources,
|
||||
)?
|
||||
.or_configured(
|
||||
settings.vertex_project.as_deref(),
|
||||
settings.vertex_location.as_deref(),
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) fn validate_destination(connection: &OcrConnection) -> Result<(), Error> {
|
||||
if connection.api_base.is_some() && connection.api_base_source == InputSource::Request {
|
||||
|
|
|
|||
|
|
@ -1,19 +1,17 @@
|
|||
use litellm_auth_gcp::{self as vertex, VertexConfig};
|
||||
use litellm_auth_gcp as vertex;
|
||||
use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::transformation::VertexAiOcrConfig;
|
||||
use crate::{
|
||||
base_llm::ocr::{
|
||||
error::Error,
|
||||
transformation::{
|
||||
BaseOcrConfig, LiteLLMOcrResponse, OcrDocument, OcrPage, OcrPageDimensions,
|
||||
OcrPageImage, OcrRequestContext, OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest,
|
||||
credential_env, decode_and_normalize_response, decode_response_value,
|
||||
},
|
||||
use super::{common_utils::vertex_config, transformation::VertexAiOcrConfig};
|
||||
use crate::base_llm::ocr::{
|
||||
error::Error,
|
||||
handler::OcrClient,
|
||||
transformation::{
|
||||
BaseOcrConfig, LiteLLMOcrResponse, OcrDocument, OcrPage, OcrPageDimensions, OcrPageImage,
|
||||
OcrRequestContext, OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest,
|
||||
decode_and_normalize_response, decode_response_value,
|
||||
},
|
||||
custom_httpx::llm_http_handler::OcrClient,
|
||||
};
|
||||
|
||||
const DEFAULT_API_BASE: &str = "https://aiplatform.googleapis.com";
|
||||
|
|
@ -124,12 +122,10 @@ impl BaseOcrConfig for VertexAIDeepSeekOCRConfig {
|
|||
_params: &Self::OcrParams,
|
||||
environment: &Self::Environment,
|
||||
) -> Result<String, Error> {
|
||||
let config = VertexConfig::from_sourced_optional_params(
|
||||
&request.optional_params,
|
||||
&request.input_sources,
|
||||
)?;
|
||||
let location = vertex::get_vertex_ai_location(&config, &credential_env)
|
||||
.unwrap_or_else(|| DEFAULT_LOCATION.to_string());
|
||||
let config = vertex_config(request)?;
|
||||
let location =
|
||||
vertex::get_vertex_ai_location(&config, &|name: &str| request.connection.secret(name))
|
||||
.unwrap_or_else(|| DEFAULT_LOCATION.to_string());
|
||||
self.get_complete_url(
|
||||
request.connection.api_base.as_deref(),
|
||||
&environment.project_id,
|
||||
|
|
|
|||
|
|
@ -2,17 +2,17 @@ use litellm_auth_gcp::{self as vertex, VertexConfig};
|
|||
use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::common_utils::validate_destination;
|
||||
use super::common_utils::{validate_destination, vertex_config};
|
||||
use crate::{
|
||||
base_llm::ocr::{
|
||||
document::{inline_remote_document, validate_inline_document},
|
||||
error::Error,
|
||||
handler::OcrClient,
|
||||
transformation::{
|
||||
BaseOcrConfig, LiteLLMOcrResponse, OcrConnection, OcrDocument, OcrEnvironment,
|
||||
OcrRequestContext, OcrResponseFormat, PreparedOcrRequest, credential_env,
|
||||
OcrRequestContext, OcrResponseFormat, PreparedOcrRequest,
|
||||
},
|
||||
},
|
||||
custom_httpx::llm_http_handler::OcrClient,
|
||||
mistral::ocr::transformation::{MistralOcrConfig, MistralOcrRequest},
|
||||
};
|
||||
|
||||
|
|
@ -47,10 +47,7 @@ impl BaseOcrConfig for VertexAiOcrConfig {
|
|||
request: &PreparedOcrRequest,
|
||||
client: &OcrClient,
|
||||
) -> Result<Self::Environment, Error> {
|
||||
let config = VertexConfig::from_sourced_optional_params(
|
||||
&request.optional_params,
|
||||
&request.input_sources,
|
||||
)?;
|
||||
let config = vertex_config(request)?;
|
||||
self.resolve_environment(&request.connection, &config, client)
|
||||
.await
|
||||
}
|
||||
|
|
@ -61,12 +58,10 @@ impl BaseOcrConfig for VertexAiOcrConfig {
|
|||
_optional_params: &Self::OcrParams,
|
||||
environment: &Self::Environment,
|
||||
) -> Result<String, Error> {
|
||||
let config = VertexConfig::from_sourced_optional_params(
|
||||
&request.optional_params,
|
||||
&request.input_sources,
|
||||
)?;
|
||||
let location = vertex::get_vertex_ai_location(&config, &credential_env)
|
||||
.unwrap_or_else(|| DEFAULT_LOCATION.to_string());
|
||||
let config = vertex_config(request)?;
|
||||
let location =
|
||||
vertex::get_vertex_ai_location(&config, &|name: &str| request.connection.secret(name))
|
||||
.unwrap_or_else(|| DEFAULT_LOCATION.to_string());
|
||||
self.build_ocr_url(
|
||||
request.connection.api_base.as_deref(),
|
||||
&environment.project_id,
|
||||
|
|
@ -112,7 +107,7 @@ impl BaseOcrConfig for VertexAiOcrConfig {
|
|||
}
|
||||
|
||||
fn validate_request_body(&self, body: &Value) -> Result<(), Error> {
|
||||
validate_inline_document(&crate::custom_httpx::llm_http_handler::body_document(body)?)
|
||||
validate_inline_document(&crate::base_llm::ocr::handler::body_document(body)?)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -139,7 +134,7 @@ impl VertexAiOcrConfig {
|
|||
.as_ref()
|
||||
.map(litellm_auth::SecretValue::expose),
|
||||
config,
|
||||
&credential_env,
|
||||
&|name: &str| connection.secret(name),
|
||||
)
|
||||
.await
|
||||
.map_err(Error::from)
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ bytes.workspace = true
|
|||
litellm-auth.workspace = true
|
||||
litellm-callbacks-legacy.workspace = true
|
||||
litellm-core.workspace = true
|
||||
litellm-core-utils.workspace = true
|
||||
litellm-auth-gcp.workspace = true
|
||||
litellm-http.workspace = true
|
||||
litellm-llms.workspace = true
|
||||
|
|
|
|||
|
|
@ -14,5 +14,13 @@
|
|||
"url_policy": [
|
||||
"user_url_validation",
|
||||
"user_url_allowed_hosts"
|
||||
],
|
||||
"provider_defaults": [
|
||||
"vertex_project",
|
||||
"vertex_location",
|
||||
"enable_azure_ad_token_refresh"
|
||||
],
|
||||
"secret_manager": [
|
||||
"readable"
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
use litellm_core::{Error, audio_transcription, chat_completions, messages, responses};
|
||||
use litellm_llms::{
|
||||
base_llm::ocr::error::Error as OcrError, custom_httpx::transport::Error as TransportError,
|
||||
};
|
||||
use litellm_http::transport::Error as TransportError;
|
||||
use litellm_llms::base_llm::ocr::error::Error as OcrError;
|
||||
use pyo3::{
|
||||
exceptions::{PyRuntimeError, PyValueError},
|
||||
prelude::*,
|
||||
|
|
|
|||
|
|
@ -4,11 +4,12 @@ use std::{
|
|||
sync::{Arc, LazyLock, Mutex, PoisonError},
|
||||
};
|
||||
|
||||
use litellm_core_utils::settings::ProcessEnvironment;
|
||||
use litellm_http::{
|
||||
HttpClientConfig, HttpClientPool, HttpSettings, HttpSettingsLayer, Resolution, SslVerify,
|
||||
Unsupported,
|
||||
media::{PublicDnsResolver, UrlPolicy},
|
||||
};
|
||||
use litellm_llms::custom_httpx::media::{PublicDnsResolver, UrlPolicy};
|
||||
use pyo3::{prelude::*, types::PyDict};
|
||||
|
||||
use crate::{errors::RustBridgeDeclined, python_settings::PythonSettings};
|
||||
|
|
@ -29,7 +30,7 @@ pub(crate) fn call_config(
|
|||
) -> PyResult<HttpClientConfig> {
|
||||
let settings = HttpSettings::from_layers([
|
||||
for_call(call_ssl_verify(kwargs)?, asynchronous),
|
||||
HttpSettingsLayer::from_environment(&|name| std::env::var(name).ok()),
|
||||
HttpSettingsLayer::from_environment(&ProcessEnvironment),
|
||||
configured(&PythonSettings::Http.read(py)?)?,
|
||||
])
|
||||
.without_missing_files(&|path: &Path| path.exists());
|
||||
|
|
@ -232,7 +233,7 @@ user_agent='litellm/9.9.9',
|
|||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let settings = HttpSettings::from_layers([
|
||||
HttpSettingsLayer::from_environment(&|name| {
|
||||
HttpSettingsLayer::from_environment(&|name: &str| {
|
||||
(name == "LITELLM_USER_AGENT").then(|| "operator/1".to_string())
|
||||
}),
|
||||
configured(&python_settings(py, "")).unwrap(),
|
||||
|
|
|
|||
|
|
@ -6,16 +6,25 @@ const MODULE: &str = "litellm.rust_bridge.settings";
|
|||
pub(crate) enum PythonSettings {
|
||||
Http,
|
||||
UrlPolicy,
|
||||
ProviderDefaults,
|
||||
SecretManager,
|
||||
}
|
||||
|
||||
impl PythonSettings {
|
||||
#[cfg(test)]
|
||||
pub(crate) const ALL: [Self; 2] = [Self::Http, Self::UrlPolicy];
|
||||
pub(crate) const ALL: [Self; 4] = [
|
||||
Self::Http,
|
||||
Self::UrlPolicy,
|
||||
Self::ProviderDefaults,
|
||||
Self::SecretManager,
|
||||
];
|
||||
|
||||
pub(crate) fn name(self) -> &'static str {
|
||||
match self {
|
||||
Self::Http => "http_settings",
|
||||
Self::UrlPolicy => "url_policy",
|
||||
Self::ProviderDefaults => "provider_defaults",
|
||||
Self::SecretManager => "secret_manager",
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ use litellm_core::messages::{
|
|||
route::{Messages, MessagesCall, MessagesOp, MessagesOpResult, MessagesOutput},
|
||||
};
|
||||
use litellm_host_python::{InvokeError, RouteHost, from_py, lookup, to_py};
|
||||
use litellm_llms::custom_httpx::transport::Error as TransportError;
|
||||
use litellm_http::transport::Error as TransportError;
|
||||
use pyo3::{
|
||||
exceptions::{PyException, PyValueError},
|
||||
gc::{PyTraverseError, PyVisit},
|
||||
|
|
|
|||
|
|
@ -15,10 +15,9 @@ pub(super) fn to_pyerr(error: Error) -> PyErr {
|
|||
body,
|
||||
headers,
|
||||
} => upstream_error(py, status, body, headers)?,
|
||||
Error::Transport(litellm_llms::custom_httpx::transport::Error::Http {
|
||||
status,
|
||||
body,
|
||||
}) => upstream_error(py, status, body, Vec::new())?,
|
||||
Error::Transport(litellm_http::transport::Error::Http { status, body }) => {
|
||||
upstream_error(py, status, body, Vec::new())?
|
||||
}
|
||||
Error::RequestFormat => {
|
||||
let error = core_error_to_pyerr(Error::RequestFormat.into());
|
||||
error
|
||||
|
|
|
|||
|
|
@ -3,19 +3,23 @@ mod errors;
|
|||
mod host;
|
||||
mod project;
|
||||
|
||||
use std::sync::LazyLock;
|
||||
use std::sync::{Arc, LazyLock};
|
||||
|
||||
use host::OcrRouteHost;
|
||||
use litellm_auth_gcp::VertexAuth;
|
||||
use litellm_callbacks_legacy::{LegacySurface, PublicCall, run_legacy_call};
|
||||
use litellm_core::ocr::route::ocr_machine;
|
||||
use litellm_llms::custom_httpx::llm_http_handler::OcrClient;
|
||||
use litellm_core_utils::settings::ProcessEnvironment;
|
||||
use litellm_llms::base_llm::ocr::{
|
||||
handler::OcrClient,
|
||||
settings::{OcrSettings, Secrets},
|
||||
};
|
||||
use pyo3::{
|
||||
prelude::*,
|
||||
types::{PyDict, PyTuple},
|
||||
};
|
||||
|
||||
use crate::{errors::RustBridgeDeclined, http};
|
||||
use crate::{errors::RustBridgeDeclined, http, python_settings::PythonSettings};
|
||||
|
||||
const SURFACE: LegacySurface = LegacySurface {
|
||||
call_type: "ocr",
|
||||
|
|
@ -37,12 +41,15 @@ fn run_ocr(
|
|||
kwargs: Bound<'_, PyDict>,
|
||||
asynchronous: bool,
|
||||
) -> PyResult<Py<PyAny>> {
|
||||
let secrets = process_environment_secrets(&PythonSettings::SecretManager.read(py)?)?;
|
||||
let config = http::call_config(py, &kwargs, asynchronous)?;
|
||||
let client = OcrClient::new(
|
||||
http::pool(),
|
||||
&config,
|
||||
http::url_policy(py)?,
|
||||
VERTEX_AUTH.clone(),
|
||||
ocr_settings(py)?,
|
||||
secrets,
|
||||
)
|
||||
.map_err(|error| RustBridgeDeclined::new_err(error.to_string()))?;
|
||||
run_legacy_call(
|
||||
|
|
@ -55,6 +62,45 @@ fn run_ocr(
|
|||
)
|
||||
}
|
||||
|
||||
#[derive(FromPyObject)]
|
||||
struct PythonSecretManager {
|
||||
readable: bool,
|
||||
}
|
||||
|
||||
fn process_environment_secrets(secret_manager: &Bound<'_, PyAny>) -> PyResult<Secrets> {
|
||||
let manager: PythonSecretManager = secret_manager.extract()?;
|
||||
if manager.readable {
|
||||
return Err(RustBridgeDeclined::new_err(
|
||||
"a readable secret manager is configured and the Rust route only reads the process environment",
|
||||
));
|
||||
}
|
||||
Ok(Arc::new(ProcessEnvironment))
|
||||
}
|
||||
|
||||
#[derive(FromPyObject)]
|
||||
struct PythonProviderDefaults {
|
||||
vertex_project: Option<String>,
|
||||
vertex_location: Option<String>,
|
||||
enable_azure_ad_token_refresh: Option<bool>,
|
||||
}
|
||||
|
||||
fn ocr_settings(py: Python<'_>) -> PyResult<OcrSettings> {
|
||||
let defaults: PythonProviderDefaults = PythonSettings::ProviderDefaults
|
||||
.read(py)?
|
||||
.extract()
|
||||
.map_err(|error: PyErr| {
|
||||
RustBridgeDeclined::new_err(format!(
|
||||
"litellm provider defaults cannot be used by the Rust route: {error}"
|
||||
))
|
||||
})?;
|
||||
Ok(OcrSettings {
|
||||
vertex_project: defaults.vertex_project,
|
||||
vertex_location: defaults.vertex_location,
|
||||
enable_azure_ad_token_refresh: defaults.enable_azure_ad_token_refresh == Some(true),
|
||||
..OcrSettings::from_environment(&ProcessEnvironment)
|
||||
})
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub(crate) fn ocr(
|
||||
py: Python<'_>,
|
||||
|
|
@ -74,3 +120,47 @@ pub(crate) fn aocr(
|
|||
) -> PyResult<Py<PyAny>> {
|
||||
run_ocr(py, request, args, kwargs, true)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use pyo3::{prelude::*, types::PyDict};
|
||||
|
||||
use super::process_environment_secrets;
|
||||
use crate::errors::RustBridgeDeclined;
|
||||
|
||||
fn secret_manager<'py>(py: Python<'py>, readable: bool) -> Bound<'py, PyAny> {
|
||||
let locals = PyDict::new(py);
|
||||
locals.set_item("readable", readable).unwrap();
|
||||
py.run(
|
||||
c"import types\nmanager = types.SimpleNamespace(readable=readable)",
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
locals.get_item("manager").unwrap().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_readable_secret_manager_sends_the_call_back_to_python() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let declined = process_environment_secrets(&secret_manager(py, true))
|
||||
.err()
|
||||
.expect("the Rust route declines");
|
||||
assert!(declined.is_instance_of::<RustBridgeDeclined>(py));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_a_readable_secret_manager_secrets_are_the_process_environment() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let secrets = process_environment_secrets(&secret_manager(py, false)).unwrap();
|
||||
assert_eq!(
|
||||
secrets.get("LITELLM_RUST_BRIDGE_UNSET_VARIABLE_FOR_TEST"),
|
||||
None
|
||||
);
|
||||
assert_eq!(secrets.get("PATH"), std::env::var("PATH").ok());
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -592,7 +592,7 @@ kwargs = {
|
|||
);
|
||||
assert_eq!(
|
||||
projected.transport.timeout,
|
||||
std::time::Duration::from_secs(5)
|
||||
Some(std::time::Duration::from_secs(5))
|
||||
);
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,12 +24,42 @@ class UrlPolicy:
|
|||
user_url_allowed_hosts: Sequence[str]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProviderDefaults:
|
||||
vertex_project: str | None
|
||||
vertex_location: str | None
|
||||
enable_azure_ad_token_refresh: bool | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SecretManager:
|
||||
readable: bool
|
||||
|
||||
|
||||
def warn(message: str) -> None:
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
verbose_logger.warning("%s", message)
|
||||
|
||||
|
||||
def secret_manager() -> SecretManager:
|
||||
from litellm.secret_managers.main import (
|
||||
_should_read_secret_from_secret_manager, # pyright: ignore[reportPrivateUsage] # canonical resolver is private
|
||||
)
|
||||
|
||||
return SecretManager(readable=_should_read_secret_from_secret_manager())
|
||||
|
||||
|
||||
def provider_defaults() -> ProviderDefaults:
|
||||
import litellm
|
||||
|
||||
return ProviderDefaults(
|
||||
vertex_project=litellm.vertex_project,
|
||||
vertex_location=litellm.vertex_location,
|
||||
enable_azure_ad_token_refresh=litellm.enable_azure_ad_token_refresh,
|
||||
)
|
||||
|
||||
|
||||
def url_policy() -> UrlPolicy:
|
||||
import litellm
|
||||
|
||||
|
|
|
|||
|
|
@ -3,12 +3,16 @@ import logging
|
|||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.custom_secret_manager import CustomSecretManager
|
||||
from litellm.llms.custom_httpx.http_handler import default_user_agent
|
||||
from litellm.rust_bridge import settings
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem
|
||||
|
||||
CONTRACT_PATH: Final = Path(__file__).parents[3] / "litellm-rust/crates/python-bridge/python_settings.json"
|
||||
|
||||
|
|
@ -19,6 +23,8 @@ def test_the_rust_contract_matches_the_returned_fields() -> None:
|
|||
assert contract == {
|
||||
"http_settings": [field.name for field in dataclasses.fields(settings.http_settings())],
|
||||
"url_policy": [field.name for field in dataclasses.fields(settings.url_policy())],
|
||||
"provider_defaults": [field.name for field in dataclasses.fields(settings.provider_defaults())],
|
||||
"secret_manager": [field.name for field in dataclasses.fields(settings.secret_manager())],
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -73,3 +79,59 @@ def test_warn_reaches_the_litellm_logger(caplog: pytest.LogCaptureFixture) -> No
|
|||
settings.warn("ssl_ecdh_curve 'secp521r1' is not supported")
|
||||
|
||||
assert [record.getMessage() for record in caplog.records] == ["ssl_ecdh_curve 'secp521r1' is not supported"]
|
||||
|
||||
|
||||
class _VaultSecrets(CustomSecretManager):
|
||||
def __init__(self, secrets: dict[str, str]) -> None:
|
||||
super().__init__(secret_manager_name="rust_bridge_settings_test")
|
||||
self.secrets = secrets
|
||||
|
||||
async def async_read_secret(
|
||||
self,
|
||||
secret_name: str,
|
||||
optional_params: dict[str, object] | None = None,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
) -> str | None:
|
||||
return self.secrets.get(secret_name)
|
||||
|
||||
def sync_read_secret(
|
||||
self,
|
||||
secret_name: str,
|
||||
optional_params: dict[str, object] | None = None,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
) -> str | None:
|
||||
return self.secrets.get(secret_name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("access_mode", "readable"),
|
||||
[("read_only", True), ("read_and_write", True), ("write_only", False)],
|
||||
)
|
||||
def test_secret_manager_is_readable_only_when_litellm_would_read_secrets_from_it(
|
||||
monkeypatch: pytest.MonkeyPatch, access_mode: str, readable: bool
|
||||
) -> None:
|
||||
monkeypatch.setenv("MISTRAL_API_KEY", "env-key")
|
||||
monkeypatch.setattr(litellm, "secret_manager_client", _VaultSecrets({"MISTRAL_API_KEY": "vault-key"}))
|
||||
monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.CUSTOM)
|
||||
monkeypatch.setattr(litellm, "_key_management_settings", KeyManagementSettings(access_mode=access_mode))
|
||||
|
||||
assert settings.secret_manager() == settings.SecretManager(readable=readable)
|
||||
assert (get_secret_str("MISTRAL_API_KEY") == "vault-key") is readable
|
||||
|
||||
|
||||
def test_secret_manager_is_not_readable_without_a_client(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "secret_manager_client", None)
|
||||
|
||||
assert settings.secret_manager() == settings.SecretManager(readable=False)
|
||||
|
||||
|
||||
def test_provider_defaults_read_the_litellm_globals(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "vertex_project", "configured-project")
|
||||
monkeypatch.setattr(litellm, "vertex_location", "europe-west4")
|
||||
monkeypatch.setattr(litellm, "enable_azure_ad_token_refresh", True)
|
||||
|
||||
assert settings.provider_defaults() == settings.ProviderDefaults(
|
||||
vertex_project="configured-project",
|
||||
vertex_location="europe-west4",
|
||||
enable_azure_ad_token_refresh=True,
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue