From 9e03b5322f7f5e13d2a78a980cd1a6073da34d64 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 9 Sep 2026 15:42:33 -0700 Subject: [PATCH] wip --- litellm-rust/Cargo.lock | 8 +++ litellm-rust/crates/ai-gateway/src/ocr/mod.rs | 14 +++- .../crates/ai-gateway/tests/ocr_lifecycle.rs | 10 ++- litellm-rust/crates/core/Cargo.toml | 1 + litellm-rust/crates/core/src/constants.rs | 2 +- litellm-rust/crates/core/src/ocr/client.rs | 42 +++++------- litellm-rust/crates/core/src/ocr/document.rs | 40 +++++++++++ litellm-rust/crates/core/src/ocr/error.rs | 8 ++- litellm-rust/crates/core/src/ocr/handler.rs | 43 +++++++++--- litellm-rust/crates/core/src/ocr/mod.rs | 12 +++- .../crates/core/src/ocr/transformation.rs | 2 + .../ocr/document_intelligence/polling.rs | 8 ++- .../document_intelligence/transformation.rs | 22 ++----- .../providers/azure_ai/ocr/transformation.rs | 4 +- .../providers/mistral/ocr/transformation.rs | 1 + .../providers/reducto/ocr/transformation.rs | 36 ++++------ .../vertex_ai/ocr/deepseek/transformation.rs | 1 + .../providers/vertex_ai/ocr/transformation.rs | 4 +- .../tests/azure_document_intelligence_ocr.rs | 29 ++++++-- litellm-rust/crates/core/tests/ocr.rs | 45 ++++++++++++- .../crates/core/tests/ocr_document.rs | 66 +++++++++++++++++++ litellm-rust/crates/core/tests/reducto_ocr.rs | 21 ++++-- litellm-rust/crates/python-bridge/Cargo.toml | 1 + litellm-rust/crates/python-bridge/src/lib.rs | 1 + .../crates/python-bridge/src/routes/ocr.rs | 3 +- .../crates/python-bridge/src/transport.rs | 36 ++++++++++ 26 files changed, 356 insertions(+), 104 deletions(-) create mode 100644 litellm-rust/crates/core/src/ocr/document.rs create mode 100644 litellm-rust/crates/core/tests/ocr_document.rs create mode 100644 litellm-rust/crates/python-bridge/src/transport.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 13a097ac181..af04630db1f 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -916,6 +916,12 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +[[package]] +name = "data-url" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be1e0bca6c3637f992fc1cc7cbc52a78c1ef6db076dbf1059c4323d6a2048376" + [[package]] name = "deranged" version = "0.5.8" @@ -1708,6 +1714,7 @@ dependencies = [ "azure_core", "azure_identity", "base64 0.22.1", + "data-url", "moka", "percent-encoding", "rand 0.8.7", @@ -1737,6 +1744,7 @@ dependencies = [ "litellm-python-interop", "pyo3", "pyo3-async-runtimes", + "reqwest 0.12.28", "serde", "serde_json", "tokio", diff --git a/litellm-rust/crates/ai-gateway/src/ocr/mod.rs b/litellm-rust/crates/ai-gateway/src/ocr/mod.rs index afd3fa639cd..b88edd9bb87 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/mod.rs @@ -10,7 +10,7 @@ mod hooks; mod types; pub use types::OcrRequest; -pub async fn ocr(request: OcrRequest<'_>) -> Result { +pub async fn ocr(http_client: &reqwest::Client, request: OcrRequest<'_>) -> Result { let mut core_request = decode_request(OcrWireRequest { model: request.model.into(), document: request.document, @@ -27,7 +27,7 @@ pub async fn ocr(request: OcrRequest<'_>) -> Result { request.guardrails, request.request_metadata, )); - perform_ocr(core_request) + perform_ocr(http_client, core_request) .await .map(|response| response.into_json()) } @@ -38,9 +38,17 @@ mod tests { use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; - use super::{OcrRequest, ocr}; + use super::{OcrRequest, ocr as run_ocr}; use crate::integrations::types::RequestMetadata; + async fn ocr(request: OcrRequest<'_>) -> Result { + let http_client = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("test HTTP client builds"); + run_ocr(&http_client, request).await + } + async fn read_http_request(socket: &mut TcpStream) -> String { let mut request = Vec::new(); let mut buffer = [0_u8; 1024]; diff --git a/litellm-rust/crates/ai-gateway/tests/ocr_lifecycle.rs b/litellm-rust/crates/ai-gateway/tests/ocr_lifecycle.rs index 50d7820427e..9e6acb6f0cb 100644 --- a/litellm-rust/crates/ai-gateway/tests/ocr_lifecycle.rs +++ b/litellm-rust/crates/ai-gateway/tests/ocr_lifecycle.rs @@ -12,7 +12,7 @@ use litellm_ai_gateway::integrations::custom_logger::{ CallbackTiming, CallbackValue, CustomLogger, LogFuture, ModelCallDetails, }; use litellm_ai_gateway::integrations::types::RequestMetadata; -use litellm_ai_gateway::ocr::{OcrRequest, ocr}; +use litellm_ai_gateway::ocr::{OcrRequest, ocr as run_ocr}; use litellm_core::error::Error; #[cfg(feature = "trace-parity")] @@ -20,6 +20,14 @@ use litellm_core::observability::FunctionTrace; #[cfg(feature = "trace-parity")] use tracing::instrument::WithSubscriber; +async fn ocr(request: OcrRequest<'_>) -> Result { + let http_client = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("test HTTP client builds"); + run_ocr(&http_client, request).await +} + async fn read_http_headers(socket: &mut TcpStream) -> String { let mut request = Vec::new(); let mut buffer = [0_u8; 1024]; diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index 21831d601c4..193b0d3310a 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -11,6 +11,7 @@ name = "workspace_crate_allowlist" path = "tests/workspace_crate_allowlist.rs" [dependencies] +data-url = "0.3.2" base64.workspace = true azure_core.workspace = true azure_identity.workspace = true diff --git a/litellm-rust/crates/core/src/constants.rs b/litellm-rust/crates/core/src/constants.rs index 8ea0bbbf253..a396a2b875d 100644 --- a/litellm-rust/crates/core/src/constants.rs +++ b/litellm-rust/crates/core/src/constants.rs @@ -45,9 +45,9 @@ pub const EMPTY_TEXT_PLACEHOLDER: &str = pub const FUNCTION_TRACE_TARGET: &str = "litellm::function_trace"; pub(crate) const OCR_HTTP_TIMEOUT_SECS: u64 = 600; -pub(crate) const OCR_CONNECT_TIMEOUT_SECS: u64 = 10; pub(crate) const OCR_POLL_TIMEOUT_SECS: u64 = 120; pub(crate) const OCR_POLL_RETRY_SECS: u64 = 2; +pub(crate) const OCR_INLINE_MAX_BYTES: usize = 50 * 1024 * 1024; pub(crate) const OCR_DOWNLOAD_MAX_BYTES: u64 = 50 * 1024 * 1024; pub(crate) const OCR_MAX_FETCH_REDIRECTS: usize = 10; pub(crate) const OCR_ERROR_BODY_MAX_CHARS: usize = 256; diff --git a/litellm-rust/crates/core/src/ocr/client.rs b/litellm-rust/crates/core/src/ocr/client.rs index e18de8814f1..4e0acf9bee3 100644 --- a/litellm-rust/crates/core/src/ocr/client.rs +++ b/litellm-rust/crates/core/src/ocr/client.rs @@ -2,33 +2,14 @@ use crate::ocr::error::OcrError; use crate::ocr::error::OcrRequestError; use crate::ocr::error::OcrResponseError; use std::net::IpAddr; -use std::sync::OnceLock; use std::time::Duration; use super::types::{OcrConnection, OcrDocument}; -use crate::constants::{ - OCR_CONNECT_TIMEOUT_SECS, OCR_ERROR_BODY_MAX_CHARS, OCR_HTTP_TIMEOUT_SECS, - OCR_MAX_FETCH_REDIRECTS, -}; +use crate::constants::{OCR_ERROR_BODY_MAX_CHARS, OCR_MAX_FETCH_REDIRECTS}; use crate::error::TransportError; use base64::{Engine, engine::general_purpose::STANDARD}; use reqwest::Url; -pub(crate) fn http_client() -> Result<&'static reqwest::Client, TransportError> { - static CLIENT: OnceLock> = OnceLock::new(); - CLIENT - .get_or_init(|| { - reqwest::Client::builder() - .connect_timeout(Duration::from_secs(OCR_CONNECT_TIMEOUT_SECS)) - .timeout(Duration::from_secs(OCR_HTTP_TIMEOUT_SECS)) - .redirect(reqwest::redirect::Policy::none()) - .build() - .map_err(|error| error.to_string()) - }) - .as_ref() - .map_err(|error| TransportError::Network(error.clone())) -} - pub(crate) fn network_error(error: reqwest::Error) -> TransportError { TransportError::from(error) } @@ -97,6 +78,7 @@ async fn validate_safe_fetch_url(url: &Url) -> Result<(), OcrError> { } async fn safe_get_document_url( + http_client: &reqwest::Client, source: &str, timeout: Duration, ) -> Result { @@ -105,7 +87,7 @@ async fn safe_get_document_url( })?; for _ in 0..OCR_MAX_FETCH_REDIRECTS { validate_safe_fetch_url(&url).await?; - let response = http_client()? + let response = http_client .get(url.clone()) .timeout(timeout) .send() @@ -136,9 +118,13 @@ fn enforce_download_size(length: u64, max_bytes: u64) -> Result<(), OcrError> { Ok(()) } -async fn download_data_uri(source: &str, connection: &OcrConnection) -> Result { +async fn download_data_uri( + http_client: &reqwest::Client, + source: &str, + connection: &OcrConnection, +) -> Result { enforce_download_size(0, connection.max_download_bytes)?; - let mut response = safe_get_document_url(source, connection.timeout).await?; + let mut response = safe_get_document_url(http_client, source, connection.timeout).await?; if !response.status().is_success() { let status = response.status().as_u16(); return Err(TransportError::Http { @@ -172,6 +158,7 @@ async fn download_data_uri(source: &str, connection: &OcrConnection) -> Result Result { @@ -179,8 +166,11 @@ pub(crate) async fn convert_document_url_to_data_uri( if !source.starts_with("http://") && !source.starts_with("https://") { return Ok(document); } - let data_uri = tokio::time::timeout(connection.timeout, download_data_uri(source, connection)) - .await - .map_err(|_| TransportError::Network("OCR document download timed out".into()))??; + let data_uri = tokio::time::timeout( + connection.timeout, + download_data_uri(http_client, source, connection), + ) + .await + .map_err(|_| TransportError::Network("OCR document download timed out".into()))??; Ok(document.with_source(data_uri)) } diff --git a/litellm-rust/crates/core/src/ocr/document.rs b/litellm-rust/crates/core/src/ocr/document.rs new file mode 100644 index 00000000000..be7d69e4994 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/document.rs @@ -0,0 +1,40 @@ +use data_url::{DataUrl, DataUrlError, forgiving_base64::DecodeError, mime::Mime}; + +use super::error::OcrRequestError; + +pub(crate) struct InlineDocument<'a>(DataUrl<'a>); + +impl<'a> InlineDocument<'a> { + pub(crate) fn parse(source: &'a str) -> Result, OcrRequestError> { + match DataUrl::process(source) { + Ok(url) => Ok(Some(Self(url))), + Err(DataUrlError::NotADataUrl) => Ok(None), + Err(DataUrlError::NoComma) => Err(OcrRequestError::InvalidDataUri), + } + } + + pub(crate) fn mime_type(&self) -> &Mime { + self.0.mime_type() + } + + pub(crate) fn decode(&self, max_bytes: usize) -> Result, OcrRequestError> { + let mut body = Vec::new(); + self.0 + .decode(|bytes| { + if bytes.len() > max_bytes.saturating_sub(body.len()) { + return Err(OcrRequestError::InlineDocumentTooLarge); + } + body.extend_from_slice(bytes); + Ok(()) + }) + .map_err(|error| match error { + DecodeError::InvalidBase64(_) => OcrRequestError::InvalidDataUri, + DecodeError::WriteError(error) => error, + })?; + Ok(body) + } +} + +#[cfg(test)] +#[path = "../../tests/ocr_document.rs"] +mod tests; diff --git a/litellm-rust/crates/core/src/ocr/error.rs b/litellm-rust/crates/core/src/ocr/error.rs index cd404fda472..23eb970bd98 100644 --- a/litellm-rust/crates/core/src/ocr/error.rs +++ b/litellm-rust/crates/core/src/ocr/error.rs @@ -36,11 +36,13 @@ pub enum OcrRequestError { #[error("model_id cannot be a dot path segment")] DotModel, #[error( - "Reducto requires a reducto:// id or a base64 data URI after OCR preprocessing; plain http(s) URLs are not supported" + "Reducto requires a reducto:// id or a data URI after OCR preprocessing; plain http(s) URLs are not supported" )] ReductoSource, - #[error("Invalid Reducto base64 data URI")] - ReductoDataUri, + #[error("Invalid OCR document data URI")] + InvalidDataUri, + #[error("OCR inline document exceeds maximum allowed size")] + InlineDocumentTooLarge, #[error("OCR document URL rejected by SSRF protection")] BlockedDocumentUrl, #[error("Too many redirects while fetching OCR document URL")] diff --git a/litellm-rust/crates/core/src/ocr/handler.rs b/litellm-rust/crates/core/src/ocr/handler.rs index 40c8f1d71b5..97d5ad14e7d 100644 --- a/litellm-rust/crates/core/src/ocr/handler.rs +++ b/litellm-rust/crates/core/src/ocr/handler.rs @@ -15,7 +15,16 @@ use crate::providers::reducto::ocr::transformation::{ use crate::providers::vertex_ai::ocr::deepseek::transformation::VertexAiDeepSeekOcrConfig; use crate::providers::vertex_ai::ocr::transformation::VertexAiOcrConfig; -pub(crate) async fn perform_ocr_request(request: OcrRequest) -> Result { +struct OcrExecution<'a> { + http_client: &'a reqwest::Client, + lifecycle_context: CallLifecycleContext, + hooks: Arc, +} + +pub(crate) async fn perform_ocr_request( + http_client: &reqwest::Client, + request: OcrRequest, +) -> Result { let context = CallLifecycleContext::new( "ocr", request.model.clone(), @@ -32,8 +41,11 @@ pub(crate) async fn perform_ocr_request(request: OcrRequest) -> Result( document: OcrDocument, params: C::InputParams, connection: OcrConnection, - context: CallLifecycleContext, - hooks: Arc, + execution: OcrExecution<'_>, ) -> Result { + let OcrExecution { + http_client, + lifecycle_context, + hooks, + } = execution; let request = prepare_ocr_call(config, model, document, params, connection); let lifecycle_hooks = OcrLifecycleHooks { hooks: hooks.clone(), - provider_name: context.custom_llm_provider.clone(), + provider_name: lifecycle_context.custom_llm_provider.clone(), marker: std::marker::PhantomData, }; CallLifecycle::default() - .run(context, request, &lifecycle_hooks, |request| { - execute_ocr_provider_call(request, hooks.as_ref(), &lifecycle_hooks.provider_name) + .run(lifecycle_context, request, &lifecycle_hooks, |request| { + execute_ocr_provider_call( + http_client, + request, + hooks.as_ref(), + &lifecycle_hooks.provider_name, + ) }) .await } #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] async fn execute_ocr_provider_call( + http_client: &reqwest::Client, request: PreparedOcrRequest, hooks: &dyn OcrHooks, provider_name: &str, @@ -109,7 +131,7 @@ async fn execute_ocr_provider_call( request.document }; let document = config - .prepare_document(document, &request.connection, &headers) + .prepare_document(http_client, document, &request.connection, &headers) .await?; let body = serde_json::to_value(config.transform_ocr_request( &request.model, @@ -130,7 +152,7 @@ async fn execute_ocr_provider_call( } else { body }; - let mut builder = super::client::http_client()? + let mut builder = http_client .post(&url) .json(&body) .timeout(request.connection.timeout); @@ -142,6 +164,7 @@ async fn execute_ocr_provider_call( .map_err(super::client::network_error)?; let decoded = config .read_response( + http_client, response, &url, &headers, diff --git a/litellm-rust/crates/core/src/ocr/mod.rs b/litellm-rust/crates/core/src/ocr/mod.rs index 475b961db52..5f306bce2ce 100644 --- a/litellm-rust/crates/core/src/ocr/mod.rs +++ b/litellm-rust/crates/core/src/ocr/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod client; +pub(crate) mod document; pub mod error; mod handler; pub mod hooks; @@ -9,14 +10,21 @@ pub mod wire; pub use types::{OcrRequest, OcrResponseData}; +/// Execute an OCR request with a host-owned HTTP client. +/// +/// The client must not follow redirects automatically. OCR document downloads +/// validate each redirect target before issuing the next request. #[tracing::instrument( name = "ocr", target = "litellm::function_trace", level = "trace", skip_all )] -pub async fn perform_ocr(request: OcrRequest) -> Result { - handler::perform_ocr_request(request).await +pub async fn perform_ocr( + http_client: &reqwest::Client, + request: OcrRequest, +) -> Result { + handler::perform_ocr_request(http_client, request).await } #[cfg(test)] diff --git a/litellm-rust/crates/core/src/ocr/transformation.rs b/litellm-rust/crates/core/src/ocr/transformation.rs index 43398fb6b66..d87919bc65e 100644 --- a/litellm-rust/crates/core/src/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/ocr/transformation.rs @@ -44,6 +44,7 @@ pub trait OcrProviderConfig: Send + Sync + Sized + 'static { ) -> impl Future, AuthError>> + Send; fn prepare_document( &self, + http_client: &reqwest::Client, document: OcrDocument, connection: &OcrConnection, headers: &[(String, String)], @@ -57,6 +58,7 @@ pub trait OcrProviderConfig: Send + Sync + Sized + 'static { fn read_response( &self, + _http_client: &reqwest::Client, response: reqwest::Response, _url: &str, _headers: &[(String, String)], diff --git a/litellm-rust/crates/core/src/providers/azure_ai/ocr/document_intelligence/polling.rs b/litellm-rust/crates/core/src/providers/azure_ai/ocr/document_intelligence/polling.rs index 15a1ead8b23..b0def7ba1dc 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/ocr/document_intelligence/polling.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/ocr/document_intelligence/polling.rs @@ -5,13 +5,14 @@ use std::time::Duration; use super::types::{AzureDocumentIntelligenceOperation, OperationStatus}; use crate::constants::{AZURE_DI_SUBSCRIPTION_HEADER, OCR_POLL_RETRY_SECS}; -use crate::ocr::client::{http_client, network_error}; +use crate::ocr::client::network_error; use crate::ocr::types::OcrConnection; use crate::ocr::wire::{DecodedOcrResponse, read_json_response}; use reqwest::Url; use tokio::time::Instant; pub(crate) async fn read_operation_response( + http_client: &reqwest::Client, response: reqwest::Response, original_url: &str, headers: &[(String, String)], @@ -34,11 +35,12 @@ pub(crate) async fn read_operation_response( { return Err(OcrPollingError::PollOrigin.into()); } - poll_document_intelligence(operation, headers, connection, native).await + poll_document_intelligence(http_client, operation, headers, connection, native).await } #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] async fn poll_document_intelligence( + http_client: &reqwest::Client, url: Url, headers: &[(String, String)], connection: &OcrConnection, @@ -52,7 +54,7 @@ async fn poll_document_intelligence( .checked_duration_since(Instant::now()) .filter(|remaining| !remaining.is_zero()) .ok_or(OcrPollingError::PollTimeout)?; - let mut builder = http_client()? + let mut builder = http_client .get(url.clone()) .timeout(remaining.min(connection.timeout)); for (name, value) in headers { diff --git a/litellm-rust/crates/core/src/providers/azure_ai/ocr/document_intelligence/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/ocr/document_intelligence/transformation.rs index 0ddc731a965..b6bbf79aba0 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/ocr/document_intelligence/transformation.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/ocr/document_intelligence/transformation.rs @@ -1,8 +1,10 @@ use crate::auth::AuthError; +use crate::ocr::document::InlineDocument; use crate::ocr::error::OcrError; use crate::ocr::error::OcrRequestError; use crate::ocr::error::OcrResponseError; use crate::ocr::error::PagesError; +use base64::{Engine, engine::general_purpose::STANDARD}; use std::collections::BTreeSet; use super::types::*; @@ -99,19 +101,6 @@ fn normalize_features_param( Ok(Some(NormalizedFeatures(normalized.join(",")))) } -fn extract_base64_from_data_uri(source: &str) -> &str { - let Some((header, data)) = source.split_once(',') else { - return source; - }; - let media_type = header.strip_prefix("data:").unwrap_or_default(); - let media_type = media_type.strip_suffix(";base64").unwrap_or(media_type); - if media_type.is_empty() || media_type.contains(';') || data.is_empty() { - source - } else { - data - } -} - fn pixel_dimension(value: f64, scale: f64, field: &'static str) -> Result { let pixels = (value * scale).trunc(); if !pixels.is_finite() || pixels < i64::MIN as f64 || pixels >= -(i64::MIN as f64) { @@ -218,6 +207,7 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { async fn prepare_document( &self, + _http_client: &reqwest::Client, document: OcrDocument, _connection: &OcrConnection, _headers: &[(String, String)], @@ -236,9 +226,9 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { if source.is_empty() { return Err(OcrRequestError::MissingField("document URL")); } - Ok(if source.starts_with("data:") { + Ok(if let Some(document) = InlineDocument::parse(source)? { DocumentIntelligenceRequest::Base64Source( - extract_base64_from_data_uri(source).to_string(), + STANDARD.encode(document.decode(crate::constants::OCR_INLINE_MAX_BYTES)?), ) } else { DocumentIntelligenceRequest::UrlSource(source.to_string()) @@ -284,6 +274,7 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { async fn read_response( &self, + http_client: &reqwest::Client, response: reqwest::Response, url: &str, headers: &[(String, String)], @@ -291,6 +282,7 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { params: &Self::MappedParams, ) -> Result, OcrError> { super::polling::read_operation_response( + http_client, response, url, headers, diff --git a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs index ddeabc32e1f..41de1269d21 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs @@ -61,11 +61,13 @@ impl OcrProviderConfig for AzureAiOcrConfig { async fn prepare_document( &self, + http_client: &reqwest::Client, document: OcrDocument, connection: &OcrConnection, _headers: &[(String, String)], ) -> Result { - crate::ocr::client::convert_document_url_to_data_uri(document, connection).await + crate::ocr::client::convert_document_url_to_data_uri(http_client, document, connection) + .await } async fn authenticate( diff --git a/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs index d04ec493dac..fa2a95b1cfd 100644 --- a/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs @@ -82,6 +82,7 @@ impl OcrProviderConfig for MistralOcrConfig { async fn prepare_document( &self, + _http_client: &reqwest::Client, document: OcrDocument, _connection: &OcrConnection, _headers: &[(String, String)], diff --git a/litellm-rust/crates/core/src/providers/reducto/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/reducto/ocr/transformation.rs index 3501c2cd2f0..7a06769aa34 100644 --- a/litellm-rust/crates/core/src/providers/reducto/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/reducto/ocr/transformation.rs @@ -6,11 +6,10 @@ use std::collections::BTreeMap; use super::types::*; use crate::constants::{REDUCTO_ID_PREFIX, REDUCTO_OCR_API_BASE}; +use crate::ocr::document::InlineDocument; use crate::ocr::transformation::OcrProviderConfig; use crate::ocr::types::{OcrConnection, OcrDocument, OcrPage, OcrResponseData, OcrUsageInfo}; use crate::providers::reducto::auth; -use base64::Engine; -use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; pub struct ReductoParseV3Config; pub struct ReductoParseLegacyConfig; @@ -26,6 +25,7 @@ pub fn normalize_api_base(api_base: Option<&str>) -> &str { } async fn prepare_reducto_document( + http_client: &reqwest::Client, document: OcrDocument, connection: &OcrConnection, headers: &[(String, String)], @@ -34,28 +34,14 @@ async fn prepare_reducto_document( if source.starts_with(REDUCTO_ID_PREFIX) { return Ok(ReductoFileId(source.to_string())); } - if !source.starts_with("data:") { - return Err(OcrRequestError::ReductoSource.into()); - } - let (header, encoded) = source - .split_once(',') - .ok_or(OcrRequestError::ReductoDataUri)?; - if !header.split(';').any(|part| part == "base64") { - return Err(OcrRequestError::ReductoDataUri.into()); - } - let mime = header - .strip_prefix("data:") - .and_then(|header| header.split(';').next()) - .filter(|mime| !mime.is_empty()) - .unwrap_or("application/octet-stream"); - let bytes = BASE64_STANDARD - .decode(encoded) - .map_err(|_| OcrRequestError::ReductoDataUri)?; + let document = InlineDocument::parse(source)?.ok_or(OcrRequestError::ReductoSource)?; + let mime = document.mime_type().to_string(); + let bytes = document.decode(crate::constants::OCR_INLINE_MAX_BYTES)?; let part = reqwest::multipart::Part::bytes(bytes) .file_name("document") - .mime_str(mime) - .map_err(|_| OcrRequestError::ReductoDataUri)?; - let mut builder = crate::ocr::client::http_client()? + .mime_str(&mime) + .map_err(|_| OcrRequestError::InvalidDataUri)?; + let mut builder = http_client .post(format!( "{}/upload", normalize_api_base(connection.api_base.as_deref()) @@ -203,11 +189,12 @@ impl OcrProviderConfig for ReductoParseV3Config { async fn prepare_document( &self, + http_client: &reqwest::Client, document: OcrDocument, connection: &OcrConnection, headers: &[(String, String)], ) -> Result { - prepare_reducto_document(document, connection, headers).await + prepare_reducto_document(http_client, document, connection, headers).await } fn guard_document_before_preparation(&self) -> bool { @@ -282,11 +269,12 @@ impl OcrProviderConfig for ReductoParseLegacyConfig { async fn prepare_document( &self, + http_client: &reqwest::Client, document: OcrDocument, connection: &OcrConnection, headers: &[(String, String)], ) -> Result { - prepare_reducto_document(document, connection, headers).await + prepare_reducto_document(http_client, document, connection, headers).await } fn guard_document_before_preparation(&self) -> bool { diff --git a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/deepseek/transformation.rs b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/deepseek/transformation.rs index 21bc61d8dcb..2edff887c3e 100644 --- a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/deepseek/transformation.rs +++ b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/deepseek/transformation.rs @@ -125,6 +125,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig { async fn prepare_document( &self, + _http_client: &reqwest::Client, document: OcrDocument, _connection: &OcrConnection, _headers: &[(String, String)], diff --git a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs index c1ea3d88ee7..f72301a57cb 100644 --- a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs @@ -102,11 +102,13 @@ impl OcrProviderConfig for VertexAiOcrConfig { async fn prepare_document( &self, + http_client: &reqwest::Client, document: OcrDocument, connection: &OcrConnection, _headers: &[(String, String)], ) -> Result { - crate::ocr::client::convert_document_url_to_data_uri(document, connection).await + crate::ocr::client::convert_document_url_to_data_uri(http_client, document, connection) + .await } async fn authenticate( diff --git a/litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs b/litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs index ef0795c282b..53893b26c11 100644 --- a/litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs +++ b/litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs @@ -1,6 +1,6 @@ use super::transformation::AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG as CONFIG; -use crate::ocr::perform_ocr; use crate::ocr::prepare::OcrProviderKind; +use crate::ocr::tests::perform_ocr; use crate::ocr::tests::{MockResponse, body, mock_server, params, transform, wire_request}; use crate::ocr::transformation::OcrProviderConfig; use crate::ocr::types::OcrConnection; @@ -106,10 +106,10 @@ async fn document_intelligence_mistral_pages_flow_to_query_only() { } #[rstest] #[case("data:application/pdf;base64,YWJj", "YWJj")] -#[case("data:application/pdf,YWJj", "YWJj")] -#[case("data:application/pdf;base64,", "data:application/pdf;base64,")] -#[case("data:application/pdf;other,YWJj", "data:application/pdf;other,YWJj")] -#[case("data:application/pdf;base64", "data:application/pdf;base64")] +#[case("data:application/pdf,abc", "YWJj")] +#[case("data:application/pdf;base64,", "")] +#[case("data:application/pdf;version=1.7;base64,YWJj", "YWJj")] +#[case("DATA:;BASE64,YQ%3D%3D#page", "YQ==")] #[tokio::test] async fn document_intelligence_request_uses_base64_source_for_data_uri( #[case] source: &str, @@ -350,3 +350,22 @@ async fn polling_deadline_bounds_retry_after() { server.await.unwrap(); assert!(error.to_string().contains("timed out")); } + +#[rstest] +#[case("data:application/pdf;base64")] +#[case("data:application/pdf;base64,INVALID!")] +#[tokio::test] +async fn document_intelligence_rejects_invalid_data_uri(#[case] source: &str) { + let error = body( + &CONFIG, + "model", + json!({"type":"document_url","document_url":source}), + json!({}), + ) + .await + .unwrap_err(); + assert!(matches!( + error, + crate::ocr::error::OcrError::Request(crate::ocr::error::OcrRequestError::InvalidDataUri) + )); +} diff --git a/litellm-rust/crates/core/tests/ocr.rs b/litellm-rust/crates/core/tests/ocr.rs index e790193ea26..3a0539bc63f 100644 --- a/litellm-rust/crates/core/tests/ocr.rs +++ b/litellm-rust/crates/core/tests/ocr.rs @@ -6,9 +6,22 @@ use tokio::net::TcpListener; use super::transformation::OcrProviderConfig; use super::types::OcrConnection; use super::wire::{OcrWireRequest, decode_request, decode_response}; -use super::{OcrRequest, perform_ocr}; +use super::{OcrRequest, perform_ocr as run_ocr}; use crate::ocr::error::OcrError; +pub(crate) fn http_client() -> reqwest::Client { + reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("test HTTP client builds") +} + +pub(crate) async fn perform_ocr( + request: OcrRequest, +) -> Result { + run_ocr(&http_client(), request).await +} + pub(crate) fn params(config: &C, value: Value) -> C::MappedParams { config .map_ocr_params(serde_json::from_value(value).unwrap()) @@ -36,8 +49,10 @@ pub(crate) async fn body( options: Value, ) -> Result { let params = config.map_ocr_params(serde_json::from_value(options).unwrap())?; + let http_client = http_client(); let document = config .prepare_document( + &http_client, serde_json::from_value(document).unwrap(), &OcrConnection::default(), &[], @@ -151,6 +166,34 @@ async fn performs_mistral_ocr_from_typed_request() { assert!(body.get("unknown").is_none()); } +#[tokio::test] +async fn uses_the_host_injected_http_client() { + let (base, seen, server) = mock_server(vec![MockResponse::json( + json!({"pages":[],"usage_info":{"pages_processed":0}}), + )]) + .await; + let mut default_headers = reqwest::header::HeaderMap::new(); + default_headers.insert( + "x-transport-owner", + reqwest::header::HeaderValue::from_static("python-sdk"), + ); + let http_client = reqwest::Client::builder() + .default_headers(default_headers) + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("test HTTP client builds"); + + run_ocr( + &http_client, + wire_request("mistral/model", &base, json!({})), + ) + .await + .expect("OCR request succeeds"); + + server.await.expect("mock server completes"); + assert!(seen.lock().unwrap()[0].contains("x-transport-owner: python-sdk")); +} + #[tokio::test] async fn invalid_pages_fail_before_network_and_invoke_failure_hook() { use crate::call_lifecycle::{CallLifecycleContext, CallLifecycleTiming}; diff --git a/litellm-rust/crates/core/tests/ocr_document.rs b/litellm-rust/crates/core/tests/ocr_document.rs new file mode 100644 index 00000000000..70cd5522a6c --- /dev/null +++ b/litellm-rust/crates/core/tests/ocr_document.rs @@ -0,0 +1,66 @@ +use super::InlineDocument; +use crate::ocr::error::OcrRequestError; +use rstest::rstest; + +#[rstest] +#[case("data:application/pdf;base64,YWJj", b"abc")] +#[case("DATA:application/pdf;BASE64,YWI", b"ab")] +#[case("data:application/pdf;base64,YQ%3D%3D#fragment", b"a")] +#[case("data:application/pdf;base64,Y W J j", b"abc")] +#[case("data:,a%20b%00%FF", b"a b\0\xff")] +#[case("data:;base64,", b"")] +fn decodes_data_urls(#[case] source: &str, #[case] expected: &[u8]) { + let document = InlineDocument::parse(source).unwrap().unwrap(); + assert_eq!(document.decode(expected.len()).unwrap(), expected); + if !expected.is_empty() { + assert_eq!( + document.decode(expected.len() - 1), + Err(OcrRequestError::InlineDocumentTooLarge) + ); + } +} + +#[test] +fn preserves_mime_parameters_and_uses_standard_default() { + let document = InlineDocument::parse("data:application/pdf;version=1.7;base64,YQ==") + .unwrap() + .unwrap(); + assert!(document.mime_type().matches("application", "pdf")); + assert_eq!(document.mime_type().get_parameter("version"), Some("1.7")); + let default = InlineDocument::parse("data:,a").unwrap().unwrap(); + assert!(default.mime_type().matches("text", "plain")); + assert_eq!( + default.mime_type().get_parameter("charset"), + Some("US-ASCII") + ); +} + +#[rstest] +#[case("data:application/pdf;base64")] +#[case("data:application/pdf;base64,INVALID!")] +#[case("data:application/pdf;base64,Y")] +fn rejects_invalid_input(#[case] source: &str) { + let result = InlineDocument::parse(source).and_then(|document| document.unwrap().decode(100)); + assert_eq!(result, Err(OcrRequestError::InvalidDataUri)); +} + +#[test] +fn leaves_other_sources_to_the_provider() { + for source in ["https://example.com/file.pdf", "reducto://file.pdf"] { + assert!(InlineDocument::parse(source).unwrap().is_none()); + } +} + +#[test] +fn limits_total_decoded_bytes_across_chunks() { + use base64::{Engine, engine::general_purpose::STANDARD}; + + let bytes = vec![0xff; 8193]; + let source = format!("data:application/pdf;base64,{}", STANDARD.encode(&bytes)); + let document = InlineDocument::parse(&source).unwrap().unwrap(); + assert_eq!(document.decode(bytes.len()).unwrap(), bytes); + assert_eq!( + document.decode(bytes.len() - 1), + Err(OcrRequestError::InlineDocumentTooLarge) + ); +} diff --git a/litellm-rust/crates/core/tests/reducto_ocr.rs b/litellm-rust/crates/core/tests/reducto_ocr.rs index 49ef5e80801..25ddc3760b0 100644 --- a/litellm-rust/crates/core/tests/reducto_ocr.rs +++ b/litellm-rust/crates/core/tests/reducto_ocr.rs @@ -1,6 +1,5 @@ use super::transformation::{REDUCTO_PARSE_LEGACY_CONFIG as LEGACY, REDUCTO_PARSE_V3_CONFIG as V3}; -use crate::ocr::perform_ocr; -use crate::ocr::tests::{MockResponse, body, mock_server, transform, wire_request}; +use crate::ocr::tests::{MockResponse, body, mock_server, perform_ocr, transform, wire_request}; use rstest::rstest; use serde_json::json; @@ -36,7 +35,7 @@ async fn test_parse_legacy_wraps_enhance_under_options() { #[rstest] #[case("http://example.com/a.pdf")] #[case("https://example.com/a.pdf")] -#[case("data:application/pdf,YWJj")] +#[case("data:application/pdf;base64")] #[case("data:application/pdf;base64,INVALID!")] #[tokio::test] async fn test_parse_v3_rejects_plain_http_urls(#[case] source: &str) { @@ -86,7 +85,15 @@ fn reducto_missing_result_and_null_result_are_distinct() { #[case("parse-v3")] #[case("parse-legacy")] #[tokio::test] -async fn test_parse_v3_file_upload_and_response_mapping(#[case] model: &str) { +async fn test_parse_v3_file_upload_and_response_mapping( + #[case] model: &str, + #[values( + "data:application/pdf;base64,YWJj", + "data:application/pdf,abc", + "DATA:application/pdf;BASE64,YWJj#page" + )] + source: &str, +) { let (base, seen, server) = mock_server(vec![ MockResponse::json(json!({"file_id":"reducto://uploaded.pdf"})), MockResponse::json( @@ -94,9 +101,9 @@ async fn test_parse_v3_file_upload_and_response_mapping(#[case] model: &str) { ), ]) .await; - let response = perform_ocr(wire_request(&format!("reducto/{model}"), &base, json!({}))) - .await - .unwrap(); + let mut request = wire_request(&format!("reducto/{model}"), &base, json!({})); + request.document = request.document.with_source(source.to_string()); + let response = perform_ocr(request).await.unwrap(); server.await.unwrap(); assert_eq!(response.pages[0].markdown, "hello"); let seen = seen.lock().unwrap(); diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index bda09a7d840..672efbb569d 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -28,6 +28,7 @@ litellm-ai-gateway = { workspace = true, default-features = false } litellm-python-interop.workspace = true pyo3.workspace = true pyo3-async-runtimes.workspace = true +reqwest.workspace = true serde.workspace = true serde_json.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 384f0be5a1b..6ef1fcb28e6 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -5,6 +5,7 @@ mod execution; mod function_trace; mod marshal; mod routes; +mod transport; use litellm_ai_gateway::io::responses_ws::ResponsesWebSocketConnection as RustResponsesWebSocketConnection; use pyo3::prelude::*; diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr.rs b/litellm-rust/crates/python-bridge/src/routes/ocr.rs index 772cc77044b..0218c70c737 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr.rs @@ -12,12 +12,13 @@ use serde_json::Value; fn prepare_ocr( inputs: OcrInputs, ) -> PyResult> + Send + 'static> { + let http_client = crate::transport::ocr_http_client().map_err(ocr_error_to_pyerr)?; let wire: OcrWireRequest = litellm_core::ocr::wire::decode_request_value(inputs.request, "request") .map_err(|error| ocr_error_to_pyerr(error.into()))?; let request = decode_request(wire).map_err(ocr_error_to_pyerr)?; Ok(async move { - perform_ocr(request) + perform_ocr(&http_client, request) .await .map(|response| response.into_json()) }) diff --git a/litellm-rust/crates/python-bridge/src/transport.rs b/litellm-rust/crates/python-bridge/src/transport.rs new file mode 100644 index 00000000000..f91804e2501 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/transport.rs @@ -0,0 +1,36 @@ +use std::sync::OnceLock; +use std::time::Duration; + +use litellm_core::error::{Error, TransportError}; + +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); + +struct BridgeTransport { + /// Automatic redirects must stay disabled: OCR validates each document + /// redirect target before following it. + ocr: reqwest::Client, +} + +impl BridgeTransport { + fn new() -> Result { + Ok(Self { + ocr: reqwest::Client::builder() + .connect_timeout(CONNECT_TIMEOUT) + .redirect(reqwest::redirect::Policy::none()) + .build()?, + }) + } +} + +/// Borrow the Python SDK's native OCR transport at the Rust/Python boundary. +/// +/// `reqwest::Client` is internally reference counted, so this clone lets a +/// `'static` bridge future share the boundary-owned connection pool. +pub(crate) fn ocr_http_client() -> Result { + static TRANSPORT: OnceLock> = OnceLock::new(); + TRANSPORT + .get_or_init(|| BridgeTransport::new().map_err(|error| error.to_string())) + .as_ref() + .map(|transport| transport.ocr.clone()) + .map_err(|error| TransportError::Network(error.clone()).into()) +}