mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
wip
This commit is contained in:
parent
40e7b024b7
commit
9e03b5322f
26 changed files with 356 additions and 104 deletions
8
litellm-rust/Cargo.lock
generated
8
litellm-rust/Cargo.lock
generated
|
|
@ -916,6 +916,12 @@ version = "2.11.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8"
|
||||
|
||||
[[package]]
|
||||
name = "data-url"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be1e0bca6c3637f992fc1cc7cbc52a78c1ef6db076dbf1059c4323d6a2048376"
|
||||
|
||||
[[package]]
|
||||
name = "deranged"
|
||||
version = "0.5.8"
|
||||
|
|
@ -1708,6 +1714,7 @@ dependencies = [
|
|||
"azure_core",
|
||||
"azure_identity",
|
||||
"base64 0.22.1",
|
||||
"data-url",
|
||||
"moka",
|
||||
"percent-encoding",
|
||||
"rand 0.8.7",
|
||||
|
|
@ -1737,6 +1744,7 @@ dependencies = [
|
|||
"litellm-python-interop",
|
||||
"pyo3",
|
||||
"pyo3-async-runtimes",
|
||||
"reqwest 0.12.28",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ mod hooks;
|
|||
mod types;
|
||||
pub use types::OcrRequest;
|
||||
|
||||
pub async fn ocr(request: OcrRequest<'_>) -> Result<Value, Error> {
|
||||
pub async fn ocr(http_client: &reqwest::Client, request: OcrRequest<'_>) -> Result<Value, Error> {
|
||||
let mut core_request = decode_request(OcrWireRequest {
|
||||
model: request.model.into(),
|
||||
document: request.document,
|
||||
|
|
@ -27,7 +27,7 @@ pub async fn ocr(request: OcrRequest<'_>) -> Result<Value, Error> {
|
|||
request.guardrails,
|
||||
request.request_metadata,
|
||||
));
|
||||
perform_ocr(core_request)
|
||||
perform_ocr(http_client, core_request)
|
||||
.await
|
||||
.map(|response| response.into_json())
|
||||
}
|
||||
|
|
@ -38,9 +38,17 @@ mod tests {
|
|||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
|
||||
use super::{OcrRequest, ocr};
|
||||
use super::{OcrRequest, ocr as run_ocr};
|
||||
use crate::integrations::types::RequestMetadata;
|
||||
|
||||
async fn ocr(request: OcrRequest<'_>) -> Result<serde_json::Value, litellm_core::Error> {
|
||||
let http_client = reqwest::Client::builder()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.build()
|
||||
.expect("test HTTP client builds");
|
||||
run_ocr(&http_client, request).await
|
||||
}
|
||||
|
||||
async fn read_http_request(socket: &mut TcpStream) -> String {
|
||||
let mut request = Vec::new();
|
||||
let mut buffer = [0_u8; 1024];
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ use litellm_ai_gateway::integrations::custom_logger::{
|
|||
CallbackTiming, CallbackValue, CustomLogger, LogFuture, ModelCallDetails,
|
||||
};
|
||||
use litellm_ai_gateway::integrations::types::RequestMetadata;
|
||||
use litellm_ai_gateway::ocr::{OcrRequest, ocr};
|
||||
use litellm_ai_gateway::ocr::{OcrRequest, ocr as run_ocr};
|
||||
use litellm_core::error::Error;
|
||||
|
||||
#[cfg(feature = "trace-parity")]
|
||||
|
|
@ -20,6 +20,14 @@ use litellm_core::observability::FunctionTrace;
|
|||
#[cfg(feature = "trace-parity")]
|
||||
use tracing::instrument::WithSubscriber;
|
||||
|
||||
async fn ocr(request: OcrRequest<'_>) -> Result<Value, Error> {
|
||||
let http_client = reqwest::Client::builder()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.build()
|
||||
.expect("test HTTP client builds");
|
||||
run_ocr(&http_client, request).await
|
||||
}
|
||||
|
||||
async fn read_http_headers(socket: &mut TcpStream) -> String {
|
||||
let mut request = Vec::new();
|
||||
let mut buffer = [0_u8; 1024];
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ name = "workspace_crate_allowlist"
|
|||
path = "tests/workspace_crate_allowlist.rs"
|
||||
|
||||
[dependencies]
|
||||
data-url = "0.3.2"
|
||||
base64.workspace = true
|
||||
azure_core.workspace = true
|
||||
azure_identity.workspace = true
|
||||
|
|
|
|||
|
|
@ -45,9 +45,9 @@ pub const EMPTY_TEXT_PLACEHOLDER: &str =
|
|||
pub const FUNCTION_TRACE_TARGET: &str = "litellm::function_trace";
|
||||
|
||||
pub(crate) const OCR_HTTP_TIMEOUT_SECS: u64 = 600;
|
||||
pub(crate) const OCR_CONNECT_TIMEOUT_SECS: u64 = 10;
|
||||
pub(crate) const OCR_POLL_TIMEOUT_SECS: u64 = 120;
|
||||
pub(crate) const OCR_POLL_RETRY_SECS: u64 = 2;
|
||||
pub(crate) const OCR_INLINE_MAX_BYTES: usize = 50 * 1024 * 1024;
|
||||
pub(crate) const OCR_DOWNLOAD_MAX_BYTES: u64 = 50 * 1024 * 1024;
|
||||
pub(crate) const OCR_MAX_FETCH_REDIRECTS: usize = 10;
|
||||
pub(crate) const OCR_ERROR_BODY_MAX_CHARS: usize = 256;
|
||||
|
|
|
|||
|
|
@ -2,33 +2,14 @@ use crate::ocr::error::OcrError;
|
|||
use crate::ocr::error::OcrRequestError;
|
||||
use crate::ocr::error::OcrResponseError;
|
||||
use std::net::IpAddr;
|
||||
use std::sync::OnceLock;
|
||||
use std::time::Duration;
|
||||
|
||||
use super::types::{OcrConnection, OcrDocument};
|
||||
use crate::constants::{
|
||||
OCR_CONNECT_TIMEOUT_SECS, OCR_ERROR_BODY_MAX_CHARS, OCR_HTTP_TIMEOUT_SECS,
|
||||
OCR_MAX_FETCH_REDIRECTS,
|
||||
};
|
||||
use crate::constants::{OCR_ERROR_BODY_MAX_CHARS, OCR_MAX_FETCH_REDIRECTS};
|
||||
use crate::error::TransportError;
|
||||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
use reqwest::Url;
|
||||
|
||||
pub(crate) fn http_client() -> Result<&'static reqwest::Client, TransportError> {
|
||||
static CLIENT: OnceLock<Result<reqwest::Client, String>> = OnceLock::new();
|
||||
CLIENT
|
||||
.get_or_init(|| {
|
||||
reqwest::Client::builder()
|
||||
.connect_timeout(Duration::from_secs(OCR_CONNECT_TIMEOUT_SECS))
|
||||
.timeout(Duration::from_secs(OCR_HTTP_TIMEOUT_SECS))
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.build()
|
||||
.map_err(|error| error.to_string())
|
||||
})
|
||||
.as_ref()
|
||||
.map_err(|error| TransportError::Network(error.clone()))
|
||||
}
|
||||
|
||||
pub(crate) fn network_error(error: reqwest::Error) -> TransportError {
|
||||
TransportError::from(error)
|
||||
}
|
||||
|
|
@ -97,6 +78,7 @@ async fn validate_safe_fetch_url(url: &Url) -> Result<(), OcrError> {
|
|||
}
|
||||
|
||||
async fn safe_get_document_url(
|
||||
http_client: &reqwest::Client,
|
||||
source: &str,
|
||||
timeout: Duration,
|
||||
) -> Result<reqwest::Response, OcrError> {
|
||||
|
|
@ -105,7 +87,7 @@ async fn safe_get_document_url(
|
|||
})?;
|
||||
for _ in 0..OCR_MAX_FETCH_REDIRECTS {
|
||||
validate_safe_fetch_url(&url).await?;
|
||||
let response = http_client()?
|
||||
let response = http_client
|
||||
.get(url.clone())
|
||||
.timeout(timeout)
|
||||
.send()
|
||||
|
|
@ -136,9 +118,13 @@ fn enforce_download_size(length: u64, max_bytes: u64) -> Result<(), OcrError> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
async fn download_data_uri(source: &str, connection: &OcrConnection) -> Result<String, OcrError> {
|
||||
async fn download_data_uri(
|
||||
http_client: &reqwest::Client,
|
||||
source: &str,
|
||||
connection: &OcrConnection,
|
||||
) -> Result<String, OcrError> {
|
||||
enforce_download_size(0, connection.max_download_bytes)?;
|
||||
let mut response = safe_get_document_url(source, connection.timeout).await?;
|
||||
let mut response = safe_get_document_url(http_client, source, connection.timeout).await?;
|
||||
if !response.status().is_success() {
|
||||
let status = response.status().as_u16();
|
||||
return Err(TransportError::Http {
|
||||
|
|
@ -172,6 +158,7 @@ async fn download_data_uri(source: &str, connection: &OcrConnection) -> Result<S
|
|||
}
|
||||
|
||||
pub(crate) async fn convert_document_url_to_data_uri(
|
||||
http_client: &reqwest::Client,
|
||||
document: OcrDocument,
|
||||
connection: &OcrConnection,
|
||||
) -> Result<OcrDocument, OcrError> {
|
||||
|
|
@ -179,8 +166,11 @@ pub(crate) async fn convert_document_url_to_data_uri(
|
|||
if !source.starts_with("http://") && !source.starts_with("https://") {
|
||||
return Ok(document);
|
||||
}
|
||||
let data_uri = tokio::time::timeout(connection.timeout, download_data_uri(source, connection))
|
||||
.await
|
||||
.map_err(|_| TransportError::Network("OCR document download timed out".into()))??;
|
||||
let data_uri = tokio::time::timeout(
|
||||
connection.timeout,
|
||||
download_data_uri(http_client, source, connection),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| TransportError::Network("OCR document download timed out".into()))??;
|
||||
Ok(document.with_source(data_uri))
|
||||
}
|
||||
|
|
|
|||
40
litellm-rust/crates/core/src/ocr/document.rs
Normal file
40
litellm-rust/crates/core/src/ocr/document.rs
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
use data_url::{DataUrl, DataUrlError, forgiving_base64::DecodeError, mime::Mime};
|
||||
|
||||
use super::error::OcrRequestError;
|
||||
|
||||
pub(crate) struct InlineDocument<'a>(DataUrl<'a>);
|
||||
|
||||
impl<'a> InlineDocument<'a> {
|
||||
pub(crate) fn parse(source: &'a str) -> Result<Option<Self>, OcrRequestError> {
|
||||
match DataUrl::process(source) {
|
||||
Ok(url) => Ok(Some(Self(url))),
|
||||
Err(DataUrlError::NotADataUrl) => Ok(None),
|
||||
Err(DataUrlError::NoComma) => Err(OcrRequestError::InvalidDataUri),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn mime_type(&self) -> &Mime {
|
||||
self.0.mime_type()
|
||||
}
|
||||
|
||||
pub(crate) fn decode(&self, max_bytes: usize) -> Result<Vec<u8>, OcrRequestError> {
|
||||
let mut body = Vec::new();
|
||||
self.0
|
||||
.decode(|bytes| {
|
||||
if bytes.len() > max_bytes.saturating_sub(body.len()) {
|
||||
return Err(OcrRequestError::InlineDocumentTooLarge);
|
||||
}
|
||||
body.extend_from_slice(bytes);
|
||||
Ok(())
|
||||
})
|
||||
.map_err(|error| match error {
|
||||
DecodeError::InvalidBase64(_) => OcrRequestError::InvalidDataUri,
|
||||
DecodeError::WriteError(error) => error,
|
||||
})?;
|
||||
Ok(body)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "../../tests/ocr_document.rs"]
|
||||
mod tests;
|
||||
|
|
@ -36,11 +36,13 @@ pub enum OcrRequestError {
|
|||
#[error("model_id cannot be a dot path segment")]
|
||||
DotModel,
|
||||
#[error(
|
||||
"Reducto requires a reducto:// id or a base64 data URI after OCR preprocessing; plain http(s) URLs are not supported"
|
||||
"Reducto requires a reducto:// id or a data URI after OCR preprocessing; plain http(s) URLs are not supported"
|
||||
)]
|
||||
ReductoSource,
|
||||
#[error("Invalid Reducto base64 data URI")]
|
||||
ReductoDataUri,
|
||||
#[error("Invalid OCR document data URI")]
|
||||
InvalidDataUri,
|
||||
#[error("OCR inline document exceeds maximum allowed size")]
|
||||
InlineDocumentTooLarge,
|
||||
#[error("OCR document URL rejected by SSRF protection")]
|
||||
BlockedDocumentUrl,
|
||||
#[error("Too many redirects while fetching OCR document URL")]
|
||||
|
|
|
|||
|
|
@ -15,7 +15,16 @@ use crate::providers::reducto::ocr::transformation::{
|
|||
use crate::providers::vertex_ai::ocr::deepseek::transformation::VertexAiDeepSeekOcrConfig;
|
||||
use crate::providers::vertex_ai::ocr::transformation::VertexAiOcrConfig;
|
||||
|
||||
pub(crate) async fn perform_ocr_request(request: OcrRequest) -> Result<OcrResponseData, Error> {
|
||||
struct OcrExecution<'a> {
|
||||
http_client: &'a reqwest::Client,
|
||||
lifecycle_context: CallLifecycleContext,
|
||||
hooks: Arc<dyn OcrHooks>,
|
||||
}
|
||||
|
||||
pub(crate) async fn perform_ocr_request(
|
||||
http_client: &reqwest::Client,
|
||||
request: OcrRequest,
|
||||
) -> Result<OcrResponseData, Error> {
|
||||
let context = CallLifecycleContext::new(
|
||||
"ocr",
|
||||
request.model.clone(),
|
||||
|
|
@ -32,8 +41,11 @@ pub(crate) async fn perform_ocr_request(request: OcrRequest) -> Result<OcrRespon
|
|||
request.document,
|
||||
$params,
|
||||
request.connection,
|
||||
context,
|
||||
request.hooks,
|
||||
OcrExecution {
|
||||
http_client,
|
||||
lifecycle_context: context,
|
||||
hooks: request.hooks,
|
||||
},
|
||||
)
|
||||
.await
|
||||
};
|
||||
|
|
@ -69,24 +81,34 @@ async fn perform_provider_ocr<C: OcrProviderConfig>(
|
|||
document: OcrDocument,
|
||||
params: C::InputParams,
|
||||
connection: OcrConnection,
|
||||
context: CallLifecycleContext,
|
||||
hooks: Arc<dyn OcrHooks>,
|
||||
execution: OcrExecution<'_>,
|
||||
) -> Result<OcrResponseData, Error> {
|
||||
let OcrExecution {
|
||||
http_client,
|
||||
lifecycle_context,
|
||||
hooks,
|
||||
} = execution;
|
||||
let request = prepare_ocr_call(config, model, document, params, connection);
|
||||
let lifecycle_hooks = OcrLifecycleHooks {
|
||||
hooks: hooks.clone(),
|
||||
provider_name: context.custom_llm_provider.clone(),
|
||||
provider_name: lifecycle_context.custom_llm_provider.clone(),
|
||||
marker: std::marker::PhantomData,
|
||||
};
|
||||
CallLifecycle::default()
|
||||
.run(context, request, &lifecycle_hooks, |request| {
|
||||
execute_ocr_provider_call(request, hooks.as_ref(), &lifecycle_hooks.provider_name)
|
||||
.run(lifecycle_context, request, &lifecycle_hooks, |request| {
|
||||
execute_ocr_provider_call(
|
||||
http_client,
|
||||
request,
|
||||
hooks.as_ref(),
|
||||
&lifecycle_hooks.provider_name,
|
||||
)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
async fn execute_ocr_provider_call<C: OcrProviderConfig>(
|
||||
http_client: &reqwest::Client,
|
||||
request: PreparedOcrRequest<C>,
|
||||
hooks: &dyn OcrHooks,
|
||||
provider_name: &str,
|
||||
|
|
@ -109,7 +131,7 @@ async fn execute_ocr_provider_call<C: OcrProviderConfig>(
|
|||
request.document
|
||||
};
|
||||
let document = config
|
||||
.prepare_document(document, &request.connection, &headers)
|
||||
.prepare_document(http_client, document, &request.connection, &headers)
|
||||
.await?;
|
||||
let body = serde_json::to_value(config.transform_ocr_request(
|
||||
&request.model,
|
||||
|
|
@ -130,7 +152,7 @@ async fn execute_ocr_provider_call<C: OcrProviderConfig>(
|
|||
} else {
|
||||
body
|
||||
};
|
||||
let mut builder = super::client::http_client()?
|
||||
let mut builder = http_client
|
||||
.post(&url)
|
||||
.json(&body)
|
||||
.timeout(request.connection.timeout);
|
||||
|
|
@ -142,6 +164,7 @@ async fn execute_ocr_provider_call<C: OcrProviderConfig>(
|
|||
.map_err(super::client::network_error)?;
|
||||
let decoded = config
|
||||
.read_response(
|
||||
http_client,
|
||||
response,
|
||||
&url,
|
||||
&headers,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
pub(crate) mod client;
|
||||
pub(crate) mod document;
|
||||
pub mod error;
|
||||
mod handler;
|
||||
pub mod hooks;
|
||||
|
|
@ -9,14 +10,21 @@ pub mod wire;
|
|||
|
||||
pub use types::{OcrRequest, OcrResponseData};
|
||||
|
||||
/// Execute an OCR request with a host-owned HTTP client.
|
||||
///
|
||||
/// The client must not follow redirects automatically. OCR document downloads
|
||||
/// validate each redirect target before issuing the next request.
|
||||
#[tracing::instrument(
|
||||
name = "ocr",
|
||||
target = "litellm::function_trace",
|
||||
level = "trace",
|
||||
skip_all
|
||||
)]
|
||||
pub async fn perform_ocr(request: OcrRequest) -> Result<OcrResponseData, crate::Error> {
|
||||
handler::perform_ocr_request(request).await
|
||||
pub async fn perform_ocr(
|
||||
http_client: &reqwest::Client,
|
||||
request: OcrRequest,
|
||||
) -> Result<OcrResponseData, crate::Error> {
|
||||
handler::perform_ocr_request(http_client, request).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ pub trait OcrProviderConfig: Send + Sync + Sized + 'static {
|
|||
) -> impl Future<Output = Result<Vec<(String, String)>, AuthError>> + Send;
|
||||
fn prepare_document(
|
||||
&self,
|
||||
http_client: &reqwest::Client,
|
||||
document: OcrDocument,
|
||||
connection: &OcrConnection,
|
||||
headers: &[(String, String)],
|
||||
|
|
@ -57,6 +58,7 @@ pub trait OcrProviderConfig: Send + Sync + Sized + 'static {
|
|||
|
||||
fn read_response(
|
||||
&self,
|
||||
_http_client: &reqwest::Client,
|
||||
response: reqwest::Response,
|
||||
_url: &str,
|
||||
_headers: &[(String, String)],
|
||||
|
|
|
|||
|
|
@ -5,13 +5,14 @@ use std::time::Duration;
|
|||
|
||||
use super::types::{AzureDocumentIntelligenceOperation, OperationStatus};
|
||||
use crate::constants::{AZURE_DI_SUBSCRIPTION_HEADER, OCR_POLL_RETRY_SECS};
|
||||
use crate::ocr::client::{http_client, network_error};
|
||||
use crate::ocr::client::network_error;
|
||||
use crate::ocr::types::OcrConnection;
|
||||
use crate::ocr::wire::{DecodedOcrResponse, read_json_response};
|
||||
use reqwest::Url;
|
||||
use tokio::time::Instant;
|
||||
|
||||
pub(crate) async fn read_operation_response(
|
||||
http_client: &reqwest::Client,
|
||||
response: reqwest::Response,
|
||||
original_url: &str,
|
||||
headers: &[(String, String)],
|
||||
|
|
@ -34,11 +35,12 @@ pub(crate) async fn read_operation_response(
|
|||
{
|
||||
return Err(OcrPollingError::PollOrigin.into());
|
||||
}
|
||||
poll_document_intelligence(operation, headers, connection, native).await
|
||||
poll_document_intelligence(http_client, operation, headers, connection, native).await
|
||||
}
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
async fn poll_document_intelligence(
|
||||
http_client: &reqwest::Client,
|
||||
url: Url,
|
||||
headers: &[(String, String)],
|
||||
connection: &OcrConnection,
|
||||
|
|
@ -52,7 +54,7 @@ async fn poll_document_intelligence(
|
|||
.checked_duration_since(Instant::now())
|
||||
.filter(|remaining| !remaining.is_zero())
|
||||
.ok_or(OcrPollingError::PollTimeout)?;
|
||||
let mut builder = http_client()?
|
||||
let mut builder = http_client
|
||||
.get(url.clone())
|
||||
.timeout(remaining.min(connection.timeout));
|
||||
for (name, value) in headers {
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
use crate::auth::AuthError;
|
||||
use crate::ocr::document::InlineDocument;
|
||||
use crate::ocr::error::OcrError;
|
||||
use crate::ocr::error::OcrRequestError;
|
||||
use crate::ocr::error::OcrResponseError;
|
||||
use crate::ocr::error::PagesError;
|
||||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use super::types::*;
|
||||
|
|
@ -99,19 +101,6 @@ fn normalize_features_param(
|
|||
Ok(Some(NormalizedFeatures(normalized.join(","))))
|
||||
}
|
||||
|
||||
fn extract_base64_from_data_uri(source: &str) -> &str {
|
||||
let Some((header, data)) = source.split_once(',') else {
|
||||
return source;
|
||||
};
|
||||
let media_type = header.strip_prefix("data:").unwrap_or_default();
|
||||
let media_type = media_type.strip_suffix(";base64").unwrap_or(media_type);
|
||||
if media_type.is_empty() || media_type.contains(';') || data.is_empty() {
|
||||
source
|
||||
} else {
|
||||
data
|
||||
}
|
||||
}
|
||||
|
||||
fn pixel_dimension(value: f64, scale: f64, field: &'static str) -> Result<i64, OcrResponseError> {
|
||||
let pixels = (value * scale).trunc();
|
||||
if !pixels.is_finite() || pixels < i64::MIN as f64 || pixels >= -(i64::MIN as f64) {
|
||||
|
|
@ -218,6 +207,7 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig {
|
|||
|
||||
async fn prepare_document(
|
||||
&self,
|
||||
_http_client: &reqwest::Client,
|
||||
document: OcrDocument,
|
||||
_connection: &OcrConnection,
|
||||
_headers: &[(String, String)],
|
||||
|
|
@ -236,9 +226,9 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig {
|
|||
if source.is_empty() {
|
||||
return Err(OcrRequestError::MissingField("document URL"));
|
||||
}
|
||||
Ok(if source.starts_with("data:") {
|
||||
Ok(if let Some(document) = InlineDocument::parse(source)? {
|
||||
DocumentIntelligenceRequest::Base64Source(
|
||||
extract_base64_from_data_uri(source).to_string(),
|
||||
STANDARD.encode(document.decode(crate::constants::OCR_INLINE_MAX_BYTES)?),
|
||||
)
|
||||
} else {
|
||||
DocumentIntelligenceRequest::UrlSource(source.to_string())
|
||||
|
|
@ -284,6 +274,7 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig {
|
|||
|
||||
async fn read_response(
|
||||
&self,
|
||||
http_client: &reqwest::Client,
|
||||
response: reqwest::Response,
|
||||
url: &str,
|
||||
headers: &[(String, String)],
|
||||
|
|
@ -291,6 +282,7 @@ impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig {
|
|||
params: &Self::MappedParams,
|
||||
) -> Result<DecodedOcrResponse<Self::ResponseBody>, OcrError> {
|
||||
super::polling::read_operation_response(
|
||||
http_client,
|
||||
response,
|
||||
url,
|
||||
headers,
|
||||
|
|
|
|||
|
|
@ -61,11 +61,13 @@ impl OcrProviderConfig for AzureAiOcrConfig {
|
|||
|
||||
async fn prepare_document(
|
||||
&self,
|
||||
http_client: &reqwest::Client,
|
||||
document: OcrDocument,
|
||||
connection: &OcrConnection,
|
||||
_headers: &[(String, String)],
|
||||
) -> Result<OcrDocument, OcrError> {
|
||||
crate::ocr::client::convert_document_url_to_data_uri(document, connection).await
|
||||
crate::ocr::client::convert_document_url_to_data_uri(http_client, document, connection)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn authenticate(
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ impl OcrProviderConfig for MistralOcrConfig {
|
|||
|
||||
async fn prepare_document(
|
||||
&self,
|
||||
_http_client: &reqwest::Client,
|
||||
document: OcrDocument,
|
||||
_connection: &OcrConnection,
|
||||
_headers: &[(String, String)],
|
||||
|
|
|
|||
|
|
@ -6,11 +6,10 @@ use std::collections::BTreeMap;
|
|||
|
||||
use super::types::*;
|
||||
use crate::constants::{REDUCTO_ID_PREFIX, REDUCTO_OCR_API_BASE};
|
||||
use crate::ocr::document::InlineDocument;
|
||||
use crate::ocr::transformation::OcrProviderConfig;
|
||||
use crate::ocr::types::{OcrConnection, OcrDocument, OcrPage, OcrResponseData, OcrUsageInfo};
|
||||
use crate::providers::reducto::auth;
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
|
||||
|
||||
pub struct ReductoParseV3Config;
|
||||
pub struct ReductoParseLegacyConfig;
|
||||
|
|
@ -26,6 +25,7 @@ pub fn normalize_api_base(api_base: Option<&str>) -> &str {
|
|||
}
|
||||
|
||||
async fn prepare_reducto_document(
|
||||
http_client: &reqwest::Client,
|
||||
document: OcrDocument,
|
||||
connection: &OcrConnection,
|
||||
headers: &[(String, String)],
|
||||
|
|
@ -34,28 +34,14 @@ async fn prepare_reducto_document(
|
|||
if source.starts_with(REDUCTO_ID_PREFIX) {
|
||||
return Ok(ReductoFileId(source.to_string()));
|
||||
}
|
||||
if !source.starts_with("data:") {
|
||||
return Err(OcrRequestError::ReductoSource.into());
|
||||
}
|
||||
let (header, encoded) = source
|
||||
.split_once(',')
|
||||
.ok_or(OcrRequestError::ReductoDataUri)?;
|
||||
if !header.split(';').any(|part| part == "base64") {
|
||||
return Err(OcrRequestError::ReductoDataUri.into());
|
||||
}
|
||||
let mime = header
|
||||
.strip_prefix("data:")
|
||||
.and_then(|header| header.split(';').next())
|
||||
.filter(|mime| !mime.is_empty())
|
||||
.unwrap_or("application/octet-stream");
|
||||
let bytes = BASE64_STANDARD
|
||||
.decode(encoded)
|
||||
.map_err(|_| OcrRequestError::ReductoDataUri)?;
|
||||
let document = InlineDocument::parse(source)?.ok_or(OcrRequestError::ReductoSource)?;
|
||||
let mime = document.mime_type().to_string();
|
||||
let bytes = document.decode(crate::constants::OCR_INLINE_MAX_BYTES)?;
|
||||
let part = reqwest::multipart::Part::bytes(bytes)
|
||||
.file_name("document")
|
||||
.mime_str(mime)
|
||||
.map_err(|_| OcrRequestError::ReductoDataUri)?;
|
||||
let mut builder = crate::ocr::client::http_client()?
|
||||
.mime_str(&mime)
|
||||
.map_err(|_| OcrRequestError::InvalidDataUri)?;
|
||||
let mut builder = http_client
|
||||
.post(format!(
|
||||
"{}/upload",
|
||||
normalize_api_base(connection.api_base.as_deref())
|
||||
|
|
@ -203,11 +189,12 @@ impl OcrProviderConfig for ReductoParseV3Config {
|
|||
|
||||
async fn prepare_document(
|
||||
&self,
|
||||
http_client: &reqwest::Client,
|
||||
document: OcrDocument,
|
||||
connection: &OcrConnection,
|
||||
headers: &[(String, String)],
|
||||
) -> Result<ReductoFileId, OcrError> {
|
||||
prepare_reducto_document(document, connection, headers).await
|
||||
prepare_reducto_document(http_client, document, connection, headers).await
|
||||
}
|
||||
|
||||
fn guard_document_before_preparation(&self) -> bool {
|
||||
|
|
@ -282,11 +269,12 @@ impl OcrProviderConfig for ReductoParseLegacyConfig {
|
|||
|
||||
async fn prepare_document(
|
||||
&self,
|
||||
http_client: &reqwest::Client,
|
||||
document: OcrDocument,
|
||||
connection: &OcrConnection,
|
||||
headers: &[(String, String)],
|
||||
) -> Result<ReductoFileId, OcrError> {
|
||||
prepare_reducto_document(document, connection, headers).await
|
||||
prepare_reducto_document(http_client, document, connection, headers).await
|
||||
}
|
||||
|
||||
fn guard_document_before_preparation(&self) -> bool {
|
||||
|
|
|
|||
|
|
@ -125,6 +125,7 @@ impl OcrProviderConfig for VertexAiDeepSeekOcrConfig {
|
|||
|
||||
async fn prepare_document(
|
||||
&self,
|
||||
_http_client: &reqwest::Client,
|
||||
document: OcrDocument,
|
||||
_connection: &OcrConnection,
|
||||
_headers: &[(String, String)],
|
||||
|
|
|
|||
|
|
@ -102,11 +102,13 @@ impl OcrProviderConfig for VertexAiOcrConfig {
|
|||
|
||||
async fn prepare_document(
|
||||
&self,
|
||||
http_client: &reqwest::Client,
|
||||
document: OcrDocument,
|
||||
connection: &OcrConnection,
|
||||
_headers: &[(String, String)],
|
||||
) -> Result<OcrDocument, OcrError> {
|
||||
crate::ocr::client::convert_document_url_to_data_uri(document, connection).await
|
||||
crate::ocr::client::convert_document_url_to_data_uri(http_client, document, connection)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn authenticate(
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
use super::transformation::AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG as CONFIG;
|
||||
use crate::ocr::perform_ocr;
|
||||
use crate::ocr::prepare::OcrProviderKind;
|
||||
use crate::ocr::tests::perform_ocr;
|
||||
use crate::ocr::tests::{MockResponse, body, mock_server, params, transform, wire_request};
|
||||
use crate::ocr::transformation::OcrProviderConfig;
|
||||
use crate::ocr::types::OcrConnection;
|
||||
|
|
@ -106,10 +106,10 @@ async fn document_intelligence_mistral_pages_flow_to_query_only() {
|
|||
}
|
||||
#[rstest]
|
||||
#[case("data:application/pdf;base64,YWJj", "YWJj")]
|
||||
#[case("data:application/pdf,YWJj", "YWJj")]
|
||||
#[case("data:application/pdf;base64,", "data:application/pdf;base64,")]
|
||||
#[case("data:application/pdf;other,YWJj", "data:application/pdf;other,YWJj")]
|
||||
#[case("data:application/pdf;base64", "data:application/pdf;base64")]
|
||||
#[case("data:application/pdf,abc", "YWJj")]
|
||||
#[case("data:application/pdf;base64,", "")]
|
||||
#[case("data:application/pdf;version=1.7;base64,YWJj", "YWJj")]
|
||||
#[case("DATA:;BASE64,YQ%3D%3D#page", "YQ==")]
|
||||
#[tokio::test]
|
||||
async fn document_intelligence_request_uses_base64_source_for_data_uri(
|
||||
#[case] source: &str,
|
||||
|
|
@ -350,3 +350,22 @@ async fn polling_deadline_bounds_retry_after() {
|
|||
server.await.unwrap();
|
||||
assert!(error.to_string().contains("timed out"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("data:application/pdf;base64")]
|
||||
#[case("data:application/pdf;base64,INVALID!")]
|
||||
#[tokio::test]
|
||||
async fn document_intelligence_rejects_invalid_data_uri(#[case] source: &str) {
|
||||
let error = body(
|
||||
&CONFIG,
|
||||
"model",
|
||||
json!({"type":"document_url","document_url":source}),
|
||||
json!({}),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
crate::ocr::error::OcrError::Request(crate::ocr::error::OcrRequestError::InvalidDataUri)
|
||||
));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,9 +6,22 @@ use tokio::net::TcpListener;
|
|||
use super::transformation::OcrProviderConfig;
|
||||
use super::types::OcrConnection;
|
||||
use super::wire::{OcrWireRequest, decode_request, decode_response};
|
||||
use super::{OcrRequest, perform_ocr};
|
||||
use super::{OcrRequest, perform_ocr as run_ocr};
|
||||
use crate::ocr::error::OcrError;
|
||||
|
||||
pub(crate) fn http_client() -> reqwest::Client {
|
||||
reqwest::Client::builder()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.build()
|
||||
.expect("test HTTP client builds")
|
||||
}
|
||||
|
||||
pub(crate) async fn perform_ocr(
|
||||
request: OcrRequest,
|
||||
) -> Result<super::OcrResponseData, crate::Error> {
|
||||
run_ocr(&http_client(), request).await
|
||||
}
|
||||
|
||||
pub(crate) fn params<C: OcrProviderConfig>(config: &C, value: Value) -> C::MappedParams {
|
||||
config
|
||||
.map_ocr_params(serde_json::from_value(value).unwrap())
|
||||
|
|
@ -36,8 +49,10 @@ pub(crate) async fn body<C: OcrProviderConfig>(
|
|||
options: Value,
|
||||
) -> Result<Value, OcrError> {
|
||||
let params = config.map_ocr_params(serde_json::from_value(options).unwrap())?;
|
||||
let http_client = http_client();
|
||||
let document = config
|
||||
.prepare_document(
|
||||
&http_client,
|
||||
serde_json::from_value(document).unwrap(),
|
||||
&OcrConnection::default(),
|
||||
&[],
|
||||
|
|
@ -151,6 +166,34 @@ async fn performs_mistral_ocr_from_typed_request() {
|
|||
assert!(body.get("unknown").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn uses_the_host_injected_http_client() {
|
||||
let (base, seen, server) = mock_server(vec![MockResponse::json(
|
||||
json!({"pages":[],"usage_info":{"pages_processed":0}}),
|
||||
)])
|
||||
.await;
|
||||
let mut default_headers = reqwest::header::HeaderMap::new();
|
||||
default_headers.insert(
|
||||
"x-transport-owner",
|
||||
reqwest::header::HeaderValue::from_static("python-sdk"),
|
||||
);
|
||||
let http_client = reqwest::Client::builder()
|
||||
.default_headers(default_headers)
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.build()
|
||||
.expect("test HTTP client builds");
|
||||
|
||||
run_ocr(
|
||||
&http_client,
|
||||
wire_request("mistral/model", &base, json!({})),
|
||||
)
|
||||
.await
|
||||
.expect("OCR request succeeds");
|
||||
|
||||
server.await.expect("mock server completes");
|
||||
assert!(seen.lock().unwrap()[0].contains("x-transport-owner: python-sdk"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_pages_fail_before_network_and_invoke_failure_hook() {
|
||||
use crate::call_lifecycle::{CallLifecycleContext, CallLifecycleTiming};
|
||||
|
|
|
|||
66
litellm-rust/crates/core/tests/ocr_document.rs
Normal file
66
litellm-rust/crates/core/tests/ocr_document.rs
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
use super::InlineDocument;
|
||||
use crate::ocr::error::OcrRequestError;
|
||||
use rstest::rstest;
|
||||
|
||||
#[rstest]
|
||||
#[case("data:application/pdf;base64,YWJj", b"abc")]
|
||||
#[case("DATA:application/pdf;BASE64,YWI", b"ab")]
|
||||
#[case("data:application/pdf;base64,YQ%3D%3D#fragment", b"a")]
|
||||
#[case("data:application/pdf;base64,Y W J j", b"abc")]
|
||||
#[case("data:,a%20b%00%FF", b"a b\0\xff")]
|
||||
#[case("data:;base64,", b"")]
|
||||
fn decodes_data_urls(#[case] source: &str, #[case] expected: &[u8]) {
|
||||
let document = InlineDocument::parse(source).unwrap().unwrap();
|
||||
assert_eq!(document.decode(expected.len()).unwrap(), expected);
|
||||
if !expected.is_empty() {
|
||||
assert_eq!(
|
||||
document.decode(expected.len() - 1),
|
||||
Err(OcrRequestError::InlineDocumentTooLarge)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_mime_parameters_and_uses_standard_default() {
|
||||
let document = InlineDocument::parse("data:application/pdf;version=1.7;base64,YQ==")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(document.mime_type().matches("application", "pdf"));
|
||||
assert_eq!(document.mime_type().get_parameter("version"), Some("1.7"));
|
||||
let default = InlineDocument::parse("data:,a").unwrap().unwrap();
|
||||
assert!(default.mime_type().matches("text", "plain"));
|
||||
assert_eq!(
|
||||
default.mime_type().get_parameter("charset"),
|
||||
Some("US-ASCII")
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("data:application/pdf;base64")]
|
||||
#[case("data:application/pdf;base64,INVALID!")]
|
||||
#[case("data:application/pdf;base64,Y")]
|
||||
fn rejects_invalid_input(#[case] source: &str) {
|
||||
let result = InlineDocument::parse(source).and_then(|document| document.unwrap().decode(100));
|
||||
assert_eq!(result, Err(OcrRequestError::InvalidDataUri));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leaves_other_sources_to_the_provider() {
|
||||
for source in ["https://example.com/file.pdf", "reducto://file.pdf"] {
|
||||
assert!(InlineDocument::parse(source).unwrap().is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn limits_total_decoded_bytes_across_chunks() {
|
||||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
|
||||
let bytes = vec![0xff; 8193];
|
||||
let source = format!("data:application/pdf;base64,{}", STANDARD.encode(&bytes));
|
||||
let document = InlineDocument::parse(&source).unwrap().unwrap();
|
||||
assert_eq!(document.decode(bytes.len()).unwrap(), bytes);
|
||||
assert_eq!(
|
||||
document.decode(bytes.len() - 1),
|
||||
Err(OcrRequestError::InlineDocumentTooLarge)
|
||||
);
|
||||
}
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
use super::transformation::{REDUCTO_PARSE_LEGACY_CONFIG as LEGACY, REDUCTO_PARSE_V3_CONFIG as V3};
|
||||
use crate::ocr::perform_ocr;
|
||||
use crate::ocr::tests::{MockResponse, body, mock_server, transform, wire_request};
|
||||
use crate::ocr::tests::{MockResponse, body, mock_server, perform_ocr, transform, wire_request};
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
|
|
@ -36,7 +35,7 @@ async fn test_parse_legacy_wraps_enhance_under_options() {
|
|||
#[rstest]
|
||||
#[case("http://example.com/a.pdf")]
|
||||
#[case("https://example.com/a.pdf")]
|
||||
#[case("data:application/pdf,YWJj")]
|
||||
#[case("data:application/pdf;base64")]
|
||||
#[case("data:application/pdf;base64,INVALID!")]
|
||||
#[tokio::test]
|
||||
async fn test_parse_v3_rejects_plain_http_urls(#[case] source: &str) {
|
||||
|
|
@ -86,7 +85,15 @@ fn reducto_missing_result_and_null_result_are_distinct() {
|
|||
#[case("parse-v3")]
|
||||
#[case("parse-legacy")]
|
||||
#[tokio::test]
|
||||
async fn test_parse_v3_file_upload_and_response_mapping(#[case] model: &str) {
|
||||
async fn test_parse_v3_file_upload_and_response_mapping(
|
||||
#[case] model: &str,
|
||||
#[values(
|
||||
"data:application/pdf;base64,YWJj",
|
||||
"data:application/pdf,abc",
|
||||
"DATA:application/pdf;BASE64,YWJj#page"
|
||||
)]
|
||||
source: &str,
|
||||
) {
|
||||
let (base, seen, server) = mock_server(vec![
|
||||
MockResponse::json(json!({"file_id":"reducto://uploaded.pdf"})),
|
||||
MockResponse::json(
|
||||
|
|
@ -94,9 +101,9 @@ async fn test_parse_v3_file_upload_and_response_mapping(#[case] model: &str) {
|
|||
),
|
||||
])
|
||||
.await;
|
||||
let response = perform_ocr(wire_request(&format!("reducto/{model}"), &base, json!({})))
|
||||
.await
|
||||
.unwrap();
|
||||
let mut request = wire_request(&format!("reducto/{model}"), &base, json!({}));
|
||||
request.document = request.document.with_source(source.to_string());
|
||||
let response = perform_ocr(request).await.unwrap();
|
||||
server.await.unwrap();
|
||||
assert_eq!(response.pages[0].markdown, "hello");
|
||||
let seen = seen.lock().unwrap();
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ litellm-ai-gateway = { workspace = true, default-features = false }
|
|||
litellm-python-interop.workspace = true
|
||||
pyo3.workspace = true
|
||||
pyo3-async-runtimes.workspace = true
|
||||
reqwest.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
tokio.workspace = true
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ mod execution;
|
|||
mod function_trace;
|
||||
mod marshal;
|
||||
mod routes;
|
||||
mod transport;
|
||||
|
||||
use litellm_ai_gateway::io::responses_ws::ResponsesWebSocketConnection as RustResponsesWebSocketConnection;
|
||||
use pyo3::prelude::*;
|
||||
|
|
|
|||
|
|
@ -12,12 +12,13 @@ use serde_json::Value;
|
|||
fn prepare_ocr(
|
||||
inputs: OcrInputs,
|
||||
) -> PyResult<impl Future<Output = Result<Value, Error>> + Send + 'static> {
|
||||
let http_client = crate::transport::ocr_http_client().map_err(ocr_error_to_pyerr)?;
|
||||
let wire: OcrWireRequest =
|
||||
litellm_core::ocr::wire::decode_request_value(inputs.request, "request")
|
||||
.map_err(|error| ocr_error_to_pyerr(error.into()))?;
|
||||
let request = decode_request(wire).map_err(ocr_error_to_pyerr)?;
|
||||
Ok(async move {
|
||||
perform_ocr(request)
|
||||
perform_ocr(&http_client, request)
|
||||
.await
|
||||
.map(|response| response.into_json())
|
||||
})
|
||||
|
|
|
|||
36
litellm-rust/crates/python-bridge/src/transport.rs
Normal file
36
litellm-rust/crates/python-bridge/src/transport.rs
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
use std::sync::OnceLock;
|
||||
use std::time::Duration;
|
||||
|
||||
use litellm_core::error::{Error, TransportError};
|
||||
|
||||
const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
struct BridgeTransport {
|
||||
/// Automatic redirects must stay disabled: OCR validates each document
|
||||
/// redirect target before following it.
|
||||
ocr: reqwest::Client,
|
||||
}
|
||||
|
||||
impl BridgeTransport {
|
||||
fn new() -> Result<Self, reqwest::Error> {
|
||||
Ok(Self {
|
||||
ocr: reqwest::Client::builder()
|
||||
.connect_timeout(CONNECT_TIMEOUT)
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.build()?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Borrow the Python SDK's native OCR transport at the Rust/Python boundary.
|
||||
///
|
||||
/// `reqwest::Client` is internally reference counted, so this clone lets a
|
||||
/// `'static` bridge future share the boundary-owned connection pool.
|
||||
pub(crate) fn ocr_http_client() -> Result<reqwest::Client, Error> {
|
||||
static TRANSPORT: OnceLock<Result<BridgeTransport, String>> = OnceLock::new();
|
||||
TRANSPORT
|
||||
.get_or_init(|| BridgeTransport::new().map_err(|error| error.to_string()))
|
||||
.as_ref()
|
||||
.map(|transport| transport.ocr.clone())
|
||||
.map_err(|error| TransportError::Network(error.clone()).into())
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue