diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index e4e0fbb1d0e..1f87f583eef 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -916,6 +916,12 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +[[package]] +name = "data-url" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be1e0bca6c3637f992fc1cc7cbc52a78c1ef6db076dbf1059c4323d6a2048376" + [[package]] name = "deranged" version = "0.5.8" @@ -1708,12 +1714,14 @@ dependencies = [ "azure_core", "azure_identity", "base64 0.22.1", + "data-url", "moka", "rand 0.8.7", "reqwest 0.12.28", "rstest", "serde", "serde_json", + "serde_path_to_error", "sha2 0.10.9", "strum", "subtle", @@ -1736,6 +1744,7 @@ dependencies = [ "litellm-python-interop", "pyo3", "pyo3-async-runtimes", + "reqwest 0.12.28", "serde", "serde_json", "tokio", diff --git a/litellm-rust/crates/ai-gateway/src/client.rs b/litellm-rust/crates/ai-gateway/src/client.rs deleted file mode 100644 index ff2606f0229..00000000000 --- a/litellm-rust/crates/ai-gateway/src/client.rs +++ /dev/null @@ -1,14 +0,0 @@ -use std::sync::OnceLock; -use std::time::Duration; - -const HTTP_CLIENT_TIMEOUT_SECS: u64 = 600; - -pub(crate) fn http_client() -> &'static reqwest::Client { - static CLIENT: OnceLock = OnceLock::new(); - CLIENT.get_or_init(|| { - reqwest::Client::builder() - .timeout(Duration::from_secs(HTTP_CLIENT_TIMEOUT_SECS)) - .build() - .expect("failed to build reqwest client") - }) -} diff --git a/litellm-rust/crates/ai-gateway/src/lib.rs b/litellm-rust/crates/ai-gateway/src/lib.rs index 08fbde564ed..3738854cd23 100644 --- a/litellm-rust/crates/ai-gateway/src/lib.rs +++ b/litellm-rust/crates/ai-gateway/src/lib.rs @@ -13,7 +13,6 @@ //! binary turns on. pub mod audio_transcription; -mod client; pub mod io; pub mod ocr; diff --git a/litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs b/litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs deleted file mode 100644 index d2be17260a3..00000000000 --- a/litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs +++ /dev/null @@ -1,529 +0,0 @@ -use std::net::IpAddr; -use std::time::{Duration, Instant}; - -use base64::Engine; -use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; -use litellm_core::error::Error; -use litellm_core::ocr::transformation::OcrProviderConfig; -use reqwest::Url; -use serde_json::{Map, Value}; - -use litellm_core::providers::azure_ai::ocr::transformation::{ - AZURE_AI_OCR_CONFIG, AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG, -}; -use litellm_core::providers::mistral::ocr::transformation::MISTRAL_OCR_CONFIG; -use litellm_core::providers::reducto::ocr::transformation as reducto; -use litellm_core::providers::vertex_ai::ocr::transformation as vertex_ai; -use litellm_core::providers::vertex_ai::ocr::transformation::{ - VERTEX_AI_DEEPSEEK_OCR_CONFIG, VERTEX_AI_OCR_CONFIG, -}; - -use crate::client::http_client; - -const ERROR_BODY_MAX_CHARS: usize = 256; -const AZURE_DOCUMENT_INTELLIGENCE_POLL_TIMEOUT_SECS: u64 = 120; -const DEFAULT_MAX_IMAGE_URL_DOWNLOAD_SIZE_MB: f64 = 50.0; -const MAX_SAFE_FETCH_REDIRECTS: usize = 10; - -pub(super) fn truncate_error_body(body: &str) -> String { - if body.chars().count() <= ERROR_BODY_MAX_CHARS { - return body.to_string(); - } - let truncated: String = body.chars().take(ERROR_BODY_MAX_CHARS).collect(); - format!("{truncated}... (truncated)") -} - -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -pub(super) fn ocr_provider_config( - provider: &str, - model: &str, -) -> Option<&'static dyn OcrProviderConfig> { - match provider { - "mistral" => Some(&MISTRAL_OCR_CONFIG), - "reducto" => reducto::config_for_model(model), - "azure_ai" if is_azure_document_intelligence_model(model) => { - Some(&AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG) - } - "azure_ai" => Some(&AZURE_AI_OCR_CONFIG), - "vertex_ai" if vertex_ai::is_deepseek_model(model) => Some(&VERTEX_AI_DEEPSEEK_OCR_CONFIG), - "vertex_ai" => Some(&VERTEX_AI_OCR_CONFIG), - _ => None, - } -} - -fn is_azure_document_intelligence_model(model: &str) -> bool { - let model = model.to_ascii_lowercase(); - model.contains("doc-intelligence") || model.contains("documentintelligence") -} - -pub(super) fn string_headers( - extra_headers: Option>, -) -> Result, Error> { - extra_headers - .unwrap_or_default() - .into_iter() - .map(|(key, value)| { - value - .as_str() - .map(|value| (key.clone(), value.to_string())) - .ok_or_else(|| { - Error::InvalidRequest(format!( - "OCR extra_headers.{key} must be a string, got {}", - litellm_core::error::json_type_name(&value) - )) - }) - }) - .collect() -} - -fn document_url_field(document: &Value) -> Result, Error> { - let Some(object) = document.as_object() else { - return Ok(None); - }; - let Some(doc_type) = object.get("type").and_then(Value::as_str) else { - return Ok(None); - }; - let field = match doc_type { - "document_url" => "document_url", - "image_url" => "image_url", - _ => return Ok(None), - }; - let Some(url) = object.get(field).and_then(Value::as_str) else { - return Ok(None); - }; - Ok(Some((field, url))) -} - -fn is_url_requiring_fetch(url: &str) -> bool { - !url.starts_with("data:") && (url.starts_with("http://") || url.starts_with("https://")) -} - -fn max_document_download_bytes() -> u64 { - let max_size_mb = std::env::var("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB") - .ok() - .and_then(|value| value.parse::().ok()) - .unwrap_or(DEFAULT_MAX_IMAGE_URL_DOWNLOAD_SIZE_MB); - (max_size_mb.max(0.0) * 1024.0 * 1024.0) as u64 -} - -fn is_blocked_ip(ip: IpAddr) -> bool { - match ip { - IpAddr::V4(ip) => { - ip.is_private() - || ip.is_loopback() - || ip.is_link_local() - || ip.is_broadcast() - || ip.is_multicast() - || ip.is_unspecified() - } - IpAddr::V6(ip) => { - let first_segment = ip.segments()[0]; - let is_unique_local = (first_segment & 0xfe00) == 0xfc00; - let is_link_local = (first_segment & 0xffc0) == 0xfe80; - ip.is_loopback() - || ip.is_unspecified() - || ip.is_multicast() - || is_unique_local - || is_link_local - || ip - .to_ipv4_mapped() - .or_else(|| ip.to_ipv4()) - .map(|v4| is_blocked_ip(IpAddr::V4(v4))) - .unwrap_or(false) - } - } -} - -fn blocked_url_error(url: &Url) -> Error { - Error::InvalidRequest(format!( - "OCR document URL rejected by SSRF protection: {url}" - )) -} - -async fn validate_safe_fetch_url(url: &Url) -> Result<(), Error> { - if !matches!(url.scheme(), "http" | "https") { - return Err(blocked_url_error(url)); - } - - let host = url.host_str().ok_or_else(|| blocked_url_error(url))?; - if let Ok(ip) = host.parse::() { - if is_blocked_ip(ip) { - return Err(blocked_url_error(url)); - } - return Ok(()); - } - - let port = url - .port_or_known_default() - .ok_or_else(|| blocked_url_error(url))?; - let addresses = tokio::net::lookup_host((host, port)) - .await - .map_err(|err| Error::Network(err.to_string()))?; - let mut saw_address = false; - for address in addresses { - saw_address = true; - if is_blocked_ip(address.ip()) { - return Err(blocked_url_error(url)); - } - } - if !saw_address { - return Err(blocked_url_error(url)); - } - Ok(()) -} - -fn redirect_location(response: &reqwest::Response, url: &Url) -> Result { - let location = response - .headers() - .get(reqwest::header::LOCATION) - .and_then(|value| value.to_str().ok()) - .ok_or_else(|| { - Error::InvalidResponse("OCR document redirect missing Location header".to_string()) - })?; - url.join(location) - .map_err(|err| Error::InvalidResponse(format!("invalid OCR document redirect: {err}"))) -} - -async fn safe_get_document_url(url: &str) -> Result<(Url, reqwest::Response), Error> { - let client = reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .map_err(|err| Error::Network(err.to_string()))?; - let mut current_url = Url::parse(url) - .map_err(|err| Error::InvalidRequest(format!("invalid OCR document URL: {err}")))?; - - for _ in 0..MAX_SAFE_FETCH_REDIRECTS { - validate_safe_fetch_url(¤t_url).await?; - let response = client - .get(current_url.clone()) - .send() - .await - .map_err(|err| Error::Network(err.to_string()))?; - if !response.status().is_redirection() { - return Ok((current_url, response)); - } - current_url = redirect_location(&response, ¤t_url)?; - } - - Err(Error::InvalidRequest( - "Too many redirects while fetching OCR document URL".to_string(), - )) -} - -fn enforce_download_size(content_length: u64, max_bytes: u64, url: &Url) -> Result<(), Error> { - if max_bytes == 0 { - return Err(Error::InvalidRequest(format!( - "OCR document URL download is disabled (MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0). url={url}" - ))); - } - if content_length > max_bytes { - let size_mb = content_length as f64 / (1024.0 * 1024.0); - let max_size_mb = max_bytes as f64 / (1024.0 * 1024.0); - return Err(Error::InvalidRequest(format!( - "OCR document size ({size_mb:.2}MB) exceeds maximum allowed size ({max_size_mb:.2}MB). url={url}" - ))); - } - Ok(()) -} - -async fn read_response_with_limit( - mut response: reqwest::Response, - url: &Url, -) -> Result, Error> { - let max_bytes = max_document_download_bytes(); - if let Some(content_length) = response.content_length() { - enforce_download_size(content_length, max_bytes, url)?; - } else { - enforce_download_size(0, max_bytes, url)?; - } - - let mut bytes = Vec::new(); - let mut bytes_downloaded: u64 = 0; - while let Some(chunk) = response - .chunk() - .await - .map_err(|err| Error::Network(err.to_string()))? - { - bytes_downloaded += chunk.len() as u64; - enforce_download_size(bytes_downloaded, max_bytes, url)?; - bytes.extend_from_slice(&chunk); - } - Ok(bytes) -} - -pub(super) async fn convert_document_url_to_data_uri(document: Value) -> Result { - let Some((field, url)) = document_url_field(&document)? else { - return Ok(document); - }; - if !is_url_requiring_fetch(url) { - return Ok(document); - } - - let (final_url, response) = safe_get_document_url(url).await?; - let status = response.status(); - if !status.is_success() { - let body = response.text().await.unwrap_or_default(); - return Err(Error::Http { - status: status.as_u16(), - body: truncate_error_body(&body), - }); - } - let content_type = response - .headers() - .get(reqwest::header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.split(';').next()) - .map(str::trim) - .filter(|value| !value.is_empty()) - .unwrap_or("application/octet-stream") - .to_string(); - let bytes = read_response_with_limit(response, &final_url).await?; - let data_uri = format!( - "data:{content_type};base64,{}", - BASE64_STANDARD.encode(bytes) - ); - - let mut transformed = document - .as_object() - .cloned() - .ok_or_else(|| Error::InvalidRequest("OCR document must be an object".to_string()))?; - transformed.insert(field.to_string(), Value::String(data_uri)); - Ok(Value::Object(transformed)) -} - -fn same_origin(left: &str, right: &str) -> bool { - let Ok(left) = reqwest::Url::parse(left) else { - return false; - }; - let Ok(right) = reqwest::Url::parse(right) else { - return false; - }; - left.scheme() == right.scheme() - && left.host_str() == right.host_str() - && left.port_or_known_default() == right.port_or_known_default() -} - -fn retry_after_secs(response: &reqwest::Response) -> u64 { - response - .headers() - .get(reqwest::header::RETRY_AFTER) - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.parse::().ok()) - .unwrap_or(2) -} - -fn operation_status(response_json: &Value) -> Result<&str, Error> { - let status = response_json - .get("status") - .and_then(Value::as_str) - .ok_or(Error::MissingField("status"))?; - match status { - "succeeded" => Ok("succeeded"), - "running" | "notStarted" => Ok("running"), - "failed" => { - let message = response_json - .get("error") - .and_then(|error| error.get("message")) - .and_then(Value::as_str) - .unwrap_or("Unknown error"); - Err(Error::InvalidResponse(format!( - "Azure Document Intelligence analysis failed: {message}" - ))) - } - other => Err(Error::InvalidResponse(format!( - "Unknown operation status: {other}" - ))), - } -} - -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -pub(super) async fn poll_document_intelligence( - operation_url: &str, - original_url: &str, - headers: &[(String, String)], - timeout: Option, -) -> Result { - if !same_origin(operation_url, original_url) { - return Err(Error::InvalidResponse( - "Azure Document Intelligence: rejected cross-origin polling URL".to_string(), - )); - } - - let start = Instant::now(); - let timeout = timeout.unwrap_or(Duration::from_secs( - AZURE_DOCUMENT_INTELLIGENCE_POLL_TIMEOUT_SECS, - )); - loop { - if start.elapsed() > timeout { - return Err(Error::Network(format!( - "Azure Document Intelligence operation polling timed out after {} seconds", - timeout.as_secs() - ))); - } - - let mut request_builder = http_client().get(operation_url); - for (key, value) in headers { - if key.eq_ignore_ascii_case("ocp-apim-subscription-key") { - request_builder = request_builder.header(key, value); - } - } - let response = request_builder - .send() - .await - .map_err(|err| Error::Network(err.to_string()))?; - let retry_after = retry_after_secs(&response); - let status = response.status(); - let text = response - .text() - .await - .map_err(|err| Error::Network(err.to_string()))?; - if !status.is_success() { - return Err(Error::Http { - status: status.as_u16(), - body: truncate_error_body(&text), - }); - } - let response_json: Value = serde_json::from_str(&text).map_err(|err| { - Error::InvalidResponse(format!("invalid Azure DI poll response JSON: {err}")) - })?; - if operation_status(&response_json)? == "succeeded" { - return Ok(response_json); - } - tokio::time::sleep(Duration::from_secs(retry_after)).await; - } -} - -#[cfg(test)] -mod tests { - use litellm_core::ocr::transformation::OcrResponseHandling; - use serde_json::json; - - use super::*; - - #[test] - fn blocks_private_and_metadata_ips() { - assert!(is_blocked_ip("127.0.0.1".parse().unwrap())); - assert!(is_blocked_ip("10.0.0.1".parse().unwrap())); - assert!(is_blocked_ip("169.254.169.254".parse().unwrap())); - assert!(is_blocked_ip("::1".parse().unwrap())); - assert!(is_blocked_ip("fd00::1".parse().unwrap())); - assert!(is_blocked_ip("fe80::1".parse().unwrap())); - assert!(is_blocked_ip("::ffff:169.254.169.254".parse().unwrap())); - assert!(is_blocked_ip("::ffff:10.0.0.1".parse().unwrap())); - assert!(!is_blocked_ip("8.8.8.8".parse().unwrap())); - assert!(!is_blocked_ip("::ffff:8.8.8.8".parse().unwrap())); - } - - #[tokio::test] - async fn convert_document_url_rejects_loopback_fetch() { - let error = convert_document_url_to_data_uri(json!({ - "type": "image_url", - "image_url": "http://127.0.0.1/image.png" - })) - .await - .unwrap_err(); - - assert!(matches!( - error, - Error::InvalidRequest(message) - if message.contains("SSRF protection") - )); - } - - #[tokio::test] - async fn convert_document_url_leaves_data_uri_untouched() { - let document = json!({ - "type": "image_url", - "image_url": "data:image/png;base64,abcd" - }); - - let transformed = convert_document_url_to_data_uri(document.clone()) - .await - .unwrap(); - - assert_eq!(transformed, document); - } - - #[test] - fn truncate_error_body_passes_short_strings_through() { - let body = "Unauthorized"; - assert_eq!(truncate_error_body(body), "Unauthorized"); - } - - #[test] - fn truncate_error_body_caps_long_payloads() { - let body = "x".repeat(306); - let truncated = truncate_error_body(&body); - - assert!(truncated.ends_with("... (truncated)")); - let prefix_chars = truncated - .strip_suffix("... (truncated)") - .expect("truncated marker present") - .chars() - .count(); - assert_eq!(prefix_chars, 256); - } - - #[test] - fn truncate_error_body_does_not_split_multibyte_chars() { - let body = "é".repeat(266); - let truncated = truncate_error_body(&body); - assert!(truncated.is_char_boundary(truncated.len())); - } - - #[test] - fn ocr_dispatch_supports_migrated_providers() { - assert!(ocr_provider_config("mistral", "mistral-ocr-latest").is_some()); - assert!( - ocr_provider_config("azure_ai", "pixtral-12b-2409") - .expect("azure ai config resolves") - .requires_data_uri_document() - ); - assert_eq!( - ocr_provider_config("azure_ai", "doc-intelligence/prebuilt-read") - .expect("document intelligence config resolves") - .response_handling(), - OcrResponseHandling::AzureDocumentIntelligencePoll - ); - assert!( - ocr_provider_config("vertex_ai", "deepseek-ocr-maas") - .expect("vertex deepseek config resolves") - .supported_ocr_params() - .contains(&"temperature") - ); - assert!(ocr_provider_config("openai", "gpt-4o").is_none()); - } - - #[test] - fn string_headers_accepts_string_values() { - let headers = json!({ - "x-trace-id": "trace-1" - }) - .as_object() - .unwrap() - .clone(); - - assert_eq!( - string_headers(Some(headers)).expect("string headers accepted"), - vec![("x-trace-id".to_string(), "trace-1".to_string())] - ); - } - - #[test] - fn string_headers_rejects_non_string_values() { - let headers = json!({ - "x-retry-count": 3 - }) - .as_object() - .unwrap() - .clone(); - - let err = string_headers(Some(headers)).expect_err("non-string header rejected"); - assert_eq!( - err, - Error::InvalidRequest( - "OCR extra_headers.x-retry-count must be a string, got number".to_string() - ) - ); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/ocr/handler.rs b/litellm-rust/crates/ai-gateway/src/ocr/handler.rs deleted file mode 100644 index 6c6e12724cd..00000000000 --- a/litellm-rust/crates/ai-gateway/src/ocr/handler.rs +++ /dev/null @@ -1,84 +0,0 @@ -use litellm_core::error::Error; -use litellm_core::http_utils::http_request; -use litellm_core::ocr::transformation::OcrResponseHandling; -use serde_json::Value; - -use super::common_utils::{poll_document_intelligence, truncate_error_body}; -use super::hooks::OcrLifecycleHooks; -use super::types::PreparedOcrRequest; -use crate::client::http_client; - -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -pub(crate) async fn execute_ocr_provider_call( - request: PreparedOcrRequest, - hooks: &OcrLifecycleHooks, -) -> Result { - let request = hooks.prepare_provider_request(request).await?; - let mut request_builder = http_client().post(&request.url).json(&request.body); - for (key, value) in &request.upstream_headers { - request_builder = request_builder.header(key, value); - } - if let Some(duration) = request.timeout { - request_builder = request_builder.timeout(duration); - } - - let response = http_request(request_builder) - .await - .map_err(|err| Error::Network(err.to_string()))?; - - let status = response.status(); - if request.config.response_handling() == OcrResponseHandling::AzureDocumentIntelligencePoll - && status.as_u16() == 202 - { - let operation_url = response - .headers() - .get("operation-location") - .and_then(|value| value.to_str().ok()) - .map(str::to_string) - .ok_or_else(|| { - Error::InvalidResponse( - "Azure Document Intelligence returned 202 but no Operation-Location header found" - .to_string(), - ) - })?; - let response_json = poll_document_intelligence( - &operation_url, - &request.url, - &request.upstream_headers, - request.timeout, - ) - .await?; - return Ok(request - .config - .transform_ocr_response_with_params( - &request.model, - response_json, - &request.optional_params, - )? - .into_json()); - } - - let text = response - .text() - .await - .map_err(|err| Error::Network(err.to_string()))?; - - if !status.is_success() { - return Err(Error::Http { - status: status.as_u16(), - body: truncate_error_body(&text), - }); - } - - let response_json: Value = serde_json::from_str(&text) - .map_err(|err| Error::InvalidResponse(format!("invalid OCR response JSON: {err}")))?; - - Ok(request - .config - .transform_ocr_response_with_params( - &request.model, - response_json, - &request.optional_params, - )? - .into_json()) -} diff --git a/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs b/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs index ed41f1ff9e7..6da54ebf136 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/hooks.rs @@ -1,162 +1,43 @@ -use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; -use litellm_core::error::Error; -use litellm_core::providers::reducto::ocr::transformation::{ - build_upload_request, extract_document_source, extract_upload_file_id, -}; -use serde_json::{Map, Value, json}; -use std::future::Future; -use std::pin::Pin; +use serde_json::json; +use std::sync::Arc; + +use litellm_core::Error; +use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleTiming}; +use litellm_core::error::ErrorKind; +use litellm_core::ocr::hooks::{ + OcrDuringCallRequest, OcrHookFuture, OcrHooks, OcrLogFuture, OcrPreCallRequest, +}; +use litellm_core::ocr::types::OcrResponseData; +use litellm_core::ocr::wire::{decode_during_call_result, decode_pre_call_result}; -use super::common_utils::{convert_document_url_to_data_uri, string_headers, truncate_error_body}; -use super::types::{PreparedOcrRequest, ProviderOcrRequest}; -use crate::client::http_client; use crate::integrations::custom_guardrail::{ - CustomGuardrailRunner, GuardrailContext, GuardrailError, GuardrailRequest, + CustomGuardrail, CustomGuardrailRunner, GuardrailContext, GuardrailRequest, }; use crate::integrations::custom_logger::{ - CallType, CallbackTiming, CallbackValue, CustomLoggerRunner, LoggingError, ModelCallDetails, + CallType, CallbackTiming, CallbackValue, CustomLogger, CustomLoggerRunner, LoggingError, + ModelCallDetails, }; use crate::integrations::types::{ RequestMetadata, StandardLoggingMetadata, StandardLoggingPayload, }; -pub(crate) struct OcrLifecycleHooks { +pub(super) struct OcrGatewayHooks { logger_runner: CustomLoggerRunner, guardrail_runner: CustomGuardrailRunner, request_metadata: RequestMetadata, } - -type OcrFuture<'a, T> = Pin> + Send + 'a>>; -type OcrLogFuture<'a> = Pin + Send + 'a>>; - -impl OcrLifecycleHooks { - pub(crate) fn new( - logger_runner: CustomLoggerRunner, - guardrail_runner: CustomGuardrailRunner, +impl OcrGatewayHooks { + pub(super) fn new( + callbacks: Vec>, + guardrails: Vec>, request_metadata: RequestMetadata, ) -> Self { Self { - logger_runner, - guardrail_runner, + logger_runner: CustomLoggerRunner::new(callbacks), + guardrail_runner: CustomGuardrailRunner::new(guardrails), request_metadata, } } - - async fn run_pre_call_guardrails( - &self, - request: PreparedOcrRequest, - ) -> Result { - if self.guardrail_runner.is_empty() { - return Ok(request); - } - - let context = guardrail_context(&self.request_metadata); - let guardrail_request = GuardrailRequest::new(json!({ - "model": request.model, - "custom_llm_provider": request.custom_llm_provider, - "document": request.document, - "optional_params": request.optional_params, - })); - let (guardrail_request, _) = self - .guardrail_runner - .run_pre_call(&context, guardrail_request) - .await - .map_err(guardrail_error_to_core_error)?; - let (document, optional_params) = parse_ocr_pre_call_guardrail_request(guardrail_request)?; - let optional_params = match &request.config { - Ok(config) => config.map_ocr_params(&optional_params), - Err(_) => optional_params, - }; - Ok(PreparedOcrRequest { - document, - optional_params, - ..request - }) - } - - pub(crate) async fn prepare_provider_request( - &self, - request: PreparedOcrRequest, - ) -> Result { - let config = request.config?; - let env_lookup = |key: &str| std::env::var(key).ok(); - let upstream_headers = config.validate_environment( - string_headers(request.extra_headers)?, - request.api_key.as_deref(), - &env_lookup, - )?; - let url = config.complete_url( - request.api_base.as_deref(), - &request.model, - &request.optional_params, - &env_lookup, - )?; - let model = request.model.clone(); - let custom_llm_provider = request.custom_llm_provider.clone(); - let is_reducto = custom_llm_provider == "reducto"; - let document = if is_reducto { - let guarded_document = self - .run_during_call_guardrails(&model, &custom_llm_provider, &url, request.document) - .await?; - upload_reducto_document( - &guarded_document, - request.api_base.as_deref(), - request.timeout, - &upstream_headers, - ) - .await? - } else if config.requires_data_uri_document() { - convert_document_url_to_data_uri(request.document).await? - } else { - request.document - }; - let optional_params = request.optional_params; - let body = config - .transform_ocr_request(&request.model, document, optional_params.clone())? - .data; - let body = if is_reducto { - body - } else { - self.run_during_call_guardrails(&model, &custom_llm_provider, &url, body) - .await? - }; - Ok(ProviderOcrRequest { - model, - config, - url, - body, - optional_params, - upstream_headers, - timeout: request.timeout, - }) - } - - async fn run_during_call_guardrails( - &self, - model: &str, - custom_llm_provider: &str, - url: &str, - body: Value, - ) -> Result { - if self.guardrail_runner.is_empty() { - return Ok(body); - } - - let context = guardrail_context(&self.request_metadata); - let guardrail_request = GuardrailRequest::new(json!({ - "model": model, - "custom_llm_provider": custom_llm_provider, - "url": url, - "body": body, - })); - let (guardrail_request, _) = self - .guardrail_runner - .run_during_call(&context, guardrail_request) - .await - .map_err(guardrail_error_to_core_error)?; - parse_ocr_during_call_guardrail_request(guardrail_request) - } - fn standard_logging_payload( &self, context: &CallLifecycleContext, @@ -186,140 +67,80 @@ impl OcrLifecycleHooks { } } -async fn upload_reducto_document( - document: &Value, - api_base: Option<&str>, - timeout: Option, - upstream_headers: &[(String, String)], -) -> Result { - let source = extract_document_source(document)?; - let Some(authorization) = upstream_headers - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case("authorization")) - .map(|(_, value)| value.as_str()) - else { - return Err(Error::Auth( - "Reducto upload requires an Authorization header".to_string(), - )); - }; - let Some(upload) = build_upload_request(source, authorization, api_base) else { - return Ok(document.clone()); - }; - let part = reqwest::multipart::Part::bytes(upload.bytes) - .file_name(upload.file_name) - .mime_str(&upload.mime_type) - .map_err(|error| Error::InvalidRequest(error.to_string()))?; - let form = reqwest::multipart::Form::new().part("file", part); - let mut request_builder = http_client().post(upload.url).multipart(form); - for (name, value) in upstream_headers { - if !name.eq_ignore_ascii_case("content-type") - && !name.eq_ignore_ascii_case("content-length") - { - request_builder = request_builder.header(name, value); - } - } - if let Some(timeout) = timeout { - request_builder = request_builder.timeout(timeout); - } - let response = request_builder - .send() - .await - .map_err(|error| Error::Network(error.to_string()))?; - let status = response.status(); - let body = response - .text() - .await - .map_err(|error| Error::Network(error.to_string()))?; - if !status.is_success() { - return Err(Error::Http { - status: status.as_u16(), - body: truncate_error_body(&body), - }); - } - let response_json: Value = serde_json::from_str(&body).map_err(|error| { - Error::InvalidResponse(format!("invalid Reducto upload response JSON: {error}")) - })?; - let file_id = extract_upload_file_id(&response_json)?; - Ok(json!({"type": "document_url", "document_url": file_id})) -} - -impl CallLifecycleHooks for OcrLifecycleHooks { - type PreCallFuture<'a> = OcrFuture<'a, PreparedOcrRequest>; - type DuringCallFuture<'a> = OcrFuture<'a, PreparedOcrRequest>; - type SuccessFuture<'a> = OcrLogFuture<'a>; - type FailureFuture<'a> = OcrLogFuture<'a>; - - fn async_pre_call_hook<'a>( - &'a self, - _context: &'a CallLifecycleContext, - request: PreparedOcrRequest, - ) -> Self::PreCallFuture<'a> { - Box::pin(async move { self.run_pre_call_guardrails(request).await }) +impl OcrHooks for OcrGatewayHooks { + fn has_guardrails(&self) -> bool { + !self.guardrail_runner.is_empty() } - fn async_during_call_hook<'a>( - &'a self, - _context: &'a CallLifecycleContext, - request: PreparedOcrRequest, - ) -> Self::DuringCallFuture<'a> { - Box::pin(async move { Ok(request) }) + fn pre_call(&self, request: OcrPreCallRequest) -> OcrHookFuture<'_, OcrPreCallRequest> { + Box::pin(async move { + let context = guardrail_context(&self.request_metadata); + let payload = GuardrailRequest::new(json!(&request)); + let (changed, _) = self + .guardrail_runner + .run_pre_call(&context, payload) + .await + .map_err(|error| { + Error::InvalidRequest(format!("{}: {}", error.kind, error.message)) + })?; + Ok(decode_pre_call_result(request, changed.data)?) + }) } - - #[tracing::instrument( - name = "success_callback", - target = "litellm::function_trace", - level = "trace", - skip_all - )] - fn async_log_success_event<'a>( + fn during_call( + &self, + request: OcrDuringCallRequest, + ) -> OcrHookFuture<'_, OcrDuringCallRequest> { + Box::pin(async move { + let context = guardrail_context(&self.request_metadata); + let payload = GuardrailRequest::new(json!(&request)); + let (changed, _) = self + .guardrail_runner + .run_during_call(&context, payload) + .await + .map_err(|error| { + Error::InvalidRequest(format!("{}: {}", error.kind, error.message)) + })?; + Ok(decode_during_call_result(request, changed.data)?) + }) + } + fn success<'a>( &'a self, context: &'a CallLifecycleContext, - response: &'a Value, + response: &'a OcrResponseData, timing: &'a CallLifecycleTiming, - ) -> Self::SuccessFuture<'a> { + ) -> OcrLogFuture<'a> { Box::pin(async move { if self.logger_runner.is_empty() { return; } - let response_obj = CallbackValue::new("ocr", response.clone()); self.logger_runner .async_log_success_event( &ModelCallDetails::from_standard_logging_payload( self.standard_logging_payload(context, timing), ), - &response_obj, + &CallbackValue::new("ocr", json!(response)), CallbackTiming::new(timing.start_time, timing.end_time), ) .await; }) } - - #[tracing::instrument( - name = "failure_callback", - target = "litellm::function_trace", - level = "trace", - skip_all - )] - fn async_log_failure_event<'a>( + fn failure<'a>( &'a self, context: &'a CallLifecycleContext, error: &'a Error, timing: &'a CallLifecycleTiming, - ) -> Self::FailureFuture<'a> { + ) -> OcrLogFuture<'a> { Box::pin(async move { if self.logger_runner.is_empty() { return; } let logging_error = LoggingError { message: error.to_string(), - kind: core_error_kind(error).to_string(), + kind: core_error_kind(error).into(), }; - let response_obj = CallbackValue::new( + let response = CallbackValue::new( "error", - json!({ - "message": logging_error.message, - "kind": logging_error.kind, - }), + json!({ "message": logging_error.message, "kind": logging_error.kind }), ); self.logger_runner .async_log_failure_event( @@ -327,7 +148,7 @@ impl CallLifecycleHooks for OcrLi self.standard_logging_payload(context, timing), ) .with_failure_error(logging_error), - Some(&response_obj), + Some(&response), CallbackTiming::new(timing.start_time, timing.end_time), ) .await; @@ -347,55 +168,18 @@ fn guardrail_context(metadata: &RequestMetadata) -> GuardrailContext { } } -fn parse_ocr_pre_call_guardrail_request( - request: GuardrailRequest, -) -> Result<(Value, Map), Error> { - let Value::Object(mut data) = request.data else { - return Err(Error::InvalidRequest( - "OCR pre_call guardrail must return an object".to_string(), - )); - }; - let document = data.remove("document").ok_or_else(|| { - Error::InvalidRequest("OCR pre_call guardrail removed document".to_string()) - })?; - let optional_params = match data.remove("optional_params") { - Some(Value::Object(params)) => params, - Some(_) => { - return Err(Error::InvalidRequest( - "OCR pre_call guardrail optional_params must be an object".to_string(), - )); - } - None => Map::new(), - }; - Ok((document, optional_params)) -} - -fn parse_ocr_during_call_guardrail_request(request: GuardrailRequest) -> Result { - let Value::Object(mut data) = request.data else { - return Err(Error::InvalidRequest( - "OCR during_call guardrail must return an object".to_string(), - )); - }; - data.remove("body") - .ok_or_else(|| Error::InvalidRequest("OCR during_call guardrail removed body".to_string())) -} - -fn guardrail_error_to_core_error(error: GuardrailError) -> Error { - Error::InvalidRequest(format!("{}: {}", error.kind, error.message)) -} - fn core_error_kind(error: &Error) -> &'static str { - match error { - Error::Auth(_) => "AuthError", - Error::InvalidProvider(_) => "InvalidProvider", - Error::InvalidRequest(_) => "InvalidRequest", - Error::InvalidType { .. } => "InvalidType", - Error::MissingField(_) => "MissingField", - Error::Http { .. } => "HttpError", - Error::InvalidResponse(_) => "InvalidResponse", - Error::Network(_) => "NetworkError", - Error::Connect(_) => "ConnectError", - Error::Routing(_) => "RoutingError", - Error::Unsupported(_) => "UnsupportedRequest", + match error.kind() { + ErrorKind::Auth => "AuthError", + ErrorKind::InvalidProvider => "InvalidProvider", + ErrorKind::InvalidRequest => "InvalidRequest", + ErrorKind::InvalidType => "InvalidType", + ErrorKind::MissingField => "MissingField", + ErrorKind::Http => "HttpError", + ErrorKind::InvalidResponse => "InvalidResponse", + ErrorKind::Network => "NetworkError", + ErrorKind::Connect => "ConnectError", + ErrorKind::Routing => "RoutingError", + ErrorKind::Unsupported => "UnsupportedRequest", } } diff --git a/litellm-rust/crates/ai-gateway/src/ocr/mod.rs b/litellm-rust/crates/ai-gateway/src/ocr/mod.rs index 2acdd232c80..4865a640530 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/mod.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/mod.rs @@ -1,26 +1,41 @@ use litellm_core::Error; -use litellm_core::call_lifecycle::CallLifecycle; +use litellm_core::ocr::{ + OcrClient, + wire::{OcrWireRequest, decode_request}, +}; use serde_json::Value; +use std::sync::Arc; -mod common_utils; -mod handler; mod hooks; -mod prepare; mod types; - pub use types::OcrRequest; -use handler::execute_ocr_provider_call; -use prepare::{PreparedOcrCall, prepare_ocr_call}; - -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -pub async fn ocr(request: OcrRequest<'_>) -> Result { - let PreparedOcrCall { request, hooks } = prepare_ocr_call(request); - CallLifecycle::default() - .run_request(request, &hooks, |request| { - execute_ocr_provider_call(request, &hooks) - }) +pub async fn ocr(client: &OcrClient, request: OcrRequest<'_>) -> Result { + let mut core_request = decode_request(OcrWireRequest { + model: request.model.into(), + document: request.document, + api_key: request.api_key.map(str::to_string), + api_base: request.api_base.map(str::to_string), + custom_llm_provider: request.custom_llm_provider.map(str::to_string), + extra_headers: request.extra_headers, + optional_params: request.optional_params, + timeout_seconds: request.timeout.map(|timeout| timeout.as_secs_f64()), + })?; + core_request.connection.max_download_bytes = litellm_core::ocr::types::download_limit_bytes( + std::env::var("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB") + .ok() + .as_deref(), + ); + core_request.litellm_call_id = request.litellm_call_id.map(str::to_string); + core_request.hooks = Arc::new(hooks::OcrGatewayHooks::new( + request.callbacks, + request.guardrails, + request.request_metadata, + )); + client + .perform(core_request) .await + .map(|response| response.into_json()) } #[cfg(test)] @@ -29,9 +44,14 @@ mod tests { use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; - use super::{OcrRequest, ocr}; + use super::{OcrRequest, ocr as run_ocr}; use crate::integrations::types::RequestMetadata; + async fn ocr(request: OcrRequest<'_>) -> Result { + let client = litellm_core::ocr::OcrClient::new(reqwest::Client::new())?; + run_ocr(&client, request).await + } + async fn read_http_request(socket: &mut TcpStream) -> String { let mut request = Vec::new(); let mut buffer = [0_u8; 1024]; @@ -150,7 +170,7 @@ mod tests { assert_eq!(response["pages"][1]["markdown"], "Page 2 block A"); assert_eq!(response["pages"][2]["markdown"], "Page 3 block A"); assert_eq!(response["usage_info"]["pages_processed"], 3); - assert_eq!(response["usage_info"]["credits"], 3); + assert_eq!(response["usage_info"]["credits"], 3.0); assert_eq!(response["provider_native_response"]["job_id"], "job_123"); let (upload_request, parse_request) = server.await.expect("server task completes"); assert!( diff --git a/litellm-rust/crates/ai-gateway/src/ocr/prepare.rs b/litellm-rust/crates/ai-gateway/src/ocr/prepare.rs deleted file mode 100644 index fa9ca1a193e..00000000000 --- a/litellm-rust/crates/ai-gateway/src/ocr/prepare.rs +++ /dev/null @@ -1,163 +0,0 @@ -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{SystemTime, UNIX_EPOCH}; - -use litellm_core::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; -use serde_json::{Map, Value}; - -use super::common_utils::ocr_provider_config; -use super::hooks::OcrLifecycleHooks; -use super::types::{OcrRequest, PreparedOcrRequest}; -use crate::integrations::custom_guardrail::CustomGuardrailRunner; -use crate::integrations::custom_logger::CustomLoggerRunner; - -pub(crate) struct PreparedOcrCall { - pub(crate) request: PreparedOcrRequest, - pub(crate) hooks: OcrLifecycleHooks, -} - -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -pub(crate) fn prepare_ocr_call(request: OcrRequest<'_>) -> PreparedOcrCall { - let call_id = request - .litellm_call_id - .map(str::to_string) - .unwrap_or_else(new_ocr_call_id); - let provider_info = get_custom_llm_provider(request.model, request.custom_llm_provider) - .unwrap_or(CustomLlmProvider { - model: request.model, - custom_llm_provider: "mistral", - }); - let model = provider_info.model.to_string(); - let custom_llm_provider = provider_info.custom_llm_provider.to_string(); - let config = ocr_provider_config(&custom_llm_provider, &model) - .ok_or_else(|| litellm_core::Error::InvalidProvider(custom_llm_provider.clone())) - .and_then(|config| { - validate_request_format(config, &request.optional_params, &custom_llm_provider)?; - Ok(config) - }); - let optional_params = match &config { - Ok(config) => { - let supported = config.supported_ocr_params(); - let mut mapped = config.map_ocr_params( - &request - .optional_params - .iter() - .filter(|(name, _)| supported.contains(&name.as_str())) - .map(|(name, value)| (name.clone(), value.clone())) - .collect(), - ); - for name in [ - "vertex_project", - "vertex_ai_project", - "vertex_location", - "vertex_ai_location", - ] { - if let Some(value) = request.optional_params.get(name) { - mapped.insert(name.to_string(), value.clone()); - } - } - mapped - } - Err(_) => request.optional_params, - }; - - PreparedOcrCall { - request: PreparedOcrRequest { - config, - model, - custom_llm_provider, - litellm_call_id: call_id, - document: request.document, - api_key: request.api_key.map(str::to_string), - api_base: request.api_base.map(str::to_string), - extra_headers: request.extra_headers, - optional_params, - timeout: request.timeout, - }, - hooks: OcrLifecycleHooks::new( - CustomLoggerRunner::new(request.callbacks), - CustomGuardrailRunner::new(request.guardrails), - request.request_metadata, - ), - } -} - -fn validate_request_format( - config: &'static dyn litellm_core::ocr::transformation::OcrProviderConfig, - optional_params: &Map, - provider: &str, -) -> Result<(), litellm_core::Error> { - let Some(format) = optional_params.get("req_format") else { - return Ok(()); - }; - match format.as_str() { - Some("litellm") => Ok(()), - Some("native") if config.supported_ocr_params().contains(&"req_format") => Ok(()), - Some("native") => Err(litellm_core::Error::InvalidRequest(format!( - "`req_format=native` is not supported for provider {provider}" - ))), - _ => Err(litellm_core::Error::InvalidRequest(format!( - "Invalid `req_format`: {format}. Expected `litellm` or `native`" - ))), - } -} - -fn new_ocr_call_id() -> String { - static COUNTER: AtomicU64 = AtomicU64::new(1); - let sequence = COUNTER.fetch_add(1, Ordering::Relaxed); - let timestamp = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|duration| duration.as_nanos()) - .unwrap_or(0); - format!("ocr-{timestamp}-{sequence}") -} - -#[cfg(test)] -mod tests { - use litellm_core::error::Error; - use serde_json::{Map, json}; - - use super::{OcrRequest, prepare_ocr_call}; - use crate::integrations::types::RequestMetadata; - - fn base_ocr_request(model: &str) -> OcrRequest<'_> { - OcrRequest { - model, - document: json!({ - "type": "document_url", - "document_url": "https://example.com/doc.pdf" - }), - api_key: Some("sk-test"), - api_base: None, - custom_llm_provider: None, - extra_headers: None, - optional_params: Map::new(), - timeout: None, - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: RequestMetadata::default(), - litellm_call_id: None, - } - } - - fn request_with_format(format: &str) -> OcrRequest<'_> { - let mut request = base_ocr_request("mistral/mistral-ocr-latest"); - request.optional_params = Map::from_iter([("req_format".to_string(), json!(format))]); - request - } - - #[test] - fn native_format_rejected_for_provider_without_support_as_bad_request() { - let prepared = prepare_ocr_call(request_with_format("native")); - assert!( - matches!(prepared.request.config, Err(Error::InvalidRequest(message)) if message.contains("not supported for provider")) - ); - } - - #[test] - fn unknown_format_rejected_for_provider_without_support_as_bad_request() { - let prepared = prepare_ocr_call(request_with_format("raw")); - assert!( - matches!(prepared.request.config, Err(Error::InvalidRequest(message)) if message.contains("Invalid `req_format`")) - ); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/ocr/types.rs b/litellm-rust/crates/ai-gateway/src/ocr/types.rs index 75a8e61ddbf..5dba2e6708d 100644 --- a/litellm-rust/crates/ai-gateway/src/ocr/types.rs +++ b/litellm-rust/crates/ai-gateway/src/ocr/types.rs @@ -1,13 +1,9 @@ -use std::sync::Arc; -use std::time::Duration; - -use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleRequest}; -use litellm_core::ocr::transformation::OcrProviderConfig; -use serde_json::{Map, Value}; - use crate::integrations::custom_guardrail::CustomGuardrail; use crate::integrations::custom_logger::CustomLogger; use crate::integrations::types::RequestMetadata; +use serde_json::{Map, Value}; +use std::sync::Arc; +use std::time::Duration; pub struct OcrRequest<'a> { pub model: &'a str, @@ -23,37 +19,3 @@ pub struct OcrRequest<'a> { pub request_metadata: RequestMetadata, pub litellm_call_id: Option<&'a str>, } - -pub(crate) struct PreparedOcrRequest { - pub(crate) config: Result<&'static dyn OcrProviderConfig, litellm_core::Error>, - pub(crate) model: String, - pub(crate) custom_llm_provider: String, - pub(crate) litellm_call_id: String, - pub(crate) document: Value, - pub(crate) api_key: Option, - pub(crate) api_base: Option, - pub(crate) extra_headers: Option>, - pub(crate) optional_params: Map, - pub(crate) timeout: Option, -} - -impl CallLifecycleRequest for PreparedOcrRequest { - fn lifecycle_context(&self) -> CallLifecycleContext { - CallLifecycleContext::new( - "ocr", - self.model.clone(), - self.custom_llm_provider.clone(), - self.litellm_call_id.clone(), - ) - } -} - -pub(crate) struct ProviderOcrRequest { - pub(crate) model: String, - pub(crate) config: &'static dyn OcrProviderConfig, - pub(crate) url: String, - pub(crate) body: Value, - pub(crate) optional_params: Map, - pub(crate) upstream_headers: Vec<(String, String)>, - pub(crate) timeout: Option, -} diff --git a/litellm-rust/crates/ai-gateway/tests/ocr_lifecycle.rs b/litellm-rust/crates/ai-gateway/tests/ocr_lifecycle.rs index 60e90ed2a7c..465d3b93901 100644 --- a/litellm-rust/crates/ai-gateway/tests/ocr_lifecycle.rs +++ b/litellm-rust/crates/ai-gateway/tests/ocr_lifecycle.rs @@ -1,5 +1,8 @@ +use serde_json::{Map, Value, json}; use std::sync::{Arc, Mutex}; use std::time::Duration; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; use litellm_ai_gateway::integrations::custom_guardrail::{ CustomGuardrail, GuardrailContext, GuardrailDecision, GuardrailError, GuardrailEventHook, @@ -9,16 +12,19 @@ use litellm_ai_gateway::integrations::custom_logger::{ CallbackTiming, CallbackValue, CustomLogger, LogFuture, ModelCallDetails, }; use litellm_ai_gateway::integrations::types::RequestMetadata; -use litellm_ai_gateway::ocr::{OcrRequest, ocr}; +use litellm_ai_gateway::ocr::{OcrRequest, ocr as run_ocr}; use litellm_core::error::Error; + #[cfg(feature = "trace-parity")] use litellm_core::observability::FunctionTrace; -use serde_json::{Map, Value, json}; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::{TcpListener, TcpStream}; #[cfg(feature = "trace-parity")] use tracing::instrument::WithSubscriber; +async fn ocr(request: OcrRequest<'_>) -> Result { + let client = litellm_core::ocr::OcrClient::new(reqwest::Client::new())?; + run_ocr(&client, request).await +} + async fn read_http_headers(socket: &mut TcpStream) -> String { let mut request = Vec::new(); let mut buffer = [0_u8; 1024]; @@ -193,7 +199,7 @@ impl CustomGuardrail for RecordingOcrGuardrail { "blocked before provider", ))); } - request.data["document"]["guarded_pre"] = json!(true); + request.data["optional_params"]["include_image_base64"] = json!(true); Ok(GuardrailDecision::Mask(request)) }) } @@ -379,7 +385,10 @@ async fn ocr_lifecycle_runs_pre_during_and_success_hooks() { ); let request = server.await.expect("server task completes"); - assert!(request.contains(r#""guarded_pre":true"#), "{request}"); + assert!( + request.contains(r#""include_image_base64":true"#), + "{request}" + ); assert!(request.contains(r#""guarded_during":true"#), "{request}"); } @@ -569,6 +578,127 @@ async fn ocr_does_not_duplicate_authorization_header_when_header_is_supplied() { ); } +#[tokio::test] +async fn azure_ai_mistral_ocr_sends_api_key_as_bearer() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("test listener binds"); + let addr = listener.local_addr().expect("listener has local addr"); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accepts request"); + let request = read_http_request(&mut socket).await; + let response_body = + r#"{"pages":[{"index":0,"markdown":"ok"}],"model":"mistral-ocr-latest"}"#; + let response = format!( + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", + response_body.len(), + response_body + ); + socket + .write_all(response.as_bytes()) + .await + .expect("writes response"); + request + }); + + let response = ocr(OcrRequest { + model: "mistral-ocr-latest", + document: json!({ + "type": "document_url", + "document_url": "data:application/pdf;base64,YWJj" + }), + api_key: Some("azure-ai-key"), + api_base: Some(&format!("http://{addr}")), + custom_llm_provider: Some("azure_ai"), + extra_headers: None, + optional_params: Map::new(), + timeout: Some(Duration::from_secs(5)), + callbacks: Vec::new(), + guardrails: Vec::new(), + request_metadata: RequestMetadata::default(), + litellm_call_id: None, + }) + .await + .expect("Azure AI OCR request succeeds"); + + assert_eq!(response["pages"][0]["markdown"], "ok"); + + let request = server.await.expect("server task completes"); + assert!( + request + .to_ascii_lowercase() + .contains("authorization: bearer azure-ai-key"), + "{request}" + ); + assert!( + !request + .to_ascii_lowercase() + .contains("api-key: azure-ai-key"), + "{request}" + ); +} + +#[tokio::test] +async fn azure_ai_entra_token_survives_pre_call_guardrail_mapping() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("test listener binds"); + let addr = listener.local_addr().expect("listener has local addr"); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accepts request"); + let request = read_http_request(&mut socket).await; + let response_body = + r#"{"pages":[{"index":0,"markdown":"ok"}],"model":"mistral-ocr-latest"}"#; + let response = format!( + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", + response_body.len(), + response_body + ); + socket + .write_all(response.as_bytes()) + .await + .expect("writes response"); + request + }); + let guardrail = Arc::new(RecordingOcrGuardrail::new(vec![ + GuardrailEventHook::PreCall, + ])); + + let response = ocr(OcrRequest { + model: "mistral-ocr-latest", + document: json!({ + "type": "document_url", + "document_url": "data:application/pdf;base64,YWJj" + }), + api_key: None, + api_base: Some(&format!("http://{addr}")), + custom_llm_provider: Some("azure_ai"), + extra_headers: None, + optional_params: json!({"azure_ad_token": "entra-token"}) + .as_object() + .unwrap() + .clone(), + timeout: Some(Duration::from_secs(5)), + callbacks: Vec::new(), + guardrails: vec![guardrail.clone()], + request_metadata: RequestMetadata::default(), + litellm_call_id: None, + }) + .await + .expect("Azure AI OCR request succeeds"); + + assert_eq!(response["pages"][0]["markdown"], "ok"); + assert_eq!(guardrail.events(), vec!["async_pre_call_hook"]); + let request = server.await.expect("server task completes"); + assert!( + request + .to_ascii_lowercase() + .contains("authorization: bearer entra-token"), + "{request}" + ); + assert!(!request.contains("azure_ad_token"), "{request}"); +} + #[tokio::test] async fn document_intelligence_poll_uses_resolved_subscription_key() { let listener = TcpListener::bind("127.0.0.1:0") diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index eb6b5ecfc25..f048e4f71ee 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -4,16 +4,23 @@ version = "0.1.0" edition.workspace = true license.workspace = true repository.workspace = true +autotests = false + +[[test]] +name = "workspace_crate_allowlist" +path = "tests/workspace_crate_allowlist.rs" [dependencies] azure_core.workspace = true azure_identity.workspace = true base64.workspace = true +data-url = "0.3.2" moka.workspace = true rand.workspace = true reqwest.workspace = true serde.workspace = true serde_json.workspace = true +serde_path_to_error = "0.1" strum.workspace = true subtle.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/core/src/auth/azure/mod.rs b/litellm-rust/crates/core/src/auth/azure/mod.rs index b1da3265b77..5e6fbc6f965 100644 --- a/litellm-rust/crates/core/src/auth/azure/mod.rs +++ b/litellm-rust/crates/core/src/auth/azure/mod.rs @@ -1,9 +1,3 @@ -#![allow( - dead_code, - unused_imports, - reason = "used by the OCR architecture in the next stacked PR" -)] - mod credential_provider_cache; mod native; mod resolve; diff --git a/litellm-rust/crates/core/src/constants.rs b/litellm-rust/crates/core/src/constants.rs index 1feb9a951ec..885a9895696 100644 --- a/litellm-rust/crates/core/src/constants.rs +++ b/litellm-rust/crates/core/src/constants.rs @@ -44,8 +44,21 @@ pub const EMPTY_TEXT_PLACEHOLDER: &str = pub const FUNCTION_TRACE_TARGET: &str = "litellm::function_trace"; -#[allow( - dead_code, - reason = "used by the OCR architecture in the next stacked PR" -)] pub(crate) const MEDIA_CONNECT_TIMEOUT_SECS: u64 = 10; + +pub(crate) const OCR_HTTP_TIMEOUT_SECS: u64 = 600; +pub(crate) const OCR_CONNECT_TIMEOUT_SECS: u64 = 10; +pub(crate) const OCR_POLL_TIMEOUT_SECS: u64 = 120; +pub(crate) const OCR_POLL_RETRY_SECS: u64 = 2; +pub(crate) const OCR_INLINE_MAX_BYTES: usize = 50 * 1024 * 1024; +pub(crate) const OCR_DOWNLOAD_MAX_BYTES: u64 = 50 * 1024 * 1024; +pub(crate) const OCR_MAX_FETCH_REDIRECTS: usize = 10; +pub(crate) const AZURE_DI_API_VERSION: &str = "2024-11-30"; +pub(crate) const AZURE_DI_DEFAULT_DPI: i64 = 96; +pub(crate) const AZURE_DI_DEFAULT_WIDTH: f64 = 8.5; +pub(crate) const AZURE_DI_DEFAULT_HEIGHT: f64 = 11.0; +pub(crate) const AZURE_DI_SUBSCRIPTION_HEADER: &str = "Ocp-Apim-Subscription-Key"; +pub(crate) const AZURE_AI_OCR_PATH: &str = "/providers/mistral/azure/ocr"; +pub(crate) const MISTRAL_OCR_API_BASE: &str = "https://api.mistral.ai/v1"; +pub(crate) const REDUCTO_OCR_API_BASE: &str = "https://platform.reducto.ai"; +pub(crate) const REDUCTO_ID_PREFIX: &str = "reducto://"; diff --git a/litellm-rust/crates/core/src/error.rs b/litellm-rust/crates/core/src/error.rs index 3157a956d3b..75538f0375a 100644 --- a/litellm-rust/crates/core/src/error.rs +++ b/litellm-rust/crates/core/src/error.rs @@ -38,10 +38,6 @@ pub enum Error { Unsupported(&'static str), } -#[allow( - dead_code, - reason = "used by the OCR architecture in the next stacked PR" -)] #[derive(Debug, ThisError)] pub(crate) enum MediaError { #[error("media URL rejected by network policy")] @@ -92,6 +88,79 @@ impl From for TransportError { } } +impl From for Error { + fn from(error: AuthError) -> Self { + Self::Auth(error.to_string()) + } +} + +impl From for Error { + fn from(error: crate::ocr::error::OcrRequestError) -> Self { + match error { + crate::ocr::error::OcrRequestError::MissingField(field) => Self::MissingField(field), + error => Self::InvalidRequest(error.to_string()), + } + } +} + +impl From for Error { + fn from(error: crate::ocr::error::OcrResponseError) -> Self { + Self::InvalidResponse(error.to_string()) + } +} + +impl From for Error { + fn from(error: crate::ocr::error::OcrPollingError) -> Self { + match error { + crate::ocr::error::OcrPollingError::PollTimeout => Self::Network(error.to_string()), + error => Self::InvalidResponse(error.to_string()), + } + } +} + +impl From for Error { + fn from(error: TransportError) -> Self { + match error { + TransportError::Http { status, body } => Self::Http { status, body }, + TransportError::Network(message) => Self::Network(message), + TransportError::Connect(message) => Self::Connect(message), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ErrorKind { + InvalidType, + MissingField, + InvalidRequest, + InvalidResponse, + InvalidProvider, + Auth, + Http, + Network, + Connect, + Routing, + Unsupported, +} + +impl Error { + pub fn kind(&self) -> ErrorKind { + match self { + Self::InvalidType { .. } => ErrorKind::InvalidType, + Self::MissingField(_) => ErrorKind::MissingField, + Self::InvalidRequest(_) => ErrorKind::InvalidRequest, + Self::InvalidResponse(_) => ErrorKind::InvalidResponse, + Self::InvalidProvider(_) => ErrorKind::InvalidProvider, + Self::Auth(_) => ErrorKind::Auth, + Self::Http { .. } => ErrorKind::Http, + Self::Network(_) => ErrorKind::Network, + Self::Connect(_) => ErrorKind::Connect, + Self::Routing(_) => ErrorKind::Routing, + Self::Unsupported(_) => ErrorKind::Unsupported, + } + } +} + pub fn json_type_name(value: &serde_json::Value) -> &'static str { match value { serde_json::Value::Null => "null", diff --git a/litellm-rust/crates/core/src/http_utils.rs b/litellm-rust/crates/core/src/http_utils.rs index 1c85c962c19..25bf6e26454 100644 --- a/litellm-rust/crates/core/src/http_utils.rs +++ b/litellm-rust/crates/core/src/http_utils.rs @@ -5,20 +5,12 @@ use serde_json::{Map, Value}; use crate::constants::UPSTREAM_ERROR_BODY_MAX_CHARS; use crate::error::{Error, json_type_name}; -#[allow( - dead_code, - reason = "used by the OCR architecture in the next stacked PR" -)] pub(crate) enum HeaderPolicy<'a> { All, Only(&'a [&'a str]), Except(&'a [&'a str]), } -#[allow( - dead_code, - reason = "used by the OCR architecture in the next stacked PR" -)] pub(crate) fn with_headers( builder: reqwest::RequestBuilder, headers: &[(String, String)], @@ -96,20 +88,6 @@ pub fn has_bearer_auth(headers: &[(String, String)]) -> bool { }) } -#[allow( - dead_code, - reason = "used by the OCR architecture in the next stacked PR" -)] -pub(crate) fn deserialize_optional_param<'de, D, T>( - deserializer: D, -) -> Result>, D::Error> -where - D: serde::Deserializer<'de>, - T: serde::Deserialize<'de>, -{ - as serde::Deserialize>::deserialize(deserializer).map(Some) -} - #[cfg(test)] mod tests { use super::*; diff --git a/litellm-rust/crates/core/src/media.rs b/litellm-rust/crates/core/src/media.rs index 47c83e2634b..4412a289f63 100644 --- a/litellm-rust/crates/core/src/media.rs +++ b/litellm-rust/crates/core/src/media.rs @@ -1,8 +1,3 @@ -#![allow( - dead_code, - reason = "used by the OCR architecture in the next stacked PR" -)] - use std::io; use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; diff --git a/litellm-rust/crates/core/src/ocr/backends/azure_ai.rs b/litellm-rust/crates/core/src/ocr/backends/azure_ai.rs new file mode 100644 index 00000000000..24139352253 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/backends/azure_ai.rs @@ -0,0 +1,72 @@ +use serde_json::{Map, Value}; + +use crate::Error; +use crate::auth::azure::AzureAuthInputs; +use crate::ocr::backends::OcrBackend; +use crate::ocr::error::OcrError; +use crate::ocr::types::OcrConnection; +use crate::providers::azure_ai::auth; + +#[derive(Clone, Debug)] +pub struct AzureBackend; + +impl OcrBackend for AzureBackend { + type Config = AzureAuthInputs; + const PROVIDER: crate::ocr::registry::OcrProvider = crate::ocr::registry::OcrProvider::AzureAi; + + fn decode_config(params: &Map) -> Result { + Ok(AzureAuthInputs::from_optional_params(params)?) + } +} + +pub(crate) fn resolve_integration( + model: &crate::ocr::registry::OcrModel, +) -> crate::ocr::registry::OcrIntegrationKind { + let model = model.as_str().to_ascii_lowercase(); + if model.contains("doc-intelligence") || model.contains("documentintelligence") { + crate::ocr::registry::OcrIntegrationKind::AzureDocumentIntelligence + } else { + crate::ocr::registry::OcrIntegrationKind::AzureMistral + } +} + +pub(crate) async fn authenticate_mistral( + connection: &OcrConnection, + config: &AzureAuthInputs, + env_lookup: &(dyn Fn(&str) -> Option + Sync), +) -> Result, OcrError> { + Ok(auth::authenticate( + connection.extra_headers.clone(), + connection.api_key.as_deref(), + Some(config), + env_lookup, + ) + .await?) +} + +pub(crate) async fn prepare_document_intelligence_auth( + connection: &OcrConnection, + config: &AzureAuthInputs, + env_lookup: &(dyn Fn(&str) -> Option + Sync), +) -> Result<(String, Vec<(String, String)>), OcrError> { + let endpoint = + auth::resolve_document_intelligence_endpoint(connection.api_base.as_deref(), env_lookup)?; + let headers = auth::authenticate_document_intelligence( + connection.extra_headers.clone(), + connection.api_key.as_deref(), + Some(config), + env_lookup, + ) + .await?; + Ok((endpoint, headers)) +} + +pub(crate) fn resolve_mistral_api_base( + connection: &OcrConnection, + env_lookup: &(dyn Fn(&str) -> Option + Sync), +) -> Result { + Ok(auth::resolve_api_base( + connection.api_base.as_deref(), + env_lookup, + )?) +} diff --git a/litellm-rust/crates/core/src/ocr/backends/azure_document_intelligence/mod.rs b/litellm-rust/crates/core/src/ocr/backends/azure_document_intelligence/mod.rs new file mode 100644 index 00000000000..10fc41d55ce --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/backends/azure_document_intelligence/mod.rs @@ -0,0 +1 @@ +pub(crate) mod polling; diff --git a/litellm-rust/crates/core/src/ocr/backends/azure_document_intelligence/polling.rs b/litellm-rust/crates/core/src/ocr/backends/azure_document_intelligence/polling.rs new file mode 100644 index 00000000000..530499f4c5b --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/backends/azure_document_intelligence/polling.rs @@ -0,0 +1,99 @@ +use crate::ocr::error::OcrError; +use crate::ocr::error::OcrPollingError; +use crate::ocr::error::OcrResponseError; +use std::time::Duration; + +use crate::constants::{AZURE_DI_SUBSCRIPTION_HEADER, OCR_POLL_RETRY_SECS}; +use crate::ocr::client::read_json_response; +use crate::ocr::formats::document_intelligence::types::{ + AzureDocumentIntelligenceOperation, OperationStatus, +}; +use crate::ocr::types::OcrConnection; +use crate::ocr::wire::DecodedOcrResponse; +use reqwest::Url; +use tokio::time::Instant; + +pub(crate) async fn read_operation_response( + http_client: &reqwest::Client, + response: reqwest::Response, + original_url: &str, + headers: &[(String, String)], + connection: &OcrConnection, + native: bool, +) -> Result, OcrError> { + if response.status() != reqwest::StatusCode::ACCEPTED { + return read_json_response(response, native).await; + } + let location = response + .headers() + .get("operation-location") + .and_then(|value| value.to_str().ok()) + .ok_or(OcrPollingError::PollLocation)?; + let original = Url::parse(original_url).map_err(|_| OcrPollingError::PollOrigin)?; + let operation = Url::parse(location).map_err(|_| OcrPollingError::PollOrigin)?; + if original.origin() != operation.origin() + || !operation.username().is_empty() + || operation.password().is_some() + { + return Err(OcrPollingError::PollOrigin.into()); + } + poll_document_intelligence(http_client, operation, headers, connection, native).await +} + +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +async fn poll_document_intelligence( + http_client: &reqwest::Client, + url: Url, + headers: &[(String, String)], + connection: &OcrConnection, + native: bool, +) -> Result, OcrError> { + let deadline = Instant::now() + .checked_add(connection.poll_timeout) + .ok_or(OcrPollingError::PollTimeout)?; + loop { + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|remaining| !remaining.is_zero()) + .ok_or(OcrPollingError::PollTimeout)?; + let builder = http_client + .get(url.clone()) + .timeout(remaining.min(connection.timeout)); + let builder = crate::http_utils::with_headers( + builder, + headers, + crate::http_utils::HeaderPolicy::Only(&[AZURE_DI_SUBSCRIPTION_HEADER, "authorization"]), + ); + let response = tokio::time::timeout_at(deadline, crate::http_utils::http_request(builder)) + .await + .map_err(|_| OcrPollingError::PollTimeout)? + .map_err(crate::error::TransportError::from)?; + let retry = response + .headers() + .get(reqwest::header::RETRY_AFTER) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .unwrap_or(OCR_POLL_RETRY_SECS); + let decoded: DecodedOcrResponse = + tokio::time::timeout_at(deadline, read_json_response(response, native)) + .await + .map_err(|_| OcrPollingError::PollTimeout)??; + match &decoded.data.status { + Some(OperationStatus::Succeeded) => return Ok(decoded), + Some(OperationStatus::Running | OperationStatus::NotStarted) => { + tokio::time::timeout_at(deadline, tokio::time::sleep(Duration::from_secs(retry))) + .await + .map_err(|_| OcrPollingError::PollTimeout)?; + } + status => { + return Err(OcrResponseError::OperationStatus( + status + .as_ref() + .map(|s| s.to_string()) + .unwrap_or_else(|| "None".into()), + ) + .into()); + } + } + } +} diff --git a/litellm-rust/crates/core/src/ocr/backends/mistral.rs b/litellm-rust/crates/core/src/ocr/backends/mistral.rs new file mode 100644 index 00000000000..8a627dc27ff --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/backends/mistral.rs @@ -0,0 +1,48 @@ +use serde_json::{Map, Value}; + +use crate::Error; +use crate::constants::MISTRAL_OCR_API_BASE; +use crate::ocr::backends::OcrBackend; +use crate::ocr::error::{OcrError, OcrRequestError}; +use crate::ocr::types::OcrConnection; +use crate::providers::mistral::auth; +use crate::url_utils::ApiUrl; + +#[derive(Clone, Debug)] +pub struct MistralBackend; + +impl OcrBackend for MistralBackend { + type Config = (); + const PROVIDER: crate::ocr::registry::OcrProvider = crate::ocr::registry::OcrProvider::Mistral; + + fn decode_config(_params: &Map) -> Result { + Ok(()) + } +} + +pub(crate) fn complete_url(api_base: Option<&str>) -> Result { + let base = api_base + .map(str::trim) + .filter(|base| !base.is_empty()) + .unwrap_or(MISTRAL_OCR_API_BASE); + ApiUrl::parse(base) + .and_then(|url| url.complete_path(&["v1", "ocr"])) + .map(|url| url.into_string()) + .map_err(|_| { + OcrRequestError::RequestField { + path: "api_base".into(), + } + .into() + }) +} + +pub(crate) fn authenticate( + connection: &OcrConnection, + env_lookup: &(dyn Fn(&str) -> Option + Sync), +) -> Result, OcrError> { + Ok(auth::validate_environment( + connection.extra_headers.clone(), + connection.api_key.as_deref(), + env_lookup, + )?) +} diff --git a/litellm-rust/crates/core/src/ocr/backends/mod.rs b/litellm-rust/crates/core/src/ocr/backends/mod.rs new file mode 100644 index 00000000000..f56513b0e95 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/backends/mod.rs @@ -0,0 +1,20 @@ +pub(crate) mod azure_ai; +pub(crate) mod azure_document_intelligence; +pub(crate) mod mistral; +pub(crate) mod reducto; + +use serde_json::{Map, Value}; + +use crate::Error; + +pub struct PreparedOcrBackend { + pub url: String, + pub headers: Vec<(String, String)>, +} + +pub trait OcrBackend: Send + Sync + 'static { + type Config: Clone + std::fmt::Debug + Send + Sync + 'static; + const PROVIDER: super::registry::OcrProvider; + + fn decode_config(params: &Map) -> Result; +} diff --git a/litellm-rust/crates/core/src/ocr/backends/reducto.rs b/litellm-rust/crates/core/src/ocr/backends/reducto.rs new file mode 100644 index 00000000000..5cc2121ec27 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/backends/reducto.rs @@ -0,0 +1,111 @@ +use crate::Error; +use crate::constants::{REDUCTO_ID_PREFIX, REDUCTO_OCR_API_BASE}; +use crate::ocr::backends::OcrBackend; +use crate::ocr::document::{DocumentPreparation, InlineDocument}; +use crate::ocr::error::{OcrError, OcrRequestError}; +use crate::ocr::formats::reducto::types::{ReductoFileId, ReductoUploadResponse}; +use crate::ocr::types::{OcrConnection, OcrDocument}; +use crate::providers::reducto::auth; +use crate::url_utils::ApiUrl; +use serde_json::{Map, Value}; + +#[derive(Clone, Debug)] +pub struct ReductoBackend; + +impl OcrBackend for ReductoBackend { + type Config = (); + const PROVIDER: crate::ocr::registry::OcrProvider = crate::ocr::registry::OcrProvider::Reducto; + + fn decode_config(_params: &Map) -> Result { + Ok(()) + } +} + +pub(crate) fn resolve_integration( + model: &crate::ocr::registry::OcrModel, +) -> crate::ocr::registry::OcrIntegrationKind { + match model { + crate::ocr::registry::OcrModel::ReductoLegacy => { + crate::ocr::registry::OcrIntegrationKind::ReductoLegacy + } + _ => crate::ocr::registry::OcrIntegrationKind::ReductoV3, + } +} + +pub(crate) fn complete_url(api_base: Option<&str>, path: &str) -> Result { + let base = api_base + .map(str::trim) + .filter(|base| !base.is_empty()) + .unwrap_or(REDUCTO_OCR_API_BASE); + ApiUrl::parse(base) + .and_then(|url| url.complete_path(&[path])) + .map(|url| url.into_string()) + .map_err(|_| { + OcrRequestError::RequestField { + path: "api_base".into(), + } + .into() + }) +} + +pub struct ReductoUpload; + +impl DocumentPreparation for ReductoUpload { + type Output = ReductoFileId; + + async fn prepare( + client: &crate::ocr::OcrClient, + document: OcrDocument, + connection: &OcrConnection, + headers: &[(String, String)], + ) -> Result { + let source = document.source(); + if source.starts_with(REDUCTO_ID_PREFIX) { + return Ok(ReductoFileId(source.to_string())); + } + let document = InlineDocument::parse(source)?.ok_or(OcrRequestError::ReductoSource)?; + upload_document(client, document, connection, headers).await + } +} + +async fn upload_document( + client: &crate::ocr::OcrClient, + document: InlineDocument<'_>, + connection: &OcrConnection, + headers: &[(String, String)], +) -> Result { + let mime = document.mime_type().to_string(); + let bytes = document.decode(crate::constants::OCR_INLINE_MAX_BYTES)?; + let part = reqwest::multipart::Part::bytes(bytes) + .file_name("document") + .mime_str(&mime) + .map_err(|_| OcrRequestError::InvalidDataUri)?; + let builder = client + .provider_http() + .post(complete_url(connection.api_base.as_deref(), "upload")?) + .multipart(reqwest::multipart::Form::new().part("file", part)) + .timeout(connection.timeout); + let builder = crate::http_utils::with_headers( + builder, + headers, + crate::http_utils::HeaderPolicy::Except(&["content-type", "content-length"]), + ); + let response = crate::http_utils::http_request(builder) + .await + .map_err(crate::error::TransportError::from)?; + let response = crate::ocr::client::read_json_response::(response, false) + .await? + .data; + response.try_into() +} + +pub(crate) fn authenticate( + connection: &OcrConnection, + env_lookup: &(dyn Fn(&str) -> Option + Sync), +) -> Result, OcrError> { + Ok(auth::validate_environment( + connection.extra_headers.clone(), + connection.api_key.as_deref(), + env_lookup, + )?) +} diff --git a/litellm-rust/crates/core/src/ocr/client.rs b/litellm-rust/crates/core/src/ocr/client.rs new file mode 100644 index 00000000000..08ca1e4b853 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/client.rs @@ -0,0 +1,88 @@ +use std::sync::OnceLock; +use std::time::Duration; + +use serde::de::DeserializeOwned; + +use super::error::OcrError; +use super::handler::perform_ocr_request; +use super::types::{OcrRequest, OcrResponseData}; +use super::wire::{DecodedOcrResponse, decode_response}; +use crate::Error; +use crate::constants::OCR_CONNECT_TIMEOUT_SECS; +use crate::error::TransportError; +use crate::media::MediaFetcher; + +#[derive(Clone)] +pub struct OcrClient { + provider_http: reqwest::Client, + document_fetcher: MediaFetcher, +} + +impl OcrClient { + pub fn new(provider_http: reqwest::Client) -> Result { + let document_fetcher = MediaFetcher::new().map_err(TransportError::from)?; + Ok(Self { + provider_http, + document_fetcher, + }) + } + + #[tracing::instrument( + name = "ocr", + target = "litellm::function_trace", + level = "trace", + skip_all + )] + pub async fn perform(&self, request: OcrRequest) -> Result { + perform_ocr_request(self, request).await + } + + pub(crate) fn provider_http(&self) -> &reqwest::Client { + &self.provider_http + } + + pub(crate) fn document_fetcher(&self) -> &MediaFetcher { + &self.document_fetcher + } + + #[cfg(test)] + pub(crate) fn for_test(provider_http: reqwest::Client, document_http: reqwest::Client) -> Self { + Self { + provider_http, + document_fetcher: MediaFetcher::for_test(document_http), + } + } +} + +pub async fn ocr(request: OcrRequest) -> Result { + static CLIENT: OnceLock> = OnceLock::new(); + let client = CLIENT + .get_or_init(|| { + reqwest::Client::builder() + .connect_timeout(Duration::from_secs(OCR_CONNECT_TIMEOUT_SECS)) + .build() + .map_err(TransportError::from) + .and_then(OcrClient::new) + }) + .clone()?; + client.perform(request).await +} + +pub async fn read_json_response( + response: reqwest::Response, + native: bool, +) -> Result, OcrError> { + let status = response.status(); + let bytes = response + .bytes() + .await + .map_err(crate::error::TransportError::from)?; + if !status.is_success() { + return Err(crate::error::TransportError::Http { + status: status.as_u16(), + body: crate::http_utils::truncate_error_body(&String::from_utf8_lossy(&bytes)), + } + .into()); + } + Ok(decode_response(&bytes, native)?) +} diff --git a/litellm-rust/crates/core/src/ocr/document.rs b/litellm-rust/crates/core/src/ocr/document.rs new file mode 100644 index 00000000000..51367ab5172 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/document.rs @@ -0,0 +1,307 @@ +use base64::{Engine, engine::general_purpose::STANDARD}; +use data_url::{DataUrl, DataUrlError, forgiving_base64::DecodeError, mime::Mime}; +use reqwest::Url; + +use super::error::{OcrError, OcrRequestError, OcrResponseError}; +use super::types::{OcrConnection, OcrDocument}; +use crate::constants::OCR_MAX_FETCH_REDIRECTS; +use crate::error::{MediaError, TransportError}; +use crate::media::{DownloadPolicy, MediaFetcher}; + +pub trait DocumentPreparation: Send + Sync + 'static { + type Output: Send; + + fn prepare( + client: &super::OcrClient, + document: OcrDocument, + connection: &OcrConnection, + headers: &[(String, String)], + ) -> impl std::future::Future> + Send; +} + +pub struct PassThrough; + +impl DocumentPreparation for PassThrough { + type Output = OcrDocument; + + async fn prepare( + _client: &super::OcrClient, + document: OcrDocument, + _connection: &OcrConnection, + _headers: &[(String, String)], + ) -> Result { + Ok(document) + } +} + +pub struct RequireInline; + +impl DocumentPreparation for RequireInline { + type Output = InlineOcrDocument; + + async fn prepare( + client: &super::OcrClient, + document: OcrDocument, + connection: &OcrConnection, + _headers: &[(String, String)], + ) -> Result { + inline_remote_document(client.document_fetcher(), document, connection).await + } +} + +pub(crate) struct InlineDocument<'a>(DataUrl<'a>); + +impl<'a> InlineDocument<'a> { + pub(crate) fn parse(source: &'a str) -> Result, OcrRequestError> { + match DataUrl::process(source) { + Ok(url) => Ok(Some(Self(url))), + Err(DataUrlError::NotADataUrl) => Ok(None), + Err(DataUrlError::NoComma) => Err(OcrRequestError::InvalidDataUri), + } + } + + pub(crate) fn mime_type(&self) -> &Mime { + self.0.mime_type() + } + + pub(crate) fn decode(&self, max_bytes: usize) -> Result, OcrRequestError> { + let mut body = Vec::new(); + self.0 + .decode(|bytes| { + if bytes.len() > max_bytes.saturating_sub(body.len()) { + return Err(OcrRequestError::InlineDocumentTooLarge); + } + body.extend_from_slice(bytes); + Ok(()) + }) + .map_err(|error| match error { + DecodeError::InvalidBase64(_) => OcrRequestError::InvalidDataUri, + DecodeError::WriteError(error) => error, + })?; + Ok(body) + } +} + +#[derive(Debug)] +pub struct InlineOcrDocument(OcrDocument); + +impl TryFrom for InlineOcrDocument { + type Error = OcrRequestError; + + fn try_from(document: OcrDocument) -> Result { + let inline = + InlineDocument::parse(document.source())?.ok_or(OcrRequestError::InvalidDataUri)?; + inline.decode(crate::constants::OCR_INLINE_MAX_BYTES)?; + Ok(Self(document)) + } +} + +impl From for OcrDocument { + fn from(document: InlineOcrDocument) -> Self { + document.0 + } +} + +pub(crate) async fn inline_remote_document( + fetcher: &MediaFetcher, + document: OcrDocument, + connection: &OcrConnection, +) -> Result { + let source = document.source(); + if !source.starts_with("http://") && !source.starts_with("https://") { + return Ok(InlineOcrDocument::try_from(document)?); + } + let url = Url::parse(source).map_err(|_| OcrRequestError::RequestField { + path: "document URL".into(), + })?; + let downloaded = fetcher + .fetch( + url, + DownloadPolicy { + timeout: connection.timeout, + max_bytes: connection.max_download_bytes, + max_redirects: OCR_MAX_FETCH_REDIRECTS, + }, + ) + .await + .map_err(map_media_error)?; + Ok(InlineOcrDocument::try_from(document.with_source(format!( + "data:{};base64,{}", + downloaded.content_type, + STANDARD.encode(downloaded.bytes) + )))?) +} + +fn map_media_error(error: MediaError) -> OcrError { + match error { + MediaError::BlockedUrl => OcrRequestError::BlockedDocumentUrl.into(), + MediaError::DownloadDisabled => OcrRequestError::DownloadDisabled.into(), + MediaError::DownloadTooLarge => OcrRequestError::DownloadTooLarge.into(), + MediaError::TooManyRedirects => OcrRequestError::TooManyRedirects.into(), + MediaError::MissingRedirectLocation => OcrResponseError::MissingRedirectLocation.into(), + MediaError::InvalidRedirect => OcrResponseError::InvalidRedirect.into(), + MediaError::Http(status) => TransportError::Http { + status, + body: "OCR document download failed".into(), + } + .into(), + MediaError::Timeout => { + TransportError::Network("OCR document download timed out".into()).into() + } + MediaError::Transport(error) => error.into(), + } +} + +#[cfg(test)] +mod tests { + use super::{InlineDocument, inline_remote_document}; + use crate::ocr::OcrClient; + use crate::ocr::error::OcrRequestError; + use crate::ocr::types::{OcrConnection, OcrDocument}; + use rstest::rstest; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + + #[rstest] + #[case("data:application/pdf;base64,YWJj", b"abc")] + #[case("DATA:application/pdf;BASE64,YWI", b"ab")] + #[case("data:application/pdf;base64,YQ%3D%3D#fragment", b"a")] + #[case("data:application/pdf;base64,Y W J j", b"abc")] + #[case("data:,a%20b%00%FF", b"a b\0\xff")] + #[case("data:;base64,", b"")] + fn decodes_data_urls(#[case] source: &str, #[case] expected: &[u8]) { + let document = InlineDocument::parse(source).unwrap().unwrap(); + assert_eq!(document.decode(expected.len()).unwrap(), expected); + if !expected.is_empty() { + assert_eq!( + document.decode(expected.len() - 1), + Err(OcrRequestError::InlineDocumentTooLarge) + ); + } + } + + #[rstest] + #[case("https://example.com/document.pdf")] + #[case("reducto://document")] + #[case("data:application/pdf;base64,INVALID!")] + #[case("")] + fn inline_document_rejects_unprepared_sources(#[case] source: &str) { + let document = OcrDocument::DocumentUrl { + document_url: source.into(), + }; + assert!(super::InlineOcrDocument::try_from(document).is_err()); + } + + #[test] + fn inline_document_preserves_valid_image_source() { + let document = OcrDocument::ImageUrl { + image_url: "data:image/png;base64,YWJj".into(), + }; + let prepared = super::InlineOcrDocument::try_from(document.clone()).unwrap(); + assert_eq!(OcrDocument::from(prepared), document); + } + + #[test] + fn preserves_mime_parameters_and_uses_standard_default() { + let document = InlineDocument::parse("data:application/pdf;version=1.7;base64,YQ==") + .unwrap() + .unwrap(); + assert!(document.mime_type().matches("application", "pdf")); + assert_eq!(document.mime_type().get_parameter("version"), Some("1.7")); + let default = InlineDocument::parse("data:,a").unwrap().unwrap(); + assert!(default.mime_type().matches("text", "plain")); + assert_eq!( + default.mime_type().get_parameter("charset"), + Some("US-ASCII") + ); + } + + #[rstest] + #[case("data:application/pdf;base64")] + #[case("data:application/pdf;base64,INVALID!")] + #[case("data:application/pdf;base64,Y")] + fn rejects_invalid_input(#[case] source: &str) { + let result = + InlineDocument::parse(source).and_then(|document| document.unwrap().decode(100)); + assert_eq!(result, Err(OcrRequestError::InvalidDataUri)); + } + + #[test] + fn leaves_other_sources_to_the_provider() { + for source in ["https://example.com/file.pdf", "reducto://file.pdf"] { + assert!(InlineDocument::parse(source).unwrap().is_none()); + } + } + + #[test] + fn limits_total_decoded_bytes_across_chunks() { + use base64::{Engine, engine::general_purpose::STANDARD}; + + let bytes = vec![0xff; 8193]; + let source = format!("data:application/pdf;base64,{}", STANDARD.encode(&bytes)); + let document = InlineDocument::parse(&source).unwrap().unwrap(); + assert_eq!(document.decode(bytes.len()).unwrap(), bytes); + assert_eq!( + document.decode(bytes.len() - 1), + Err(OcrRequestError::InlineDocumentTooLarge) + ); + } + + #[tokio::test] + async fn provider_credentials_never_reach_remote_document() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("test listener binds"); + let address = listener.local_addr().expect("listener has address"); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accepts request"); + let mut request = Vec::new(); + let mut buffer = [0_u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let bytes_read = socket.read(&mut buffer).await.expect("reads request"); + assert!(bytes_read > 0); + request.extend_from_slice(&buffer[..bytes_read]); + } + socket + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Type: application/pdf\r\nContent-Length: 3\r\nConnection: close\r\n\r\nabc", + ) + .await + .expect("writes response"); + String::from_utf8(request).expect("request is UTF-8") + }); + let mut provider_headers = reqwest::header::HeaderMap::new(); + provider_headers.insert( + reqwest::header::AUTHORIZATION, + reqwest::header::HeaderValue::from_static("Bearer provider-secret"), + ); + let provider_http = reqwest::Client::builder() + .default_headers(provider_headers) + .build() + .expect("provider client builds"); + let document_http = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("document client builds"); + let client = OcrClient::for_test(provider_http, document_http); + let document = OcrDocument::DocumentUrl { + document_url: format!("http://{address}/document.pdf"), + }; + + let prepared = inline_remote_document( + client.document_fetcher(), + document, + &OcrConnection::default(), + ) + .await + .expect("document is downloaded"); + let request = server.await.expect("server completes"); + + assert_eq!( + OcrDocument::from(prepared).source(), + "data:application/pdf;base64,YWJj" + ); + assert!(!request.to_ascii_lowercase().contains("authorization")); + assert!(!request.contains("provider-secret")); + } +} diff --git a/litellm-rust/crates/core/src/ocr/error.rs b/litellm-rust/crates/core/src/ocr/error.rs new file mode 100644 index 00000000000..f58659d8790 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/error.rs @@ -0,0 +1,104 @@ +use thiserror::Error; + +use crate::auth::AuthError; +use crate::error::TransportError; + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum PagesError { + #[error("`pages` must be integers, not booleans")] + BooleanIndex, + #[error("`pages` integers must be >= 0 (Mistral 0-based indices)")] + NegativeIndex, + #[error("`pages` integer is outside the supported range")] + IndexOutOfRange, + #[error("`pages` must be a list[int] (0-based, Mistral-style) or a string like '1-3,5,7-9'.")] + MixedElementTypes, + #[error("Invalid `pages` for Azure Document Intelligence. Expected tokens like '1' or '3-5'.")] + InvalidNativeRange, +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum OcrRequestError { + #[error("{0}")] + Pages(#[from] PagesError), + #[error( + "Invalid `features` for Azure Document Intelligence. Expected a list of feature names or a comma-separated string." + )] + Features, + #[error("Invalid `req_format`. Expected 'native' or 'litellm'.")] + RequestFormat, + #[error("`req_format=native` is not supported for provider {0}")] + NativeUnsupported(&'static str), + #[error("invalid OCR request field: {path}")] + RequestField { path: String }, + #[error("missing required field: {0}")] + MissingField(&'static str), + #[error("model_id cannot be a dot path segment")] + DotModel, + #[error( + "Reducto requires a reducto:// id or a data URI after OCR preprocessing; plain http(s) URLs are not supported" + )] + ReductoSource, + #[error("Invalid OCR document data URI")] + InvalidDataUri, + #[error("OCR inline document exceeds maximum allowed size")] + InlineDocumentTooLarge, + #[error("OCR document URL rejected by SSRF protection")] + BlockedDocumentUrl, + #[error("Too many redirects while fetching OCR document URL")] + TooManyRedirects, + #[error("OCR document URL download is disabled (MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0)")] + DownloadDisabled, + #[error("OCR document exceeds maximum allowed download size")] + DownloadTooLarge, +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum OcrResponseError { + #[error("invalid OCR response field: {path}")] + ResponseField { path: String }, + #[error("Azure Document Intelligence analysis failed with status: {0}")] + OperationStatus(String), + #[error("OCR numeric value is outside the supported range: {0}")] + NumericRange(&'static str), + #[error("OCR document redirect missing Location header")] + MissingRedirectLocation, + #[error("invalid OCR document redirect")] + InvalidRedirect, +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum OcrPollingError { + #[error("Azure Document Intelligence: rejected cross-origin polling URL")] + PollOrigin, + #[error("Azure Document Intelligence returned 202 but no Operation-Location header found")] + PollLocation, + #[error("Azure Document Intelligence operation polling timed out")] + PollTimeout, +} + +#[derive(Debug, Error)] +pub enum OcrError { + #[error("{0}")] + Request(#[from] OcrRequestError), + #[error("{0}")] + Response(#[from] OcrResponseError), + #[error("{0}")] + Polling(#[from] OcrPollingError), + #[error("{0}")] + Auth(#[from] AuthError), + #[error("{0}")] + Transport(#[from] TransportError), +} + +impl From for crate::Error { + fn from(error: OcrError) -> Self { + match error { + OcrError::Request(error) => error.into(), + OcrError::Response(error) => error.into(), + OcrError::Polling(error) => error.into(), + OcrError::Auth(error) => error.into(), + OcrError::Transport(error) => error.into(), + } + } +} diff --git a/litellm-rust/crates/core/src/ocr/formats/document_intelligence/mod.rs b/litellm-rust/crates/core/src/ocr/formats/document_intelligence/mod.rs new file mode 100644 index 00000000000..db8d9e046e8 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/formats/document_intelligence/mod.rs @@ -0,0 +1,19 @@ +mod params; +mod transformation; +pub mod types; + +use types::*; + +use crate::ocr::formats::ocr_format; +use crate::ocr::types::OcrDocument; + +ocr_format! { + pub AzureDocumentIntelligenceOcrFormat { + params: DocumentIntelligenceInputParams => DocumentIntelligenceParams, + validate: params::validate, + map: params::normalize, + document: OcrDocument, + request: DocumentIntelligenceRequest => transformation::request, + response: AzureDocumentIntelligenceOperation => transformation::response, + } +} diff --git a/litellm-rust/crates/core/src/ocr/formats/document_intelligence/params.rs b/litellm-rust/crates/core/src/ocr/formats/document_intelligence/params.rs new file mode 100644 index 00000000000..37f7f55ddfa --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/formats/document_intelligence/params.rs @@ -0,0 +1,121 @@ +use std::collections::BTreeSet; + +use serde_json::{Map, Value}; + +use super::types::*; +use crate::ocr::error::{OcrRequestError, PagesError}; + +pub(super) fn validate(params: &Map) -> Result<(), OcrRequestError> { + let Some(Value::Array(pages)) = params.get("pages") else { + return Ok(()); + }; + if pages.iter().any(Value::is_boolean) { + return Err(PagesError::BooleanIndex.into()); + } + if pages + .iter() + .any(|page| page.is_number() && page.as_i64().is_none()) + { + return Err(PagesError::IndexOutOfRange.into()); + } + if !pages.iter().all(Value::is_i64) && !pages.iter().all(Value::is_string) { + return Err(PagesError::MixedElementTypes.into()); + } + Ok(()) +} + +fn pages_token_is_valid(token: &str) -> bool { + let mut parts = token.split('-'); + let start = parts.next().unwrap_or_default(); + if start.is_empty() || !start.chars().all(|ch| ch.is_ascii_digit()) { + return false; + } + match parts.next() { + None => true, + Some(end) => { + !end.is_empty() && end.chars().all(|ch| ch.is_ascii_digit()) && parts.next().is_none() + } + } +} + +fn normalize_pages_param(pages: PagesInput) -> Result, PagesError> { + let normalized = match pages { + PagesInput::ZeroBasedIndices(indices) => { + if indices.is_empty() { + return Ok(None); + } + indices + .into_iter() + .map(|page| { + if page < 0 { + return Err(PagesError::NegativeIndex); + } + page.checked_add(1).ok_or(PagesError::IndexOutOfRange) + }) + .collect::, _>>()? + .into_iter() + .map(|page| page.to_string()) + .collect::>() + .join(",") + } + PagesInput::NativeTokens(tokens) => { + if tokens.is_empty() { + return Ok(None); + } + tokens + .iter() + .map(|token| token.trim()) + .collect::>() + .join(",") + } + PagesInput::NativeRange(range) => range + .split(',') + .map(str::trim) + .collect::>() + .join(","), + }; + if !normalized.split(',').all(pages_token_is_valid) { + return Err(PagesError::InvalidNativeRange); + } + Ok(Some(NormalizedPages(normalized))) +} + +fn normalize_features_param( + features: FeaturesInput, +) -> Result, OcrRequestError> { + let tokens = match features { + FeaturesInput::Names(names) => names, + FeaturesInput::CommaSeparated(names) => names.split(',').map(str::to_string).collect(), + }; + if tokens.is_empty() { + return Ok(None); + } + let normalized = tokens.iter().map(|token| token.trim()).collect::>(); + if !normalized.iter().all(|token| { + let Some((first, rest)) = token.as_bytes().split_first() else { + return false; + }; + first.is_ascii_alphabetic() && rest.iter().all(u8::is_ascii_alphanumeric) + }) { + return Err(OcrRequestError::Features); + } + Ok(Some(NormalizedFeatures(normalized.join(",")))) +} + +pub(super) fn normalize( + params: DocumentIntelligenceInputParams, +) -> Result { + Ok(DocumentIntelligenceParams { + pages: params + .pages + .map(normalize_pages_param) + .transpose()? + .flatten(), + features: params + .features + .map(normalize_features_param) + .transpose()? + .flatten(), + request_format: params.req_format.unwrap_or_default(), + }) +} diff --git a/litellm-rust/crates/core/src/ocr/formats/document_intelligence/transformation.rs b/litellm-rust/crates/core/src/ocr/formats/document_intelligence/transformation.rs new file mode 100644 index 00000000000..0ee4e166d0e --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/formats/document_intelligence/transformation.rs @@ -0,0 +1,102 @@ +use base64::{Engine, engine::general_purpose::STANDARD}; + +use super::types::*; +use crate::constants::{AZURE_DI_DEFAULT_DPI, AZURE_DI_DEFAULT_HEIGHT, AZURE_DI_DEFAULT_WIDTH}; +use crate::ocr::document::InlineDocument; +use crate::ocr::error::{OcrRequestError, OcrResponseError}; +use crate::ocr::types::{OcrDocument, OcrPage, OcrPageDimensions, OcrResponseData, OcrUsageInfo}; + +fn pixel_dimension(value: f64, scale: f64, field: &'static str) -> Result { + crate::ocr::wire::checked_truncated_i64(value * scale) + .ok_or(OcrResponseError::NumericRange(field)) +} + +fn convert_dimensions( + width: f64, + height: f64, + unit: &str, +) -> Result { + let scale = if unit == "inch" { + AZURE_DI_DEFAULT_DPI as f64 + } else { + 1.0 + }; + Ok(OcrPageDimensions { + dpi: Some(AZURE_DI_DEFAULT_DPI), + width: Some(pixel_dimension(width, scale, "page.width")?), + height: Some(pixel_dimension(height, scale, "page.height")?), + }) +} + +fn page_markdown(lines: &[AzureDocumentIntelligenceLine]) -> String { + lines + .iter() + .map(|line| line.content.as_deref().unwrap_or_default()) + .collect::>() + .join("\n") +} + +fn transform_azure_page(page: AzureDocumentIntelligencePage) -> Result { + let index = page + .page_number + .unwrap_or(1) + .checked_sub(1) + .ok_or(OcrResponseError::NumericRange("page.pageNumber"))?; + Ok(OcrPage { + dimensions: Some(convert_dimensions( + page.width.unwrap_or(AZURE_DI_DEFAULT_WIDTH), + page.height.unwrap_or(AZURE_DI_DEFAULT_HEIGHT), + page.unit.as_deref().unwrap_or("inch"), + )?), + ..OcrPage::text(index, page_markdown(&page.lines)) + }) +} + +pub(super) fn request( + _model: &str, + document: OcrDocument, + _params: &DocumentIntelligenceParams, +) -> Result { + let source = document.source(); + if source.is_empty() { + return Err(OcrRequestError::MissingField("document URL")); + } + Ok(if let Some(document) = InlineDocument::parse(source)? { + DocumentIntelligenceRequest::Base64Source( + STANDARD.encode(document.decode(crate::constants::OCR_INLINE_MAX_BYTES)?), + ) + } else { + DocumentIntelligenceRequest::UrlSource(source.to_string()) + }) +} + +pub(super) fn response( + model: &str, + response: AzureDocumentIntelligenceOperation, + _params: &DocumentIntelligenceParams, +) -> Result { + if response.status != Some(OperationStatus::Succeeded) { + return Err(OcrResponseError::OperationStatus( + response + .status + .map(|s| s.to_string()) + .unwrap_or_else(|| "None".into()), + )); + } + let result = response.analyze_result.unwrap_or_default(); + let pages = result + .pages + .into_iter() + .map(transform_azure_page) + .collect::, _>>()?; + Ok(OcrResponseData { + usage_info: Some(OcrUsageInfo { + pages_processed: Some(pages.len() as i64), + ..Default::default() + }), + content: result.content, + tables: result.tables, + key_value_pairs: result.key_value_pairs, + ..OcrResponseData::new(model.to_string(), pages) + }) +} diff --git a/litellm-rust/crates/core/src/ocr/formats/document_intelligence/types.rs b/litellm-rust/crates/core/src/ocr/formats/document_intelligence/types.rs new file mode 100644 index 00000000000..06e2f7a22e4 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/formats/document_intelligence/types.rs @@ -0,0 +1,123 @@ +use crate::ocr::types::OcrRequestFormat; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum PagesInput { + ZeroBasedIndices(Vec), + NativeTokens(Vec), + NativeRange(String), +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum FeaturesInput { + Names(Vec), + CommaSeparated(String), +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct DocumentIntelligenceInputParams { + #[serde(skip_serializing_if = "Option::is_none")] + pub pages: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub features: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub req_format: Option, +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct NormalizedPages(pub(crate) String); +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct NormalizedFeatures(pub(crate) String); + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct DocumentIntelligenceParams { + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) pages: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) features: Option, + #[serde(rename = "req_format")] + pub(crate) request_format: OcrRequestFormat, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum DocumentIntelligenceRequest { + #[serde(rename = "urlSource")] + UrlSource(String), + #[serde(rename = "base64Source")] + Base64Source(String), +} + +#[derive(Clone, Debug, PartialEq)] +pub enum OperationStatus { + Succeeded, + Running, + NotStarted, + Failed, + Unknown(String), +} + +impl<'de> Deserialize<'de> for OperationStatus { + fn deserialize>(deserializer: D) -> Result { + Ok(match String::deserialize(deserializer)?.as_str() { + "succeeded" => Self::Succeeded, + "running" => Self::Running, + "notStarted" => Self::NotStarted, + "failed" => Self::Failed, + other => Self::Unknown(other.to_string()), + }) + } +} + +impl std::fmt::Display for OperationStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Succeeded => "succeeded", + Self::Running => "running", + Self::NotStarted => "notStarted", + Self::Failed => "failed", + Self::Unknown(s) => s, + }) + } +} + +#[derive(Clone, Debug, Deserialize)] +pub struct AzureDocumentIntelligenceOperation { + pub status: Option, + #[serde(rename = "analyzeResult")] + pub analyze_result: Option, +} + +#[derive(Clone, Debug, Default, Deserialize)] +pub struct AzureDocumentIntelligenceAnalyzeResult { + pub content: Option, + #[serde(default)] + pub pages: Vec, + pub tables: Option>>, + #[serde(rename = "keyValuePairs")] + pub key_value_pairs: Option>>, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct AzureDocumentIntelligencePage { + #[serde( + rename = "pageNumber", + default, + deserialize_with = "crate::ocr::wire::optional_i64" + )] + pub page_number: Option, + #[serde(default, deserialize_with = "crate::ocr::wire::optional_f64")] + pub width: Option, + #[serde(default, deserialize_with = "crate::ocr::wire::optional_f64")] + pub height: Option, + pub unit: Option, + #[serde(default)] + pub lines: Vec, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct AzureDocumentIntelligenceLine { + pub content: Option, +} diff --git a/litellm-rust/crates/core/src/ocr/formats/macros.rs b/litellm-rust/crates/core/src/ocr/formats/macros.rs new file mode 100644 index 00000000000..1c4099ac078 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/formats/macros.rs @@ -0,0 +1,65 @@ +macro_rules! ocr_format { + ($vis:vis $name:ident { + params: $params:ty, + $($rest:tt)* + }) => { + $crate::ocr::formats::ocr_format! { + $vis $name { + params: $params => $params, + map: ::std::result::Result::Ok, + $($rest)* + } + } + }; + ($vis:vis $name:ident { + params: $input:ty => $mapped:ty, + $(validate: $validate:path,)? + map: $map:path, + document: $document:ty, + request: $request:ty => $build:path, + response: $response:ty => $normalize:path $(,)? + }) => { + $vis struct $name; + + impl $crate::ocr::formats::OcrFormat for $name { + type InputParams = $input; + type MappedParams = $mapped; + type PreparedDocument = $document; + type RequestBody = $request; + type ResponseBody = $response; + + $(fn validate_input_params( + params: &::serde_json::Map, + ) -> Result<(), $crate::ocr::error::OcrRequestError> { + $validate(params) + })? + + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] + fn map_params( + params: Self::InputParams, + ) -> Result { + $map(params) + } + + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] + fn transform_request( + model: &str, + document: Self::PreparedDocument, + params: &Self::MappedParams, + ) -> Result { + $build(model, document, params) + } + + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] + fn transform_response( + model: &str, + response: Self::ResponseBody, + params: &Self::MappedParams, + ) -> Result<$crate::ocr::types::OcrResponseData, $crate::ocr::error::OcrResponseError> { + $normalize(model, response, params) + } + } + }; +} + +pub(crate) use ocr_format; diff --git a/litellm-rust/crates/core/src/ocr/formats/mistral/mod.rs b/litellm-rust/crates/core/src/ocr/formats/mistral/mod.rs new file mode 100644 index 00000000000..8ce3e3fe5a9 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/formats/mistral/mod.rs @@ -0,0 +1,16 @@ +mod transformation; +pub mod types; + +use types::*; + +use crate::ocr::formats::ocr_format; +use crate::ocr::types::OcrDocument; + +ocr_format! { + pub MistralOcrFormat { + params: MistralOcrParams, + document: OcrDocument, + request: MistralOcrRequest => transformation::request, + response: MistralOcrResponse => transformation::response, + } +} diff --git a/litellm-rust/crates/core/src/ocr/formats/mistral/transformation.rs b/litellm-rust/crates/core/src/ocr/formats/mistral/transformation.rs new file mode 100644 index 00000000000..3231ac827b6 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/formats/mistral/transformation.rs @@ -0,0 +1,30 @@ +use super::types::{MistralOcrParams, MistralOcrRequest, MistralOcrResponse}; +use crate::ocr::error::{OcrRequestError, OcrResponseError}; +use crate::ocr::types::{OcrDocument, OcrResponseData}; + +pub(super) fn request( + model: &str, + document: OcrDocument, + params: &MistralOcrParams, +) -> Result { + Ok(MistralOcrRequest { + model: model.to_string(), + document, + params: params.clone(), + }) +} + +pub(super) fn response( + model: &str, + response: MistralOcrResponse, + _params: &MistralOcrParams, +) -> Result { + Ok(OcrResponseData { + document_annotation: response.document_annotation, + usage_info: response.usage_info, + ..OcrResponseData::new( + response.model.unwrap_or_else(|| model.to_string()), + response.pages, + ) + }) +} diff --git a/litellm-rust/crates/core/src/ocr/formats/mistral/types.rs b/litellm-rust/crates/core/src/ocr/formats/mistral/types.rs new file mode 100644 index 00000000000..7341d4ae869 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/formats/mistral/types.rs @@ -0,0 +1,52 @@ +use crate::ocr::types::{OcrDocument, OcrPage, OcrUsageInfo}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct MistralOcrParams { + #[serde(skip_serializing_if = "Option::is_none")] + pub pages: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub include_image_base64: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub image_limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub image_min_size: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub bbox_annotation_format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub document_annotation_format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub document_annotation_prompt: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub extract_header: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub extract_footer: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub table_format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub confidence_scores_granularity: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub include_blocks: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub id: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct MistralOcrRequest { + pub model: String, + pub document: OcrDocument, + #[serde(flatten)] + pub params: MistralOcrParams, +} + +#[derive(Clone, Debug, Default, Deserialize)] +pub struct MistralOcrResponse { + #[serde(default)] + pub pages: Vec, + pub model: Option, + pub document_annotation: Option, + pub usage_info: Option, + #[serde(flatten)] + pub extra_fields: Map, +} diff --git a/litellm-rust/crates/core/src/ocr/formats/mod.rs b/litellm-rust/crates/core/src/ocr/formats/mod.rs new file mode 100644 index 00000000000..64c0988e841 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/formats/mod.rs @@ -0,0 +1,43 @@ +use crate::ocr::error::OcrRequestError; +use crate::ocr::error::OcrResponseError; + +use super::types::OcrResponseData; +use crate::Error; +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::{Map, Value}; + +mod macros; + +pub(crate) use macros::ocr_format; + +pub mod document_intelligence; +pub mod mistral; +pub mod reducto; + +pub trait OcrFormat: Send + Sync + Sized + 'static { + type InputParams: std::fmt::Debug + Clone + Serialize + DeserializeOwned + Send + Sync; + type MappedParams: Clone + Serialize + Send + Sync; + type PreparedDocument: Send; + type RequestBody: Serialize + DeserializeOwned + Send + Sync; + type ResponseBody: DeserializeOwned + Send; + + fn validate_input_params(_params: &Map) -> Result<(), OcrRequestError> { + Ok(()) + } + + fn map_params(params: Self::InputParams) -> Result; + fn transform_request( + model: &str, + document: Self::PreparedDocument, + params: &Self::MappedParams, + ) -> Result; + fn transform_response( + model: &str, + response: Self::ResponseBody, + params: &Self::MappedParams, + ) -> Result; +} + +pub(crate) fn request_error(path: &str) -> Error { + OcrRequestError::RequestField { path: path.into() }.into() +} diff --git a/litellm-rust/crates/core/src/ocr/formats/reducto/mod.rs b/litellm-rust/crates/core/src/ocr/formats/reducto/mod.rs new file mode 100644 index 00000000000..a8ad7724049 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/formats/reducto/mod.rs @@ -0,0 +1,24 @@ +mod transformation; +pub mod types; + +use types::*; + +use crate::ocr::formats::ocr_format; + +ocr_format! { + pub ReductoParseV3Format { + params: ReductoV3Params, + document: ReductoFileId, + request: ReductoV3Request => transformation::v3_request, + response: ReductoResponse => transformation::response, + } +} + +ocr_format! { + pub ReductoParseLegacyFormat { + params: ReductoLegacyParams, + document: ReductoFileId, + request: ReductoLegacyRequest => transformation::legacy_request, + response: ReductoResponse => transformation::response, + } +} diff --git a/litellm-rust/crates/core/src/ocr/formats/reducto/transformation.rs b/litellm-rust/crates/core/src/ocr/formats/reducto/transformation.rs new file mode 100644 index 00000000000..0028cb17bf3 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/formats/reducto/transformation.rs @@ -0,0 +1,95 @@ +use std::collections::BTreeMap; + +use super::types::*; +use crate::ocr::error::{OcrRequestError, OcrResponseError}; +use crate::ocr::types::{OcrPage, OcrResponseData, OcrUsageInfo}; + +fn join_content<'a>(content: impl Iterator>) -> String { + content + .flatten() + .filter(|text| !text.is_empty()) + .collect::>() + .join("\n\n") +} + +fn build_pages(chunks: Vec) -> Vec { + let blocks_by_page = chunks + .iter() + .flat_map(|chunk| chunk.blocks.iter().flatten()) + .filter_map(|block| block.bbox.as_ref()?.page.map(|page| (page, block))) + .fold( + BTreeMap::>::new(), + |mut pages, (page, block)| { + pages.entry(page).or_default().push(block); + pages + }, + ); + if blocks_by_page.is_empty() { + let markdown = join_content(chunks.iter().map(|chunk| chunk.content.as_deref())); + return if markdown.is_empty() { + Vec::new() + } else { + vec![OcrPage::text(0, markdown)] + }; + } + blocks_by_page + .into_iter() + .map(|(page, blocks)| { + let markdown = join_content(blocks.iter().map(|block| block.content.as_deref())); + OcrPage { + extra_fields: serde_json::Map::from_iter([( + "blocks".into(), + serde_json::json!(blocks), + )]), + ..OcrPage::text(page.saturating_sub(1).max(0), markdown) + } + }) + .collect() +} + +pub(super) fn response

( + model: &str, + response: ReductoResponse, + _params: &P, +) -> Result { + let result = match response.result { + Some(result) => result.unwrap_or_default(), + None => ReductoResult { + chunks: response.chunks, + }, + }; + let usage = response.usage.unwrap_or_default(); + Ok(OcrResponseData { + usage_info: Some(OcrUsageInfo { + pages_processed: usage.num_pages, + credits: usage.credits, + ..Default::default() + }), + ..OcrResponseData::new( + model.to_string(), + build_pages(result.chunks.unwrap_or_default()), + ) + }) +} + +pub(super) fn v3_request( + _model: &str, + document: ReductoFileId, + params: &ReductoV3Params, +) -> Result { + Ok(ReductoV3Request { + input: document.0, + params: params.clone(), + }) +} + +pub(super) fn legacy_request( + _model: &str, + document: ReductoFileId, + params: &ReductoLegacyParams, +) -> Result { + Ok(ReductoLegacyRequest { + document_url: document.0, + options: params.enhance.as_ref().map(|_| params.clone()), + }) +} diff --git a/litellm-rust/crates/core/src/ocr/formats/reducto/types.rs b/litellm-rust/crates/core/src/ocr/formats/reducto/types.rs new file mode 100644 index 00000000000..53cd12ad21d --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/formats/reducto/types.rs @@ -0,0 +1,112 @@ +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct ReductoV3Params { + #[serde(skip_serializing_if = "Option::is_none")] + pub formatting: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub retrieval: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub settings: Option>, +} +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct ReductoLegacyParams { + #[serde(skip_serializing_if = "Option::is_none")] + pub enhance: Option>, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ReductoV3Request { + pub input: String, + #[serde(flatten)] + pub params: ReductoV3Params, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ReductoLegacyRequest { + pub document_url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub options: Option, +} +pub struct ReductoFileId(pub(crate) String); + +impl TryFrom for ReductoFileId { + type Error = crate::ocr::error::OcrError; + + fn try_from(response: ReductoUploadResponse) -> Result { + if response.file_id.is_empty() { + return Err(crate::ocr::error::OcrResponseError::ResponseField { + path: "file_id".into(), + } + .into()); + } + Ok(Self(response.file_id)) + } +} + +#[derive(Deserialize)] +pub struct ReductoUploadResponse { + pub file_id: String, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct ReductoResponse { + #[serde(default, deserialize_with = "present_nullable")] + pub result: Option>, + pub usage: Option, + #[serde(default)] + pub chunks: Option>, +} + +fn present_nullable<'de, D: serde::Deserializer<'de>, T: Deserialize<'de>>( + deserializer: D, +) -> Result>, D::Error> { + Option::::deserialize(deserializer).map(Some) +} +#[derive(Clone, Debug, Default, Deserialize)] +pub struct ReductoResult { + pub chunks: Option>, +} +#[derive(Clone, Debug, Default, Deserialize)] +pub struct ReductoUsage { + #[serde(default, deserialize_with = "crate::ocr::wire::optional_i64")] + pub num_pages: Option, + #[serde(default, deserialize_with = "crate::ocr::wire::optional_f64")] + pub credits: Option, +} +#[derive(Clone, Debug, Deserialize)] +pub struct ReductoChunk { + pub content: Option, + pub blocks: Option>, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ReductoBlock { + #[serde(skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub bbox: Option, + #[serde(flatten)] + pub extra_fields: Map, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ReductoBoundingBox { + #[serde(default, deserialize_with = "reducto_page")] + pub page: Option, + #[serde(flatten)] + pub extra_fields: Map, +} + +fn reducto_page<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + let value = Value::deserialize(deserializer)?; + Ok(match value { + Value::Number(number) => number.as_i64().or_else(|| { + number + .as_f64() + .and_then(crate::ocr::wire::checked_truncated_i64) + }), + Value::String(value) => value.trim().parse().ok(), + Value::Bool(value) => Some(i64::from(value)), + _ => None, + }) +} diff --git a/litellm-rust/crates/core/src/ocr/handler.rs b/litellm-rust/crates/core/src/ocr/handler.rs new file mode 100644 index 00000000000..9bdc3e6ba68 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/handler.rs @@ -0,0 +1,235 @@ +use std::sync::Arc; + +use super::OcrClient; +use super::formats::{OcrFormat, request_error}; +use super::hooks::{OcrDuringCallRequest, OcrHooks, OcrLifecycleHooks}; +use super::integrations::{GuardrailStage, MappedParams, OcrIntegration}; +use super::prepare::{MappedOcrRequest, prepare_ocr_call}; +use super::registry::{OcrIntegrationInput, OcrIntegrationRequest}; +use super::types::{OcrConnection, OcrDocument, OcrRequest, OcrResponseData}; +use crate::Error; +use crate::call_lifecycle::{CallLifecycle, CallLifecycleContext}; + +struct OcrExecution<'a> { + client: &'a OcrClient, + model: String, + document: OcrDocument, + connection: OcrConnection, + lifecycle_context: CallLifecycleContext, + hooks: Arc, +} + +pub(crate) async fn perform_ocr_request( + client: &OcrClient, + request: OcrRequest, +) -> Result { + let context = CallLifecycleContext::new( + "ocr", + request.model.clone(), + request.integration.kind().provider().as_str(), + request + .litellm_call_id + .unwrap_or_else(|| format!("ocr-{:032x}", rand::random::())), + ); + let execution = OcrExecution { + client, + model: request.model, + document: request.document, + connection: request.connection, + lifecycle_context: context, + hooks: request.hooks, + }; + macro_rules! execute_selected_integration { + ($( $variant:ident, $integration:ty, $instance:expr, $provider:ident; )+) => { + match request.integration { + $( OcrIntegrationRequest::$variant(input) => execution.run($instance, input).await, )+ + } + }; + } + + super::integrations::for_each_ocr_integration!(execute_selected_integration) +} + +impl OcrExecution<'_> { + async fn run( + self, + integration: I, + input: OcrIntegrationInput, + ) -> Result + where + I: OcrIntegration, + { + let Self { + client, + model, + document, + connection, + lifecycle_context, + hooks, + } = self; + let request = prepare_ocr_call( + integration, + model, + document, + input.params, + input.backend_config, + connection, + ); + let lifecycle_hooks = OcrLifecycleHooks { + hooks: hooks.clone(), + provider_name: lifecycle_context.custom_llm_provider.clone(), + marker: std::marker::PhantomData, + }; + CallLifecycle::default() + .run(lifecycle_context, request, &lifecycle_hooks, |request| { + execute_ocr_provider_call( + client, + request, + hooks.as_ref(), + &lifecycle_hooks.provider_name, + ) + }) + .await + } +} + +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +async fn execute_ocr_provider_call( + client: &OcrClient, + request: MappedOcrRequest, + hooks: &dyn OcrHooks, + provider_name: &str, +) -> Result +where + I: OcrIntegration, +{ + let backend = &request.integration; + let prepared_backend = backend + .prepare( + &request.connection, + &request.backend_config, + &request.model, + &request.params, + &super::prepare::credential_env, + ) + .await?; + let url = prepared_backend.url; + let headers = prepared_backend.headers; + let document = if I::GUARDRAIL_STAGE == GuardrailStage::Document && hooks.has_guardrails() { + let guarded = hooks + .during_call(OcrDuringCallRequest { + model: request.model.clone(), + custom_llm_provider: provider_name.into(), + url: url.clone(), + body: serde_json::to_value(request.document) + .map_err(|_| request_error("document"))?, + }) + .await?; + super::wire::decode_request_value(guarded.body, "guardrail.body")? + } else { + request.document + }; + let document = backend + .prepare_document(client, document, &request.connection, &headers) + .await?; + let body = I::Format::transform_request(&request.model, document.into(), &request.params)?; + let body = if I::GUARDRAIL_STAGE == GuardrailStage::RequestBody && hooks.has_guardrails() { + let guarded = hooks + .during_call(OcrDuringCallRequest { + model: request.model.clone(), + custom_llm_provider: provider_name.into(), + url: url.clone(), + body: serde_json::to_value(body).map_err(|_| request_error("body"))?, + }) + .await?; + let body: OcrWireBody<::RequestBody> = + OcrWireBody::decode(guarded.body)?; + backend.validate_request_body(&body.body)?; + body + } else { + OcrWireBody { + body, + extra: serde_json::Map::new(), + } + }; + send_ocr_call( + client, + ReadyOcrCall { + integration: request.integration, + model: request.model, + params: request.params, + connection: request.connection, + url, + headers, + body, + }, + ) + .await +} + +#[derive(serde::Serialize)] +struct OcrWireBody { + #[serde(flatten)] + body: B, + #[serde(flatten)] + extra: serde_json::Map, +} + +impl OcrWireBody { + fn decode(value: serde_json::Value) -> Result { + let body: B = super::wire::decode_request_value(value.clone(), "guardrail.body")?; + let serde_json::Value::Object(fields) = value else { + return Err(request_error("guardrail.body")); + }; + let known = serde_json::to_value(&body).map_err(|_| request_error("guardrail.body"))?; + let extra = fields + .into_iter() + .filter(|(key, _)| known.get(key).is_none()) + .collect(); + Ok(Self { body, extra }) + } +} + +struct ReadyOcrCall { + integration: I, + model: String, + params: MappedParams, + connection: OcrConnection, + url: String, + headers: Vec<(String, String)>, + body: OcrWireBody<::RequestBody>, +} + +async fn send_ocr_call( + client: &OcrClient, + request: ReadyOcrCall, +) -> Result { + let ReadyOcrCall { + integration, + model, + params, + connection, + url, + headers, + body, + } = request; + let backend = &integration; + let builder = client + .provider_http() + .post(&url) + .json(&body) + .timeout(connection.timeout); + let builder = + crate::http_utils::with_headers(builder, &headers, crate::http_utils::HeaderPolicy::All); + let response = crate::http_utils::http_request(builder) + .await + .map_err(crate::error::TransportError::from)?; + let decoded = backend + .read_response(client, response, &url, &headers, &connection, ¶ms) + .await?; + let response = I::Format::transform_response(&model, decoded.data, ¶ms)?; + Ok(OcrResponseData { + provider_native_response: decoded.native, + ..response + }) +} diff --git a/litellm-rust/crates/core/src/ocr/hooks.rs b/litellm-rust/crates/core/src/ocr/hooks.rs new file mode 100644 index 00000000000..1046d888459 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/hooks.rs @@ -0,0 +1,167 @@ +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; + +use super::formats::OcrFormat; +use super::integrations::OcrIntegration; +use super::prepare::MappedOcrRequest; +use super::types::{OcrDocument, OcrResponseData}; +use crate::Error; +use crate::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; +use serde::Serialize; +use serde_json::Value; + +pub type OcrHookFuture<'a, T> = Pin> + Send + 'a>>; +pub type OcrLogFuture<'a> = Pin + Send + 'a>>; + +#[derive(Clone, Debug, Serialize)] +pub struct OcrPreCallRequest { + pub model: String, + pub custom_llm_provider: String, + pub document: OcrDocument, + pub optional_params: Value, +} + +#[derive(Clone, Debug, Serialize)] +pub struct OcrDuringCallRequest { + pub model: String, + pub custom_llm_provider: String, + pub url: String, + pub body: Value, +} + +pub trait OcrHooks: Send + Sync { + fn has_guardrails(&self) -> bool { + false + } + fn pre_call(&self, request: OcrPreCallRequest) -> OcrHookFuture<'_, OcrPreCallRequest> { + Box::pin(async move { Ok(request) }) + } + fn during_call( + &self, + request: OcrDuringCallRequest, + ) -> OcrHookFuture<'_, OcrDuringCallRequest> { + Box::pin(async move { Ok(request) }) + } + fn success<'a>( + &'a self, + _context: &'a CallLifecycleContext, + _response: &'a OcrResponseData, + _timing: &'a CallLifecycleTiming, + ) -> OcrLogFuture<'a> { + Box::pin(async {}) + } + fn failure<'a>( + &'a self, + _context: &'a CallLifecycleContext, + _error: &'a Error, + _timing: &'a CallLifecycleTiming, + ) -> OcrLogFuture<'a> { + Box::pin(async {}) + } +} + +pub struct NoopOcrHooks; +impl OcrHooks for NoopOcrHooks {} + +pub(crate) struct OcrLifecycleHooks +where + I: OcrIntegration, +{ + pub hooks: Arc, + pub provider_name: String, + pub marker: std::marker::PhantomData, +} + +impl CallLifecycleHooks, Error>, MappedOcrRequest, OcrResponseData> + for OcrLifecycleHooks +where + I: OcrIntegration, +{ + type PreCallFuture<'a> = OcrHookFuture<'a, Result, Error>>; + type DuringCallFuture<'a> = OcrHookFuture<'a, MappedOcrRequest>; + type SuccessFuture<'a> = OcrLogFuture<'a>; + type FailureFuture<'a> = OcrLogFuture<'a>; + + fn async_pre_call_hook<'a>( + &'a self, + _context: &'a CallLifecycleContext, + request: Result, Error>, + ) -> Self::PreCallFuture<'a> { + Box::pin(async move { + let request = request?; + if !self.hooks.has_guardrails() { + return Ok(Ok(request)); + } + let changed = self + .hooks + .pre_call(OcrPreCallRequest { + model: request.model.clone(), + custom_llm_provider: self.provider_name.clone(), + document: request.document, + optional_params: serde_json::to_value(request.params).map_err(|_| { + super::error::OcrRequestError::RequestField { + path: "guardrail.optional_params".into(), + } + })?, + }) + .await?; + let params = request + .integration + .decode_input_params( + changed + .optional_params + .as_object() + .cloned() + .ok_or_else(|| super::error::OcrRequestError::RequestField { + path: "guardrail.optional_params".into(), + })?, + "guardrail.optional_params", + ) + .and_then(I::Format::map_params)?; + Ok(Ok(MappedOcrRequest { + document: changed.document, + params, + ..request + })) + }) + } + + fn async_during_call_hook<'a>( + &'a self, + _context: &'a CallLifecycleContext, + request: Result, Error>, + ) -> Self::DuringCallFuture<'a> { + Box::pin(async move { request }) + } + + #[tracing::instrument( + name = "success_callback", + target = "litellm::function_trace", + level = "trace", + skip_all + )] + fn async_log_success_event<'a>( + &'a self, + context: &'a CallLifecycleContext, + response: &'a OcrResponseData, + timing: &'a CallLifecycleTiming, + ) -> Self::SuccessFuture<'a> { + self.hooks.success(context, response, timing) + } + + #[tracing::instrument( + name = "failure_callback", + target = "litellm::function_trace", + level = "trace", + skip_all + )] + fn async_log_failure_event<'a>( + &'a self, + context: &'a CallLifecycleContext, + error: &'a Error, + timing: &'a CallLifecycleTiming, + ) -> Self::FailureFuture<'a> { + self.hooks.failure(context, error, timing) + } +} diff --git a/litellm-rust/crates/core/src/ocr/integrations/azure_document_intelligence.rs b/litellm-rust/crates/core/src/ocr/integrations/azure_document_intelligence.rs new file mode 100644 index 00000000000..17b4491fa18 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/integrations/azure_document_intelligence.rs @@ -0,0 +1,91 @@ +use crate::constants::AZURE_DI_API_VERSION; +use crate::ocr::backends::PreparedOcrBackend; +use crate::ocr::backends::azure_ai::{self, AzureBackend}; +use crate::ocr::error::{OcrError, OcrRequestError}; +use crate::ocr::formats::document_intelligence::{ + AzureDocumentIntelligenceOcrFormat, + types::{AzureDocumentIntelligenceOperation, DocumentIntelligenceParams}, +}; +use crate::ocr::types::{OcrConnection, OcrRequestFormat}; +use crate::ocr::wire::DecodedOcrResponse; +use crate::url_utils::ApiUrl; + +use super::{BackendConfig, OcrIntegration}; + +#[derive(Clone, Debug)] +pub struct AzureDocumentIntelligence; + +impl OcrIntegration for AzureDocumentIntelligence { + type Backend = AzureBackend; + type Format = AzureDocumentIntelligenceOcrFormat; + type DocumentPreparation = super::PassThrough; + + async fn prepare( + &self, + connection: &OcrConnection, + config: &BackendConfig, + model: &str, + params: &DocumentIntelligenceParams, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + ) -> Result { + let (endpoint, headers) = + azure_ai::prepare_document_intelligence_auth(connection, config, env_lookup).await?; + let model = format!("{}:analyze", model_id(model)?); + let url = ApiUrl::parse(&endpoint) + .and_then(|url| url.complete_path(&["documentintelligence", "documentModels", &model])) + .map(|url| { + url.append_query_pairs( + [("api-version", AZURE_DI_API_VERSION)] + .into_iter() + .chain(params.pages.iter().map(|pages| ("pages", pages.0.as_str()))) + .chain( + params + .features + .iter() + .map(|features| ("features", features.0.as_str())), + ), + ) + .into_string() + }) + .map_err(|_| OcrRequestError::RequestField { + path: "api_base".into(), + })?; + Ok(PreparedOcrBackend { url, headers }) + } + + fn preserve_native_response(&self, params: &DocumentIntelligenceParams) -> bool { + params.request_format == OcrRequestFormat::Native + } + + fn supports_native_request_format(&self) -> bool { + true + } + + async fn read_response( + &self, + client: &crate::ocr::OcrClient, + response: reqwest::Response, + url: &str, + headers: &[(String, String)], + connection: &OcrConnection, + params: &DocumentIntelligenceParams, + ) -> Result, OcrError> { + crate::ocr::backends::azure_document_intelligence::polling::read_operation_response( + client.provider_http(), + response, + url, + headers, + connection, + params.request_format == OcrRequestFormat::Native, + ) + .await + } +} + +fn model_id(model: &str) -> Result<&str, OcrRequestError> { + let model = model.rsplit('/').next().unwrap_or(model); + if matches!(model, "." | "..") { + return Err(OcrRequestError::DotModel); + } + Ok(model) +} diff --git a/litellm-rust/crates/core/src/ocr/integrations/azure_mistral.rs b/litellm-rust/crates/core/src/ocr/integrations/azure_mistral.rs new file mode 100644 index 00000000000..60cd53139b9 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/integrations/azure_mistral.rs @@ -0,0 +1,47 @@ +use crate::constants::AZURE_AI_OCR_PATH; +use crate::ocr::backends::PreparedOcrBackend; +use crate::ocr::backends::azure_ai::{self, AzureBackend}; +use crate::ocr::document::InlineOcrDocument; +use crate::ocr::error::{OcrError, OcrRequestError}; +use crate::ocr::formats::mistral::{MistralOcrFormat, types::MistralOcrParams}; +use crate::ocr::types::OcrConnection; +use crate::url_utils::ApiUrl; + +use super::{BackendConfig, OcrIntegration}; + +#[derive(Clone, Debug)] +pub struct AzureMistral; + +impl OcrIntegration for AzureMistral { + type Backend = AzureBackend; + type Format = MistralOcrFormat; + type DocumentPreparation = super::RequireInline; + + async fn prepare( + &self, + connection: &OcrConnection, + config: &BackendConfig, + _model: &str, + _params: &MistralOcrParams, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + ) -> Result { + let base = azure_ai::resolve_mistral_api_base(connection, env_lookup)?; + let path: Vec<&str> = AZURE_AI_OCR_PATH.trim_matches('/').split('/').collect(); + let url = ApiUrl::parse(&base) + .and_then(|url| url.complete_path(&path)) + .map(|url| url.into_string()) + .map_err(|_| OcrRequestError::RequestField { + path: "api_base".into(), + })?; + let headers = azure_ai::authenticate_mistral(connection, config, env_lookup).await?; + Ok(PreparedOcrBackend { url, headers }) + } + + fn validate_request_body( + &self, + body: &crate::ocr::formats::mistral::types::MistralOcrRequest, + ) -> Result<(), OcrRequestError> { + InlineOcrDocument::try_from(body.document.clone())?; + Ok(()) + } +} diff --git a/litellm-rust/crates/core/src/ocr/integrations/mistral_direct.rs b/litellm-rust/crates/core/src/ocr/integrations/mistral_direct.rs new file mode 100644 index 00000000000..38f56596738 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/integrations/mistral_direct.rs @@ -0,0 +1,30 @@ +use crate::ocr::backends::PreparedOcrBackend; +use crate::ocr::backends::mistral::{self, MistralBackend}; +use crate::ocr::error::OcrError; +use crate::ocr::formats::mistral::{MistralOcrFormat, types::MistralOcrParams}; +use crate::ocr::types::OcrConnection; + +use super::{BackendConfig, OcrIntegration}; + +#[derive(Clone, Debug)] +pub struct MistralDirect; + +impl OcrIntegration for MistralDirect { + type Backend = MistralBackend; + type Format = MistralOcrFormat; + type DocumentPreparation = super::PassThrough; + + async fn prepare( + &self, + connection: &OcrConnection, + _config: &BackendConfig, + _model: &str, + _params: &MistralOcrParams, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + ) -> Result { + Ok(PreparedOcrBackend { + url: mistral::complete_url(connection.api_base.as_deref())?, + headers: mistral::authenticate(connection, env_lookup)?, + }) + } +} diff --git a/litellm-rust/crates/core/src/ocr/integrations/mod.rs b/litellm-rust/crates/core/src/ocr/integrations/mod.rs new file mode 100644 index 00000000000..4ec04fec4f1 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/integrations/mod.rs @@ -0,0 +1,137 @@ +use std::future::Future; + +use serde_json::{Map, Value}; + +use super::OcrClient; +use super::backends::{OcrBackend, PreparedOcrBackend}; +use super::error::{OcrError, OcrRequestError}; +use super::formats::OcrFormat; +use super::types::{OcrConnection, OcrDocument, OcrRequestFormat}; +use super::wire::DecodedOcrResponse; + +mod azure_document_intelligence; +mod azure_mistral; +mod mistral_direct; +mod reducto_legacy; +mod reducto_v3; + +pub use super::backends::reducto::ReductoUpload; +pub use super::document::{DocumentPreparation, PassThrough, RequireInline}; +pub use azure_document_intelligence::AzureDocumentIntelligence; +pub use azure_mistral::AzureMistral; +pub use mistral_direct::MistralDirect; +pub use reducto_legacy::ReductoLegacy; +pub use reducto_v3::ReductoV3; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum GuardrailStage { + Document, + RequestBody, +} + +pub trait OcrIntegration: Send + Sync + Sized + 'static { + type Backend: OcrBackend; + type Format: OcrFormat; + type DocumentPreparation: DocumentPreparation< + Output: Into<::PreparedDocument>, + >; + const GUARDRAIL_STAGE: GuardrailStage = GuardrailStage::RequestBody; + + fn supports_native_request_format(&self) -> bool { + false + } + + fn decode_input_params( + &self, + params: Map, + prefix: &str, + ) -> Result, OcrRequestError> { + validate_request_format( + ¶ms, + self.supports_native_request_format(), + Self::Backend::PROVIDER.as_str(), + )?; + Self::Format::validate_input_params(¶ms)?; + super::wire::decode_request_value(Value::Object(params), prefix) + } + + fn prepare( + &self, + connection: &OcrConnection, + config: &BackendConfig, + model: &str, + params: &MappedParams, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + ) -> impl Future> + Send; + + fn prepare_document( + &self, + client: &OcrClient, + document: OcrDocument, + connection: &OcrConnection, + headers: &[(String, String)], + ) -> impl Future< + Output = Result<::Output, OcrError>, + > + Send { + Self::DocumentPreparation::prepare(client, document, connection, headers) + } + + fn validate_request_body( + &self, + _body: &::RequestBody, + ) -> Result<(), OcrRequestError> { + Ok(()) + } + + fn preserve_native_response(&self, _params: &MappedParams) -> bool { + false + } + + fn read_response( + &self, + _client: &OcrClient, + response: reqwest::Response, + _url: &str, + _headers: &[(String, String)], + _connection: &OcrConnection, + params: &MappedParams, + ) -> impl Future< + Output = Result::ResponseBody>, OcrError>, + > + Send { + super::client::read_json_response(response, self.preserve_native_response(params)) + } +} + +fn validate_request_format( + params: &Map, + supports_native: bool, + provider: &'static str, +) -> Result<(), OcrRequestError> { + let Some(format) = params.get("req_format") else { + return Ok(()); + }; + let format: OcrRequestFormat = + serde_json::from_value(format.clone()).map_err(|_| OcrRequestError::RequestFormat)?; + if format == OcrRequestFormat::Native && !supports_native { + return Err(OcrRequestError::NativeUnsupported(provider)); + } + Ok(()) +} + +pub type InputParams = <::Format as OcrFormat>::InputParams; +pub type MappedParams = <::Format as OcrFormat>::MappedParams; +pub type BackendConfig = <::Backend as OcrBackend>::Config; + +macro_rules! for_each_ocr_integration { + ($callback:ident) => { + $callback! { + Mistral, $crate::ocr::integrations::MistralDirect, $crate::ocr::integrations::MistralDirect, Mistral; + AzureMistral, $crate::ocr::integrations::AzureMistral, $crate::ocr::integrations::AzureMistral, AzureAi; + AzureDocumentIntelligence, $crate::ocr::integrations::AzureDocumentIntelligence, $crate::ocr::integrations::AzureDocumentIntelligence, AzureAi; + ReductoV3, $crate::ocr::integrations::ReductoV3, $crate::ocr::integrations::ReductoV3, Reducto; + ReductoLegacy, $crate::ocr::integrations::ReductoLegacy, $crate::ocr::integrations::ReductoLegacy, Reducto; + } + }; +} + +pub(crate) use for_each_ocr_integration; diff --git a/litellm-rust/crates/core/src/ocr/integrations/reducto_legacy.rs b/litellm-rust/crates/core/src/ocr/integrations/reducto_legacy.rs new file mode 100644 index 00000000000..680259e08fa --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/integrations/reducto_legacy.rs @@ -0,0 +1,35 @@ +use crate::ocr::backends::PreparedOcrBackend; +use crate::ocr::backends::reducto::{self, ReductoBackend}; +use crate::ocr::error::OcrError; +use crate::ocr::formats::reducto::{ReductoParseLegacyFormat, types::ReductoLegacyParams}; +use crate::ocr::types::OcrConnection; + +use super::{BackendConfig, OcrIntegration}; + +#[derive(Clone, Debug)] +pub struct ReductoLegacy; + +impl OcrIntegration for ReductoLegacy { + type Backend = ReductoBackend; + type Format = ReductoParseLegacyFormat; + type DocumentPreparation = super::ReductoUpload; + const GUARDRAIL_STAGE: super::GuardrailStage = super::GuardrailStage::Document; + + async fn prepare( + &self, + connection: &OcrConnection, + _config: &BackendConfig, + _model: &str, + _params: &ReductoLegacyParams, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + ) -> Result { + Ok(PreparedOcrBackend { + url: reducto::complete_url(connection.api_base.as_deref(), "parse")?, + headers: reducto::authenticate(connection, env_lookup)?, + }) + } + + fn preserve_native_response(&self, _params: &ReductoLegacyParams) -> bool { + true + } +} diff --git a/litellm-rust/crates/core/src/ocr/integrations/reducto_v3.rs b/litellm-rust/crates/core/src/ocr/integrations/reducto_v3.rs new file mode 100644 index 00000000000..673c3f75dfd --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/integrations/reducto_v3.rs @@ -0,0 +1,35 @@ +use crate::ocr::backends::PreparedOcrBackend; +use crate::ocr::backends::reducto::{self, ReductoBackend}; +use crate::ocr::error::OcrError; +use crate::ocr::formats::reducto::{ReductoParseV3Format, types::ReductoV3Params}; +use crate::ocr::types::OcrConnection; + +use super::{BackendConfig, OcrIntegration}; + +#[derive(Clone, Debug)] +pub struct ReductoV3; + +impl OcrIntegration for ReductoV3 { + type Backend = ReductoBackend; + type Format = ReductoParseV3Format; + type DocumentPreparation = super::ReductoUpload; + const GUARDRAIL_STAGE: super::GuardrailStage = super::GuardrailStage::Document; + + async fn prepare( + &self, + connection: &OcrConnection, + _config: &BackendConfig, + _model: &str, + _params: &ReductoV3Params, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + ) -> Result { + Ok(PreparedOcrBackend { + url: reducto::complete_url(connection.api_base.as_deref(), "parse")?, + headers: reducto::authenticate(connection, env_lookup)?, + }) + } + + fn preserve_native_response(&self, _params: &ReductoV3Params) -> bool { + true + } +} diff --git a/litellm-rust/crates/core/src/ocr/mod.rs b/litellm-rust/crates/core/src/ocr/mod.rs index ec2fbb969a6..0ed3a9cf325 100644 --- a/litellm-rust/crates/core/src/ocr/mod.rs +++ b/litellm-rust/crates/core/src/ocr/mod.rs @@ -1,2 +1,34 @@ -pub mod transformation; +pub(crate) mod backends; +pub mod client; +pub(crate) mod document; +pub mod error; +pub mod formats; +mod handler; +pub mod hooks; +pub mod integrations; +pub mod prepare; +pub mod registry; pub mod types; +pub mod wire; + +pub use client::{OcrClient, ocr}; +pub use types::{OcrRequest, OcrResponseData}; + +#[cfg(test)] +#[path = "../../tests/azure_ai_ocr.rs"] +mod azure_ai_tests; +#[cfg(test)] +#[path = "../../tests/azure_document_intelligence_ocr.rs"] +mod azure_document_intelligence_tests; +#[cfg(test)] +#[path = "../../tests/mistral_ocr.rs"] +mod mistral_tests; +#[cfg(test)] +#[path = "../../tests/reducto_ocr.rs"] +mod reducto_tests; +#[cfg(test)] +#[path = "../../tests/ocr/support.rs"] +pub(crate) mod test_support; +#[cfg(test)] +#[path = "../../tests/ocr.rs"] +pub(crate) mod tests; diff --git a/litellm-rust/crates/core/src/ocr/prepare.rs b/litellm-rust/crates/core/src/ocr/prepare.rs new file mode 100644 index 00000000000..f9fdb7e4450 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/prepare.rs @@ -0,0 +1,52 @@ +use super::formats::OcrFormat; +use super::integrations::{BackendConfig, InputParams, MappedParams, OcrIntegration}; + +use super::types::{OcrConnection, OcrDocument}; +use crate::Error; + +pub use super::registry::{ + OcrIntegrationKind, OcrIntegrationRequest, OcrModel, OcrProvider, resolve_ocr_integration, +}; + +pub(crate) struct MappedOcrRequest +where + I: OcrIntegration, +{ + pub integration: I, + pub model: String, + pub document: OcrDocument, + pub params: MappedParams, + pub backend_config: BackendConfig, + pub connection: OcrConnection, +} + +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +pub(crate) fn prepare_ocr_call( + integration: I, + model: String, + document: OcrDocument, + params: InputParams, + backend_config: BackendConfig, + connection: OcrConnection, +) -> Result, Error> +where + I: OcrIntegration, +{ + let params = I::Format::map_params(params)?; + Ok(MappedOcrRequest { + integration, + model, + document, + params, + backend_config, + connection, + }) +} + +pub(crate) fn credential_env(name: &str) -> Option { + std::env::var(name).ok() +} + +#[cfg(test)] +#[path = "../../tests/ocr_prepare.rs"] +mod tests; diff --git a/litellm-rust/crates/core/src/ocr/registry.rs b/litellm-rust/crates/core/src/ocr/registry.rs new file mode 100644 index 00000000000..d3dbe9c3fd5 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/registry.rs @@ -0,0 +1,159 @@ +use serde_json::{Map, Value}; +use strum::EnumString; + +use crate::Error; +use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; + +pub use super::backends::OcrBackend; +pub use super::integrations::{ + AzureDocumentIntelligence, AzureMistral, MistralDirect, ReductoLegacy, ReductoV3, +}; +use super::integrations::{BackendConfig, InputParams, OcrIntegration}; + +/// ``` +/// use litellm_core::ocr::registry::{MistralDirect, OcrIntegrationInput}; +/// use litellm_core::ocr::formats::mistral::types::MistralOcrParams; +/// let input = OcrIntegrationInput:: { +/// params: MistralOcrParams::default(), +/// backend_config: (), +/// }; +/// ``` +/// +#[derive(Clone, Debug)] +pub struct OcrIntegrationInput { + pub params: InputParams, + pub backend_config: BackendConfig, +} + +macro_rules! define_integration_types { + ($( $variant:ident, $integration:ty, $instance:expr, $provider:ident; )+) => { + #[derive(Clone, Debug)] + pub enum OcrIntegrationRequest { + $( $variant(OcrIntegrationInput<$integration>), )+ + } + + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub enum OcrIntegrationKind { + $( $variant, )+ + } + + impl OcrIntegrationKind { + pub const fn provider(self) -> OcrProvider { + match self { + $( Self::$variant => OcrProvider::$provider, )+ + } + } + } + + impl OcrIntegrationRequest { + pub fn kind(&self) -> OcrIntegrationKind { + match self { + $( Self::$variant(_) => OcrIntegrationKind::$variant, )+ + } + } + } + }; +} + +super::integrations::for_each_ocr_integration!(define_integration_types); + +#[derive(Clone, Copy, Debug, EnumString, PartialEq, Eq)] +#[strum(serialize_all = "snake_case")] +pub enum OcrProvider { + Mistral, + AzureAi, + Reducto, +} + +impl OcrProvider { + pub const fn as_str(self) -> &'static str { + match self { + Self::Mistral => "mistral", + Self::AzureAi => "azure_ai", + Self::Reducto => "reducto", + } + } +} + +#[derive(Clone, Debug, EnumString, PartialEq, Eq)] +pub enum OcrModel { + #[strum(serialize = "parse-v3")] + ReductoV3, + #[strum(serialize = "parse-legacy")] + ReductoLegacy, + #[strum(default)] + Passthrough(String), +} + +impl OcrModel { + pub fn as_str(&self) -> &str { + match self { + Self::ReductoV3 => "parse-v3", + Self::ReductoLegacy => "parse-legacy", + Self::Passthrough(model) => model, + } + } +} + +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +pub fn resolve_ocr_integration(provider: OcrProvider, model: &OcrModel) -> OcrIntegrationKind { + match provider { + OcrProvider::Mistral => OcrIntegrationKind::Mistral, + OcrProvider::AzureAi => super::backends::azure_ai::resolve_integration(model), + OcrProvider::Reducto => super::backends::reducto::resolve_integration(model), + } +} + +pub fn resolve_wire_integration( + model: &str, + custom_llm_provider: Option<&str>, +) -> Result<(OcrModel, OcrIntegrationKind), Error> { + let provider = + get_custom_llm_provider(model, custom_llm_provider).unwrap_or(CustomLlmProvider { + model, + custom_llm_provider: OcrProvider::Mistral.as_str(), + }); + let typed_provider = provider + .custom_llm_provider + .parse::() + .map_err(|_| Error::InvalidProvider(provider.custom_llm_provider.to_string()))?; + let model = OcrModel::from(provider.model); + let integration = resolve_ocr_integration(typed_provider, &model); + Ok((model, integration)) +} + +pub fn decode_integration_request( + kind: OcrIntegrationKind, + params: Map, +) -> Result { + macro_rules! decode_selected_integration { + ($( $variant:ident, $integration:ty, $instance:expr, $provider:ident; )+) => { + match kind { + $( + OcrIntegrationKind::$variant => OcrIntegrationRequest::$variant( + decode_input::<$integration>($instance, params)?, + ), + )+ + } + }; + } + + Ok(super::integrations::for_each_ocr_integration!( + decode_selected_integration + )) +} + +fn decode_input( + integration: I, + params: Map, +) -> Result, Error> +where + I: OcrIntegration, +{ + let backend_config = I::Backend::decode_config(¶ms)?; + let params = integration.decode_input_params(params, "optional_params")?; + Ok(OcrIntegrationInput { + params, + backend_config, + }) +} diff --git a/litellm-rust/crates/core/src/ocr/transformation.rs b/litellm-rust/crates/core/src/ocr/transformation.rs deleted file mode 100644 index 62299faf9ed..00000000000 --- a/litellm-rust/crates/core/src/ocr/transformation.rs +++ /dev/null @@ -1,107 +0,0 @@ -use crate::Error; -use serde_json::{Map, Value}; - -use super::types::{OcrRequestData, OcrResponseData}; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum OcrAuthStrategy { - Bearer, - Header(&'static str), -} - -impl OcrAuthStrategy { - pub fn header_name(self) -> &'static str { - match self { - Self::Bearer => "authorization", - Self::Header(header_name) => header_name, - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum OcrResponseHandling { - Json, - AzureDocumentIntelligencePoll, -} - -pub trait OcrProviderConfig: Sync { - fn supported_ocr_params(&self) -> &'static [&'static str]; - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn map_ocr_params(&self, non_default_params: &Map) -> Map { - let mut mapped_params = Map::new(); - for (param, value) in non_default_params { - if self.supported_ocr_params().contains(¶m.as_str()) { - mapped_params.insert(param.clone(), value.clone()); - } - } - mapped_params - } - - fn transform_ocr_request( - &self, - model: &str, - document: Value, - optional_params: Map, - ) -> Result; - - fn transform_ocr_response( - &self, - model: &str, - response_json: Value, - ) -> Result; - - fn transform_ocr_response_with_params( - &self, - model: &str, - response_json: Value, - _optional_params: &Map, - ) -> Result { - self.transform_ocr_response(model, response_json) - } - - fn complete_url( - &self, - api_base: Option<&str>, - model: &str, - optional_params: &Map, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result; - - fn resolve_api_key( - &self, - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result; - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn validate_environment( - &self, - headers: Vec<(String, String)>, - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result, Error> { - let strategy = self.auth_strategy(); - if crate::http_utils::has_header(&headers, strategy.header_name()) { - return Ok(headers); - } - let api_key = self.resolve_api_key(api_key, env_lookup)?; - let auth_header = match strategy { - OcrAuthStrategy::Bearer => ("Authorization".to_string(), format!("Bearer {api_key}")), - OcrAuthStrategy::Header(name) => (name.to_string(), api_key), - }; - Ok(std::iter::once(auth_header).chain(headers).collect()) - } - - fn auth_strategy(&self) -> OcrAuthStrategy { - OcrAuthStrategy::Bearer - } - - fn requires_data_uri_document(&self) -> bool { - false - } - - fn response_handling(&self) -> OcrResponseHandling { - OcrResponseHandling::Json - } -} diff --git a/litellm-rust/crates/core/src/ocr/types.rs b/litellm-rust/crates/core/src/ocr/types.rs index 71cdb232a87..1c831b3b7e9 100644 --- a/litellm-rust/crates/core/src/ocr/types.rs +++ b/litellm-rust/crates/core/src/ocr/types.rs @@ -1,38 +1,195 @@ +use std::sync::Arc; +use std::time::Duration; + use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; +use super::hooks::{NoopOcrHooks, OcrHooks}; +use super::registry::OcrIntegrationRequest; +use crate::constants::{OCR_DOWNLOAD_MAX_BYTES, OCR_HTTP_TIMEOUT_SECS}; + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct OcrRequestData { - pub data: Value, - pub files: Option, +#[serde(tag = "type")] +pub enum OcrDocument { + #[serde(rename = "document_url")] + DocumentUrl { document_url: String }, + #[serde(rename = "image_url")] + ImageUrl { image_url: String }, +} + +impl OcrDocument { + pub fn source(&self) -> &str { + match self { + Self::DocumentUrl { document_url } => document_url, + Self::ImageUrl { image_url } => image_url, + } + } + + pub fn with_source(self, source: String) -> Self { + match self { + Self::DocumentUrl { .. } => Self::DocumentUrl { + document_url: source, + }, + Self::ImageUrl { .. } => Self::ImageUrl { image_url: source }, + } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum OcrRequestFormat { + #[default] + Litellm, + Native, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct OcrPageDimensions { + #[serde(default, deserialize_with = "super::wire::optional_i64")] + pub dpi: Option, + #[serde(default, deserialize_with = "super::wire::optional_i64")] + pub height: Option, + #[serde(default, deserialize_with = "super::wire::optional_i64")] + pub width: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct OcrPageImage { + pub image_base64: Option, + pub bbox: Option>, + #[serde(flatten)] + pub extra_fields: Map, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct OcrPage { + #[serde(deserialize_with = "super::wire::required_i64")] + pub index: i64, + pub markdown: String, + pub images: Option>, + pub dimensions: Option, + #[serde(flatten)] + pub extra_fields: Map, +} + +impl OcrPage { + pub fn text(index: i64, markdown: String) -> Self { + Self { + index, + markdown, + images: None, + dimensions: None, + extra_fields: Map::new(), + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct OcrUsageInfo { + #[serde(default, deserialize_with = "super::wire::optional_i64")] + pub pages_processed: Option, + #[serde(default, deserialize_with = "super::wire::optional_i64")] + pub pages_processed_annotation: Option, + #[serde(default, deserialize_with = "super::wire::optional_f64")] + pub credits: Option, + #[serde(default, deserialize_with = "super::wire::optional_i64")] + pub doc_size_bytes: Option, + #[serde(flatten)] + pub extra_fields: Map, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct OcrResponseData { - pub pages: Vec, + pub pages: Vec, pub model: String, pub document_annotation: Option, - pub usage_info: Option, - pub object: String, + pub usage_info: Option, + pub content: Option, + pub tables: Option>>, + #[serde(rename = "keyValuePairs")] + pub key_value_pairs: Option>>, + #[serde(flatten)] pub extra_fields: Map, - pub provider_native_response: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub provider_native_response: Option>, + pub object: OcrObject, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub enum OcrObject { + #[serde(rename = "ocr")] + Ocr, } impl OcrResponseData { - pub fn into_json(self) -> Value { - let mut response = serde_json::json!({ - "pages": self.pages, - "model": self.model, - "document_annotation": self.document_annotation, - "usage_info": self.usage_info, - "object": self.object, - }); - if let Value::Object(object) = &mut response { - object.extend(self.extra_fields); - if let Some(native_response) = self.provider_native_response { - object.insert("provider_native_response".to_string(), native_response); - } + pub fn new(model: String, pages: Vec) -> Self { + Self { + model, + pages, + document_annotation: None, + usage_info: None, + content: None, + tables: None, + key_value_pairs: None, + extra_fields: Map::new(), + provider_native_response: None, + object: OcrObject::Ocr, + } + } + + pub fn into_json(self) -> Value { + serde_json::json!(self) + } +} + +#[derive(Clone)] +pub struct OcrConnection { + pub api_key: Option, + pub api_base: Option, + pub extra_headers: Vec<(String, String)>, + pub timeout: Duration, + pub poll_timeout: Duration, + pub max_download_bytes: u64, +} + +impl Default for OcrConnection { + fn default() -> Self { + Self { + api_key: None, + api_base: None, + extra_headers: Vec::new(), + timeout: Duration::from_secs(OCR_HTTP_TIMEOUT_SECS), + poll_timeout: Duration::from_secs(crate::constants::OCR_POLL_TIMEOUT_SECS), + max_download_bytes: OCR_DOWNLOAD_MAX_BYTES, + } + } +} + +pub fn download_limit_bytes(megabytes: Option<&str>) -> u64 { + megabytes + .and_then(|value| value.parse::().ok()) + .map(|value| (value.max(0.0) * 1024.0 * 1024.0) as u64) + .unwrap_or(OCR_DOWNLOAD_MAX_BYTES) +} + +pub struct OcrRequest { + pub model: String, + pub document: OcrDocument, + pub integration: OcrIntegrationRequest, + pub connection: OcrConnection, + pub hooks: Arc, + pub litellm_call_id: Option, +} + +impl OcrRequest { + pub fn new(model: String, document: OcrDocument, integration: OcrIntegrationRequest) -> Self { + Self { + model, + document, + integration, + connection: OcrConnection::default(), + hooks: Arc::new(NoopOcrHooks), + litellm_call_id: None, } - response } } diff --git a/litellm-rust/crates/core/src/ocr/wire.rs b/litellm-rust/crates/core/src/ocr/wire.rs new file mode 100644 index 00000000000..61750830d53 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/wire.rs @@ -0,0 +1,212 @@ +use crate::ocr::error::OcrRequestError; +use crate::ocr::error::OcrResponseError; +use std::time::Duration; + +use super::hooks::{OcrDuringCallRequest, OcrPreCallRequest}; +use super::registry::{decode_integration_request, resolve_wire_integration}; +use super::types::{OcrConnection, OcrDocument, OcrRequest}; +use crate::Error; +use serde::{ + Deserialize, Deserializer, + de::{DeserializeOwned, IntoDeserializer}, +}; +use serde_json::{Map, Value}; + +#[derive(Debug)] +pub struct DecodedOcrResponse { + pub data: T, + pub native: Option>, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OcrWireRequest { + pub model: String, + pub document: Value, + pub api_key: Option, + pub api_base: Option, + pub custom_llm_provider: Option, + pub extra_headers: Option>, + #[serde(default)] + pub optional_params: Map, + pub timeout_seconds: Option, +} + +pub fn decode_request(wire: OcrWireRequest) -> Result { + let (model, integration_kind) = + resolve_wire_integration(&wire.model, wire.custom_llm_provider.as_deref())?; + let integration = decode_integration_request(integration_kind, wire.optional_params)?; + let document = decode_request_value(wire.document, "document")?; + let headers = wire + .extra_headers + .unwrap_or_default() + .into_iter() + .map(|(name, value)| { + let value = value + .as_str() + .ok_or_else(|| OcrRequestError::RequestField { + path: format!("extra_headers.{name}"), + })?; + Ok((name, value.to_string())) + }) + .collect::, OcrRequestError>>()?; + let timeout = wire + .timeout_seconds + .map(|seconds| { + Duration::try_from_secs_f64(seconds).map_err(|_| OcrRequestError::RequestField { + path: "timeout_seconds".into(), + }) + }) + .transpose()?; + let defaults = OcrConnection::default(); + let mut request = OcrRequest::new(model.as_str().to_string(), document, integration); + request.connection = OcrConnection { + api_key: nonblank(wire.api_key), + api_base: nonblank(wire.api_base), + extra_headers: headers, + timeout: timeout.unwrap_or(defaults.timeout), + poll_timeout: timeout.unwrap_or(defaults.poll_timeout), + max_download_bytes: defaults.max_download_bytes, + }; + Ok(request) +} + +fn nonblank(value: Option) -> Option { + value + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) +} +pub fn decode_request_value( + value: Value, + prefix: &str, +) -> Result { + serde_path_to_error::deserialize(value.into_deserializer()).map_err(|error| { + OcrRequestError::RequestField { + path: format!("{prefix}.{}", error.path()), + } + }) +} + +pub fn decode_response( + bytes: &[u8], + native: bool, +) -> Result, OcrResponseError> { + let mut deserializer = serde_json::Deserializer::from_slice(bytes); + let data = serde_path_to_error::deserialize(&mut deserializer).map_err(|error| { + OcrResponseError::ResponseField { + path: error.path().to_string(), + } + })?; + deserializer + .end() + .map_err(|_| OcrResponseError::ResponseField { + path: "response".into(), + })?; + let native = if native { + Some( + serde_json::from_slice(bytes).map_err(|_| OcrResponseError::ResponseField { + path: "response".into(), + })?, + ) + } else { + None + }; + Ok(DecodedOcrResponse { data, native }) +} + +pub fn decode_pre_call_result( + original: OcrPreCallRequest, + value: Value, +) -> Result { + #[derive(Deserialize)] + struct Changed { + document: OcrDocument, + #[serde(default)] + optional_params: Map, + } + let changed: Changed = decode_request_value(value, "guardrail")?; + Ok(OcrPreCallRequest { + document: changed.document, + optional_params: Value::Object(changed.optional_params), + ..original + }) +} + +pub fn decode_during_call_result( + original: OcrDuringCallRequest, + value: Value, +) -> Result { + #[derive(Deserialize)] + struct Changed { + body: Value, + } + let changed: Changed = decode_request_value(value, "guardrail")?; + Ok(OcrDuringCallRequest { + body: changed.body, + ..original + }) +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum PythonNumber { + Integer(i64), + Float(f64), + String(String), + Boolean(bool), +} +impl PythonNumber { + fn integer(self) -> Option { + match self { + Self::Integer(n) => Some(n), + Self::Boolean(n) => Some(i64::from(n)), + Self::String(s) => s + .trim() + .parse::() + .ok() + .or_else(|| s.trim().parse::().ok().and_then(exact_integer)), + Self::Float(n) => exact_integer(n), + } + } + fn float(self) -> Option { + match self { + Self::Integer(n) => Some(n as f64), + Self::Float(n) => Some(n), + Self::String(s) => s.trim().parse().ok(), + Self::Boolean(n) => Some(u8::from(n) as f64), + } + } +} +fn exact_integer(n: f64) -> Option { + if n.fract() != 0.0 { + return None; + } + checked_truncated_i64(n) +} +pub(crate) fn checked_truncated_i64(value: f64) -> Option { + let truncated = value.trunc(); + (truncated.is_finite() && truncated >= i64::MIN as f64 && truncated < -(i64::MIN as f64)) + .then_some(truncated as i64) +} + +pub fn optional_i64<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + Option::::deserialize(deserializer)? + .map(|n| { + n.integer() + .ok_or_else(|| serde::de::Error::custom("expected integer")) + }) + .transpose() +} +pub fn required_i64<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + PythonNumber::deserialize(deserializer)? + .integer() + .ok_or_else(|| serde::de::Error::custom("expected integer")) +} +pub fn optional_f64<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + Option::::deserialize(deserializer)? + .map(|n| { + n.float() + .ok_or_else(|| serde::de::Error::custom("expected number")) + }) + .transpose() +} diff --git a/litellm-rust/crates/core/src/providers/azure_ai/auth.rs b/litellm-rust/crates/core/src/providers/azure_ai/auth.rs new file mode 100644 index 00000000000..4c53905463a --- /dev/null +++ b/litellm-rust/crates/core/src/providers/azure_ai/auth.rs @@ -0,0 +1,378 @@ +use crate::auth::error::AuthConfigurationError; +use crate::auth::error::MissingCredential; +use std::sync::OnceLock; + +use crate::auth::azure::{AzureAuthInputs, AzureAuthService}; +use crate::auth::http::apply_credential; +use crate::auth::{ + CredentialPlacement, CredentialPlanKind, CredentialRef, CredentialRule, ExistingHeaderBehavior, + ProviderAuthPolicy, ResolvedCredential, SecretValue, +}; +use crate::error::AuthError; + +pub const AZURE_AI_API_KEY_ENV: &str = "AZURE_AI_API_KEY"; +pub const AZURE_AI_API_BASE_ENV: &str = "AZURE_AI_API_BASE"; +pub const AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_API_KEY"; +pub const AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT"; + +const AZURE_AI_AUTH_RULES: &[CredentialRule] = &[ + CredentialRule { + kind: CredentialPlanKind::Static, + placement: CredentialPlacement::Bearer, + }, + CredentialRule { + kind: CredentialPlanKind::Entra, + placement: CredentialPlacement::Bearer, + }, +]; +const AZURE_AI_AUTH_POLICY: ProviderAuthPolicy = ProviderAuthPolicy { + rules: AZURE_AI_AUTH_RULES, + accepted_existing_headers: &["Authorization"], + existing_header_behavior: ExistingHeaderBehavior::Preserve, + scope: Some(crate::auth::azure::DEFAULT_AZURE_SCOPE), + audience: None, +}; +const DOCUMENT_INTELLIGENCE_AUTH_RULES: &[CredentialRule] = &[ + CredentialRule { + kind: CredentialPlanKind::Static, + placement: CredentialPlacement::Header("Ocp-Apim-Subscription-Key"), + }, + CredentialRule { + kind: CredentialPlanKind::Entra, + placement: CredentialPlacement::Bearer, + }, +]; +const DOCUMENT_INTELLIGENCE_AUTH_POLICY: ProviderAuthPolicy = ProviderAuthPolicy { + rules: DOCUMENT_INTELLIGENCE_AUTH_RULES, + accepted_existing_headers: &["Authorization", "Ocp-Apim-Subscription-Key"], + existing_header_behavior: ExistingHeaderBehavior::Preserve, + scope: Some(crate::auth::azure::DEFAULT_AZURE_SCOPE), + audience: None, +}; + +fn resolve_value( + explicit: Option<&str>, + env_name: &str, + env_lookup: &dyn Fn(&str) -> Option, + missing_error: AuthError, +) -> Result { + explicit + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) + .or_else(|| env_lookup(env_name).filter(|value| !value.trim().is_empty())) + .ok_or(missing_error) +} + +pub fn resolve_api_key( + api_key: Option<&str>, + env_lookup: &dyn Fn(&str) -> Option, +) -> Result { + resolve_value( + api_key, + AZURE_AI_API_KEY_ENV, + env_lookup, + AuthError::MissingApiKey { + provider: "Azure AI", + }, + ) +} + +pub fn resolve_api_base( + api_base: Option<&str>, + env_lookup: &dyn Fn(&str) -> Option, +) -> Result { + resolve_value( + api_base, + AZURE_AI_API_BASE_ENV, + env_lookup, + AuthError::MissingApiBase { + provider: "Azure AI", + environment_variable: AZURE_AI_API_BASE_ENV, + }, + ) +} + +pub fn resolve_document_intelligence_api_key( + api_key: Option<&str>, + env_lookup: &dyn Fn(&str) -> Option, +) -> Result { + resolve_value( + api_key, + AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV, + env_lookup, + AuthError::MissingApiKey { + provider: "Azure Document Intelligence", + }, + ) +} + +pub fn resolve_document_intelligence_endpoint( + api_base: Option<&str>, + env_lookup: &dyn Fn(&str) -> Option, +) -> Result { + resolve_value( + api_base, + AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT_ENV, + env_lookup, + AuthError::MissingApiBase { + provider: "Azure Document Intelligence", + environment_variable: AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT_ENV, + }, + ) +} + +pub fn validate_environment( + headers: Vec<(String, String)>, + api_key: Option<&str>, + env_lookup: &dyn Fn(&str) -> Option, +) -> Result, AuthError> { + if headers + .iter() + .any(|(name, _)| name.eq_ignore_ascii_case("Authorization")) + { + return Ok(headers); + } + let api_key = resolve_api_key(api_key, env_lookup)?; + apply_credential(headers, &api_key, CredentialPlacement::Bearer) +} + +pub async fn authenticate( + headers: Vec<(String, String)>, + api_key: Option<&str>, + auth_inputs: Option<&AzureAuthInputs>, + env_lookup: &(dyn Fn(&str) -> Option + Sync), +) -> Result, AuthError> { + authenticate_with_policy( + headers, + api_key, + AZURE_AI_API_KEY_ENV, + auth_inputs, + env_lookup, + &AZURE_AI_AUTH_POLICY, + MissingCredential::AzureAi, + ) + .await +} + +pub async fn authenticate_document_intelligence( + headers: Vec<(String, String)>, + api_key: Option<&str>, + auth_inputs: Option<&AzureAuthInputs>, + env_lookup: &(dyn Fn(&str) -> Option + Sync), +) -> Result, AuthError> { + authenticate_with_policy( + headers, + api_key, + AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV, + auth_inputs, + env_lookup, + &DOCUMENT_INTELLIGENCE_AUTH_POLICY, + MissingCredential::AzureDocumentIntelligence, + ) + .await +} + +async fn authenticate_with_policy( + headers: Vec<(String, String)>, + api_key: Option<&str>, + api_key_env: &str, + auth_inputs: Option<&AzureAuthInputs>, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + policy: &ProviderAuthPolicy, + missing_credential: MissingCredential, +) -> Result, AuthError> { + if policy.has_existing_credential(&headers) { + return Ok(headers); + } + for rule in policy.rules { + let credential = match rule.kind { + CredentialPlanKind::Static => { + resolve_static_credential(api_key, api_key_env, env_lookup) + } + CredentialPlanKind::Entra => resolve_entra_credential(auth_inputs, env_lookup).await?, + CredentialPlanKind::Caller => { + return Err(AuthError::Configuration( + AuthConfigurationError::MissingCallerInputs, + )); + } + }; + if let Some(credential) = credential { + return policy.apply(headers, rule.kind, &credential); + } + } + Err(AuthError::MissingCredential(missing_credential)) +} + +async fn resolve_entra_credential( + auth_inputs: Option<&AzureAuthInputs>, + env_lookup: &(dyn Fn(&str) -> Option + Sync), +) -> Result, AuthError> { + match auth_inputs { + Some(inputs) => auth_service().resolve(inputs, env_lookup).await, + None => Ok(None), + } +} + +fn resolve_static_credential( + explicit: Option<&str>, + env_name: &str, + env_lookup: &dyn Fn(&str) -> Option, +) -> Option { + let reference = explicit + .filter(|value| !value.is_empty()) + .map(|value| CredentialRef::Explicit(SecretValue::new(value))) + .unwrap_or_else(|| CredentialRef::Env(env_name.to_string())); + match reference { + CredentialRef::Explicit(secret) => Some(ResolvedCredential::Static(secret)), + CredentialRef::Env(name) => env_lookup(&name) + .filter(|value| !value.is_empty()) + .map(SecretValue::new) + .map(ResolvedCredential::Static), + CredentialRef::File(_) + | CredentialRef::Request(_) + | CredentialRef::Host(_) + | CredentialRef::None => None, + } +} + +fn auth_service() -> &'static AzureAuthService { + static SERVICE: OnceLock = OnceLock::new(); + SERVICE.get_or_init(AzureAuthService::default) +} + +#[cfg(test)] +mod tests { + use crate::auth::azure::AzureAuthInputs; + use crate::error::AuthError; + use serde_json::json; + + use super::{ + AZURE_AI_API_KEY_ENV, authenticate, authenticate_document_intelligence, resolve_api_base, + resolve_api_key, validate_environment, + }; + + fn azure_inputs(params: serde_json::Value) -> AzureAuthInputs { + AzureAuthInputs::from_optional_params(params.as_object().expect("object")) + .expect("valid Azure auth inputs") + } + + fn header_value<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> { + headers + .iter() + .find(|(header_name, _)| header_name.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + } + + #[test] + fn missing_api_key_has_a_typed_error() { + assert_eq!( + resolve_api_key(None, &|_| None).expect_err("missing key errors"), + AuthError::MissingApiKey { + provider: "Azure AI", + } + ); + } + + #[test] + fn missing_api_base_has_a_typed_error() { + assert_eq!( + resolve_api_base(None, &|_| None).expect_err("missing base errors"), + AuthError::MissingApiBase { + provider: "Azure AI", + environment_variable: "AZURE_AI_API_BASE", + } + ); + } + + #[test] + fn api_key_is_sent_as_bearer() { + let headers = validate_environment(Vec::new(), Some("key"), &|_| None) + .expect("api key authenticates"); + + assert_eq!( + headers, + vec![("Authorization".to_string(), "Bearer key".to_string())] + ); + } + + #[test] + fn caller_bearer_token_wins_over_environment_key() { + let headers = validate_environment( + vec![( + "authorization".to_string(), + "Bearer caller-token".to_string(), + )], + None, + &|name| (name == AZURE_AI_API_KEY_ENV).then(|| "environment-key".to_string()), + ) + .expect("caller header authenticates"); + + assert_eq!( + headers, + vec![( + "authorization".to_string(), + "Bearer caller-token".to_string() + )] + ); + } + + #[tokio::test] + async fn azure_ai_uses_supplied_entra_token() { + let inputs = azure_inputs(json!({"azure_ad_token": "entra-token"})); + let headers = authenticate(Vec::new(), None, Some(&inputs), &|_| None) + .await + .expect("Entra token authenticates"); + + assert_eq!( + header_value(&headers, "Authorization"), + Some("Bearer entra-token") + ); + } + + #[tokio::test] + async fn document_intelligence_api_key_uses_subscription_header() { + let headers = + authenticate_document_intelligence(Vec::new(), Some("my-key"), None, &|_| None) + .await + .expect("API key authenticates"); + + assert_eq!( + header_value(&headers, "Ocp-Apim-Subscription-Key"), + Some("my-key") + ); + assert_eq!(header_value(&headers, "Authorization"), None); + } + + #[tokio::test] + async fn document_intelligence_falls_back_to_entra_token() { + let inputs = azure_inputs(json!({"azure_ad_token": "entra-token"})); + let headers = + authenticate_document_intelligence(Vec::new(), None, Some(&inputs), &|_| None) + .await + .expect("Entra token authenticates"); + + assert_eq!( + header_value(&headers, "Authorization"), + Some("Bearer entra-token") + ); + assert_eq!(header_value(&headers, "Ocp-Apim-Subscription-Key"), None); + } + + #[tokio::test] + async fn document_intelligence_api_key_precedes_entra_token() { + let inputs = azure_inputs(json!({"azure_ad_token": "entra-token"})); + let headers = + authenticate_document_intelligence(Vec::new(), Some("api-key"), Some(&inputs), &|_| { + None + }) + .await + .expect("API key authenticates"); + + assert_eq!( + header_value(&headers, "Ocp-Apim-Subscription-Key"), + Some("api-key") + ); + assert_eq!(header_value(&headers, "Authorization"), None); + } +} diff --git a/litellm-rust/crates/core/src/providers/azure_ai/mod.rs b/litellm-rust/crates/core/src/providers/azure_ai/mod.rs index 5d13fa93e00..2f99ccb36da 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/mod.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/mod.rs @@ -1,2 +1,2 @@ +pub mod auth; pub mod messages; -pub mod ocr; diff --git a/litellm-rust/crates/core/src/providers/azure_ai/ocr/mod.rs b/litellm-rust/crates/core/src/providers/azure_ai/ocr/mod.rs deleted file mode 100644 index f239b6921fa..00000000000 --- a/litellm-rust/crates/core/src/providers/azure_ai/ocr/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod transformation; diff --git a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs deleted file mode 100644 index d15c032f0bc..00000000000 --- a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs +++ /dev/null @@ -1,1381 +0,0 @@ -use std::collections::BTreeSet; - -use crate::error::{Error, json_type_name}; -use crate::ocr::transformation::{OcrAuthStrategy, OcrProviderConfig, OcrResponseHandling}; -use crate::ocr::types::{OcrRequestData, OcrResponseData}; -use serde_json::{Map, Value, json}; - -use crate::providers::mistral::ocr::transformation::MISTRAL_OCR_CONFIG; - -const AZURE_AI_API_KEY_ENV: &str = "AZURE_AI_API_KEY"; -const AZURE_AI_API_BASE_ENV: &str = "AZURE_AI_API_BASE"; -const AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_API_KEY"; -const AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT"; -const AZURE_DOCUMENT_INTELLIGENCE_API_VERSION: &str = "2024-11-30"; -const AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI: i64 = 96; - -const AZURE_DOCUMENT_INTELLIGENCE_SUPPORTED_OCR_PARAMS: &[&str] = - &["pages", "features", "req_format"]; - -pub struct AzureAiOcrConfig; -pub struct AzureDocumentIntelligenceOcrConfig; - -pub const AZURE_AI_OCR_CONFIG: AzureAiOcrConfig = AzureAiOcrConfig; -pub const AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG: AzureDocumentIntelligenceOcrConfig = - AzureDocumentIntelligenceOcrConfig; - -fn non_empty(value: Option<&str>) -> Option<&str> { - value.map(str::trim).filter(|value| !value.is_empty()) -} - -fn resolve_value( - explicit: Option<&str>, - env_name: &str, - env_lookup: &dyn Fn(&str) -> Option, - missing_message: &str, -) -> Result { - non_empty(explicit) - .map(str::to_string) - .or_else(|| env_lookup(env_name).filter(|value| !value.trim().is_empty())) - .ok_or_else(|| Error::Auth(missing_message.to_string())) -} - -pub fn resolve_azure_ai_api_key( - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result { - resolve_value( - api_key, - AZURE_AI_API_KEY_ENV, - env_lookup, - "Missing Azure AI API Key - A call is being made to Azure AI but no key is set either in the environment variables or via params", - ) -} - -pub fn resolve_azure_ai_api_base( - api_base: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result { - resolve_value( - api_base, - AZURE_AI_API_BASE_ENV, - env_lookup, - "Missing Azure AI API Base - Set AZURE_AI_API_BASE environment variable or pass api_base parameter", - ) -} - -pub fn complete_azure_ai_url( - api_base: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result { - let base = resolve_azure_ai_api_base(api_base, env_lookup)?; - Ok(format!( - "{}/providers/mistral/azure/ocr", - base.trim_end_matches('/') - )) -} - -pub fn resolve_document_intelligence_api_key( - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result { - resolve_value( - api_key, - AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV, - env_lookup, - "Missing Azure Document Intelligence API Key - Set AZURE_DOCUMENT_INTELLIGENCE_API_KEY environment variable or pass api_key parameter", - ) -} - -pub fn resolve_document_intelligence_endpoint( - api_base: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result { - resolve_value( - api_base, - AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT_ENV, - env_lookup, - "Missing Azure Document Intelligence Endpoint - Set AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT environment variable or pass api_base parameter", - ) -} - -fn prepend_auth_header( - headers: Vec<(String, String)>, - name: &str, - value: String, -) -> Vec<(String, String)> { - std::iter::once((name.to_string(), value)) - .chain(headers) - .collect() -} - -pub fn validate_azure_ai_environment( - headers: Vec<(String, String)>, - api_key: Option<&str>, - azure_ad_token: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result, Error> { - if crate::http_utils::has_header(&headers, "Authorization") - || crate::http_utils::has_header(&headers, "Api-Key") - { - return Ok(headers); - } - if let Ok(api_key) = resolve_azure_ai_api_key(api_key, env_lookup) { - return Ok(prepend_auth_header(headers, "Api-Key", api_key)); - } - non_empty(azure_ad_token) - .map(|token| prepend_auth_header(headers, "Authorization", format!("Bearer {token}"))) - .ok_or_else(|| { - Error::Auth( - "Missing Azure AI credentials - set AZURE_AI_API_KEY or provide azure_ad_token" - .to_string(), - ) - }) -} - -pub fn validate_document_intelligence_environment( - headers: Vec<(String, String)>, - api_key: Option<&str>, - azure_ad_token: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result, Error> { - if crate::http_utils::has_header(&headers, "Authorization") - || crate::http_utils::has_header(&headers, "Ocp-Apim-Subscription-Key") - { - return Ok(headers); - } - if let Ok(api_key) = resolve_document_intelligence_api_key(api_key, env_lookup) { - return Ok(prepend_auth_header( - headers, - "Ocp-Apim-Subscription-Key", - api_key, - )); - } - non_empty(azure_ad_token) - .map(|token| prepend_auth_header(headers, "Authorization", format!("Bearer {token}"))) - .ok_or_else(|| { - Error::Auth( - "Missing Azure Document Intelligence credentials - set AZURE_DOCUMENT_INTELLIGENCE_API_KEY or provide azure_ad_token" - .to_string(), - ) - }) -} - -fn encode_model_id(model: &str) -> Result { - let model_id = model.rsplit('/').next().unwrap_or(model); - if matches!(model_id, "." | "..") { - return Err(Error::InvalidRequest( - "model_id cannot be a dot path segment".to_string(), - )); - } - Ok(model_id - .bytes() - .flat_map(|byte| match byte { - b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { - vec![byte as char] - } - _ => format!("%{byte:02X}").chars().collect(), - }) - .collect()) -} - -fn pages_token_is_valid(token: &str) -> bool { - let mut parts = token.split('-'); - let Some(start) = parts.next() else { - return false; - }; - if start.is_empty() || !start.chars().all(|ch| ch.is_ascii_digit()) { - return false; - } - match parts.next() { - None => true, - Some(end) => { - !end.is_empty() && end.chars().all(|ch| ch.is_ascii_digit()) && parts.next().is_none() - } - } -} - -fn normalize_pages_param(pages: &Value) -> Result, Error> { - match pages { - Value::String(value) => { - let normalized = value - .split(',') - .map(str::trim) - .collect::>() - .join(","); - if normalized.split(',').all(pages_token_is_valid) { - Ok(Some(normalized)) - } else { - Err(Error::InvalidRequest(format!( - "Invalid `pages` string for Azure Document Intelligence: {value:?}. Expected format like '1-3,5,7-9'." - ))) - } - } - Value::Array(values) => { - if values.is_empty() { - return Ok(None); - } - if values.iter().any(Value::is_boolean) { - return Err(Error::InvalidRequest( - "`pages` must be integers, not booleans".to_string(), - )); - } - if values.iter().all(Value::is_i64) { - let mut pages = BTreeSet::new(); - for value in values { - let page = value.as_i64().expect("checked is_i64"); - if page < 0 { - return Err(Error::InvalidRequest( - "`pages` integers must be >= 0 (Mistral 0-based indices)".to_string(), - )); - } - pages.insert(page + 1); - } - return Ok(Some( - pages - .into_iter() - .map(|page| page.to_string()) - .collect::>() - .join(","), - )); - } - if values.iter().all(Value::is_string) { - let normalized = values - .iter() - .filter_map(Value::as_str) - .map(str::trim) - .collect::>() - .join(","); - if normalized.split(',').all(pages_token_is_valid) { - return Ok(Some(normalized)); - } - return Err(Error::InvalidRequest(format!( - "Invalid `pages` list for Azure Document Intelligence: {values:?}. Expected tokens like '1' or '3-5'." - ))); - } - Err(Error::InvalidRequest( - "`pages` must be a list[int] (0-based, Mistral-style) or a string like '1-3,5,7-9'." - .to_string(), - )) - } - _ => Err(Error::InvalidRequest( - "`pages` must be a list[int] (0-based, Mistral-style) or a string like '1-3,5,7-9'." - .to_string(), - )), - } -} - -fn feature_token_is_valid(token: &str) -> bool { - let Some((first, rest)) = token.as_bytes().split_first() else { - return false; - }; - first.is_ascii_alphabetic() && rest.iter().all(u8::is_ascii_alphanumeric) -} - -fn invalid_features_error(features: &Value) -> Error { - Error::InvalidRequest(format!( - "Invalid `features` for Azure Document Intelligence: {features:?}. Expected a list of feature names or a comma-separated string like 'keyValuePairs' or 'keyValuePairs,languages'." - )) -} - -fn normalize_features_param(features: &Value) -> Result, Error> { - let normalized = match features { - Value::String(value) => value - .split(',') - .map(str::trim) - .collect::>() - .join(","), - Value::Array(values) if values.is_empty() => return Ok(None), - Value::Array(values) => values - .iter() - .map(Value::as_str) - .collect::>>() - .ok_or_else(|| invalid_features_error(features))? - .into_iter() - .map(str::trim) - .collect::>() - .join(","), - _ => return Err(invalid_features_error(features)), - }; - - if normalized.split(',').all(feature_token_is_valid) { - Ok(Some(normalized)) - } else { - Err(invalid_features_error(features)) - } -} - -fn normalize_req_format(req_format: &Value) -> Result { - match req_format.as_str() { - Some(value @ ("native" | "litellm")) => Ok(value.to_string()), - _ => Err(Error::InvalidRequest(format!( - "Invalid `req_format` for Azure Document Intelligence: {req_format:?}. Expected 'native' or 'litellm'." - ))), - } -} - -pub fn map_document_intelligence_ocr_params( - non_default_params: &Map, -) -> Result, Error> { - let mut mapped = Map::new(); - if let Some(pages) = non_default_params.get("pages") - && let Some(normalized) = normalize_pages_param(pages)? - { - mapped.insert("pages".to_string(), Value::String(normalized)); - } - if let Some(features) = non_default_params.get("features") - && let Some(normalized) = normalize_features_param(features)? - { - mapped.insert("features".to_string(), Value::String(normalized)); - } - if let Some(req_format) = non_default_params.get("req_format") { - mapped.insert( - "req_format".to_string(), - Value::String(normalize_req_format(req_format)?), - ); - } - Ok(mapped) -} - -pub fn complete_document_intelligence_url( - api_base: Option<&str>, - model: &str, - optional_params: &Map, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result { - let endpoint = resolve_document_intelligence_endpoint(api_base, env_lookup)?; - let mut url = format!( - "{}/documentintelligence/documentModels/{}:analyze?api-version={}", - endpoint.trim_end_matches('/'), - encode_model_id(model)?, - AZURE_DOCUMENT_INTELLIGENCE_API_VERSION - ); - - if let Some(pages) = optional_params.get("pages") - && let Some(normalized) = normalize_pages_param(pages)? - { - url.push_str("&pages="); - url.push_str(&normalized); - } - - if let Some(features) = optional_params.get("features") - && let Some(normalized) = normalize_features_param(features)? - { - url.push_str("&features="); - url.push_str(&normalized); - } - - if let Some(req_format) = optional_params.get("req_format") { - normalize_req_format(req_format)?; - } - - Ok(url) -} - -fn document_url_from_mistral_document(document: &Value) -> Result<&str, Error> { - let object = document.as_object().ok_or_else(|| Error::InvalidType { - expected: "object", - actual: json_type_name(document), - })?; - let doc_type = object - .get("type") - .and_then(Value::as_str) - .ok_or(Error::MissingField("document.type"))?; - let field_name = match doc_type { - "document_url" => "document_url", - "image_url" => "image_url", - other => { - return Err(Error::InvalidRequest(format!( - "Invalid document type: {other}. Must be 'document_url' or 'image_url'" - ))); - } - }; - object - .get(field_name) - .and_then(Value::as_str) - .filter(|value| !value.is_empty()) - .ok_or(Error::MissingField(field_name)) -} - -fn extract_base64_from_data_uri(data_uri: &str) -> &str { - data_uri - .split_once(',') - .map(|(_, data)| data) - .unwrap_or(data_uri) -} - -fn page_markdown(page: &Map) -> String { - page.get("lines") - .and_then(Value::as_array) - .map(|lines| { - lines - .iter() - .filter_map(|line| line.get("content").and_then(Value::as_str)) - .collect::>() - .join("\n") - }) - .unwrap_or_default() -} - -fn page_dimensions(page: &Map) -> Value { - let width = page.get("width").and_then(Value::as_f64).unwrap_or(8.5); - let height = page.get("height").and_then(Value::as_f64).unwrap_or(11.0); - let unit = page.get("unit").and_then(Value::as_str).unwrap_or("inch"); - let (width, height) = if unit == "inch" { - ( - (width * AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI as f64) as i64, - (height * AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI as f64) as i64, - ) - } else { - (width as i64, height as i64) - }; - json!({ - "width": width, - "height": height, - "dpi": AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI, - }) -} - -fn transform_document_intelligence_response( - model: &str, - response_json: Value, - preserve_native_response: bool, -) -> Result { - let response = response_json - .as_object() - .ok_or_else(|| Error::InvalidType { - expected: "object", - actual: json_type_name(&response_json), - })?; - let status = response - .get("status") - .and_then(Value::as_str) - .ok_or(Error::MissingField("status"))?; - if status != "succeeded" { - return Err(Error::InvalidResponse(format!( - "Azure Document Intelligence analysis failed with status: {status}" - ))); - } - - let analyze_result = response.get("analyzeResult").and_then(Value::as_object); - let azure_pages = analyze_result - .and_then(|result| result.get("pages")) - .and_then(Value::as_array) - .cloned() - .unwrap_or_default(); - let pages = azure_pages - .iter() - .filter_map(Value::as_object) - .map(|page| { - let page_number = page.get("pageNumber").and_then(Value::as_i64).unwrap_or(1); - json!({ - "index": page_number - 1, - "markdown": page_markdown(page), - "dimensions": page_dimensions(page), - }) - }) - .collect::>(); - let extra_fields = ["content", "tables", "keyValuePairs"] - .into_iter() - .map(|field| { - ( - field.to_string(), - analyze_result - .and_then(|result| result.get(field)) - .cloned() - .unwrap_or(Value::Null), - ) - }) - .collect(); - - Ok(OcrResponseData { - usage_info: Some(json!({ - "pages_processed": pages.len(), - "doc_size_bytes": null, - })), - pages, - model: model.to_string(), - document_annotation: None, - object: "ocr".to_string(), - extra_fields, - provider_native_response: preserve_native_response.then_some(response_json), - }) -} - -impl OcrProviderConfig for AzureAiOcrConfig { - fn supported_ocr_params(&self) -> &'static [&'static str] { - MISTRAL_OCR_CONFIG.supported_ocr_params() - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_request( - &self, - model: &str, - document: Value, - optional_params: Map, - ) -> Result { - MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params) - } - - fn transform_ocr_response( - &self, - model: &str, - response_json: Value, - ) -> Result { - MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn complete_url( - &self, - api_base: Option<&str>, - _model: &str, - _optional_params: &Map, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - complete_azure_ai_url(api_base, env_lookup) - } - - fn resolve_api_key( - &self, - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - resolve_azure_ai_api_key(api_key, env_lookup) - } - - fn requires_data_uri_document(&self) -> bool { - true - } -} - -impl OcrProviderConfig for AzureDocumentIntelligenceOcrConfig { - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn supported_ocr_params(&self) -> &'static [&'static str] { - AZURE_DOCUMENT_INTELLIGENCE_SUPPORTED_OCR_PARAMS - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn map_ocr_params(&self, non_default_params: &Map) -> Map { - map_document_intelligence_ocr_params(non_default_params).unwrap_or_else(|_| { - non_default_params - .iter() - .filter(|(name, _)| { - AZURE_DOCUMENT_INTELLIGENCE_SUPPORTED_OCR_PARAMS.contains(&name.as_str()) - }) - .map(|(name, value)| (name.clone(), value.clone())) - .collect() - }) - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_request( - &self, - _model: &str, - document: Value, - _optional_params: Map, - ) -> Result { - let document_url = document_url_from_mistral_document(&document)?; - let mut data = Map::new(); - if document_url.starts_with("data:") { - data.insert( - "base64Source".to_string(), - Value::String(extract_base64_from_data_uri(document_url).to_string()), - ); - } else { - data.insert( - "urlSource".to_string(), - Value::String(document_url.to_string()), - ); - } - Ok(OcrRequestData { - data: Value::Object(data), - files: None, - }) - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_response( - &self, - model: &str, - response_json: Value, - ) -> Result { - transform_document_intelligence_response(model, response_json, false) - } - - fn transform_ocr_response_with_params( - &self, - model: &str, - response_json: Value, - optional_params: &Map, - ) -> Result { - transform_document_intelligence_response( - model, - response_json, - optional_params.get("req_format").and_then(Value::as_str) == Some("native"), - ) - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn complete_url( - &self, - api_base: Option<&str>, - model: &str, - optional_params: &Map, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - complete_document_intelligence_url(api_base, model, optional_params, env_lookup) - } - - fn resolve_api_key( - &self, - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - resolve_document_intelligence_api_key(api_key, env_lookup) - } - - fn auth_strategy(&self) -> OcrAuthStrategy { - OcrAuthStrategy::Header("Ocp-Apim-Subscription-Key") - } - - fn response_handling(&self) -> OcrResponseHandling { - OcrResponseHandling::AzureDocumentIntelligencePoll - } -} - -#[cfg(test)] -mod tests { - use super::*; - use rstest::{fixture, rstest}; - - const ENDPOINT: &str = "https://example.cognitiveservices.azure.com"; - - #[fixture] - fn document_intelligence_config() -> AzureDocumentIntelligenceOcrConfig { - AzureDocumentIntelligenceOcrConfig - } - - fn header_value<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> { - headers - .iter() - .find(|(header_name, _)| header_name.eq_ignore_ascii_case(name)) - .map(|(_, value)| value.as_str()) - } - - #[fixture] - fn native_operation() -> Value { - json!({ - "status": "succeeded", - "createdDateTime": "2026-07-02T00:00:00Z", - "lastUpdatedDateTime": "2026-07-02T00:00:05Z", - "analyzeResult": { - "content": "Invoice\nInvoice No: INV-12345\nTotal: $100.00", - "pages": [{ - "pageNumber": 1, - "width": 8.5, - "height": 11, - "unit": "inch", - "angle": 0.13, - "lines": [ - {"content": "Invoice"}, - {"content": "Invoice No: INV-12345"}, - {"content": "Total: $100.00"} - ], - "words": [{"content": "Invoice", "confidence": 0.994}] - }], - "tables": [ - { - "rowCount": 2, - "columnCount": 2, - "cells": [ - {"kind": "columnHeader", "rowIndex": 0, "columnIndex": 0, "content": "Item"}, - {"kind": "columnHeader", "rowIndex": 0, "columnIndex": 1, "content": "Price"}, - {"rowIndex": 1, "columnIndex": 0, "content": "Widget"}, - {"rowIndex": 1, "columnIndex": 1, "content": "$100.00"} - ] - }, - { - "rowCount": 1, - "columnCount": 1, - "cells": [{"rowIndex": 0, "columnIndex": 0, "content": "Totals"}] - } - ], - "keyValuePairs": [ - { - "key": {"content": "Invoice No"}, - "value": {"content": "INV-12345"}, - "confidence": 0.98 - }, - { - "key": {"content": "Total"}, - "value": {"content": "$100.00"}, - "confidence": 0.95 - } - ], - "paragraphs": [{"content": "Invoice"}] - } - }) - } - - fn assert_native_fields_preserved(response: &OcrResponseData, operation: &Value) { - let analyze_result = &operation["analyzeResult"]; - - assert_eq!(response.extra_fields["content"], analyze_result["content"]); - assert_eq!(response.extra_fields["tables"], analyze_result["tables"]); - assert_eq!( - response.extra_fields["keyValuePairs"], - analyze_result["keyValuePairs"] - ); - assert_eq!(response.object, "ocr"); - assert_eq!( - response.usage_info, - Some(json!({"pages_processed": 1, "doc_size_bytes": null})) - ); - assert_eq!(response.pages[0]["index"], 0); - assert_eq!( - response.pages[0]["markdown"], - "Invoice\nInvoice No: INV-12345\nTotal: $100.00" - ); - assert_eq!( - response.pages[0]["dimensions"], - json!({"width": 816, "height": 1056, "dpi": 96}) - ); - } - - #[test] - fn azure_ai_reuses_mistral_body_transform() { - let body = AZURE_AI_OCR_CONFIG - .transform_ocr_request( - "pixtral-12b-2409", - json!({"type": "document_url", "document_url": "data:application/pdf;base64,abc"}), - serde_json::Map::from_iter([("include_image_base64".to_string(), json!(true))]), - ) - .expect("request transforms") - .data; - - assert_eq!(body["model"], "pixtral-12b-2409"); - assert_eq!(body["include_image_base64"], true); - assert_eq!( - body["document"]["document_url"], - "data:application/pdf;base64,abc" - ); - } - - #[test] - fn document_intelligence_url_normalizes_zero_based_pages() { - let params = serde_json::Map::from_iter([("pages".to_string(), json!([2, 0, 2]))]); - let url = complete_document_intelligence_url( - Some("https://example.cognitiveservices.azure.com/"), - "azure_ai/doc-intelligence/prebuilt-layout", - ¶ms, - &|_| None, - ) - .expect("url builds"); - - assert_eq!( - url, - "https://example.cognitiveservices.azure.com/documentintelligence/documentModels/prebuilt-layout:analyze?api-version=2024-11-30&pages=1,3" - ); - } - - #[test] - fn document_intelligence_url_normalizes_features() { - let params = serde_json::Map::from_iter([( - "features".to_string(), - json!("keyValuePairs, languages"), - )]); - let url = complete_document_intelligence_url( - Some("https://example.cognitiveservices.azure.com"), - "prebuilt-layout", - ¶ms, - &|_| None, - ) - .expect("url builds"); - - assert_eq!( - url, - "https://example.cognitiveservices.azure.com/documentintelligence/documentModels/prebuilt-layout:analyze?api-version=2024-11-30&features=keyValuePairs,languages" - ); - } - - #[test] - fn document_intelligence_url_combines_pages_and_feature_list() { - let params = serde_json::Map::from_iter([ - ("pages".to_string(), json!([0, 1, 2])), - ( - "features".to_string(), - json!([" keyValuePairs ", "languages"]), - ), - ]); - let url = complete_document_intelligence_url( - Some("https://example.cognitiveservices.azure.com"), - "prebuilt-layout", - ¶ms, - &|_| None, - ) - .expect("url builds"); - - assert_eq!( - url, - "https://example.cognitiveservices.azure.com/documentintelligence/documentModels/prebuilt-layout:analyze?api-version=2024-11-30&pages=1,2,3&features=keyValuePairs,languages" - ); - } - - #[test] - fn document_intelligence_url_omits_empty_feature_list() { - let params = serde_json::Map::from_iter([("features".to_string(), json!([]))]); - assert!( - map_document_intelligence_ocr_params(¶ms) - .expect("empty features map") - .is_empty() - ); - let url = complete_document_intelligence_url( - Some("https://example.cognitiveservices.azure.com"), - "prebuilt-layout", - ¶ms, - &|_| None, - ) - .expect("url builds"); - - assert_eq!( - url, - "https://example.cognitiveservices.azure.com/documentintelligence/documentModels/prebuilt-layout:analyze?api-version=2024-11-30" - ); - } - - #[rstest] - #[case::query_injection(json!("keyValuePairs&pages=9"))] - #[case::spaces(json!("key value pairs"))] - #[case::empty_string(json!(""))] - #[case::integer_list(json!([1, 2]))] - #[case::nested_list(json!([["keyValuePairs"]]))] - #[case::object(json!({"feature": "keyValuePairs"}))] - #[case::number(json!(5))] - fn document_intelligence_mapping_rejects_invalid_features(#[case] features: Value) { - let params = serde_json::Map::from_iter([("features".to_string(), features)]); - let error = - map_document_intelligence_ocr_params(¶ms).expect_err("invalid features must fail"); - - assert!(matches!( - error, - Error::InvalidRequest(message) if message.contains("Invalid `features`") - )); - } - - #[rstest] - #[case::single_list(json!(["keyValuePairs"]), "keyValuePairs")] - #[case::multiple_list( - json!(["keyValuePairs", "languages"]), - "keyValuePairs,languages" - )] - #[case::single_string(json!("keyValuePairs"), "keyValuePairs")] - #[case::comma_separated(json!("keyValuePairs,languages"), "keyValuePairs,languages")] - #[case::spaces(json!("keyValuePairs, languages"), "keyValuePairs,languages")] - fn document_intelligence_maps_features(#[case] features: Value, #[case] expected: &str) { - let params = Map::from_iter([ - ("features".to_string(), features), - ("unsupported".to_string(), json!(true)), - ]); - - assert_eq!( - AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG.map_ocr_params(¶ms), - Map::from_iter([("features".to_string(), json!(expected))]) - ); - } - - #[test] - fn document_intelligence_request_uses_base64_source_for_data_uri() { - let body = AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG - .transform_ocr_request( - "prebuilt-read", - json!({"type": "document_url", "document_url": "data:application/pdf;base64,abc123"}), - Map::new(), - ) - .expect("request transforms") - .data; - - assert_eq!(body, json!({"base64Source": "abc123"})); - } - - #[rstest] - fn document_intelligence_response_normalizes_pages(native_operation: Value) { - let response = AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG - .transform_ocr_response("prebuilt-layout", native_operation.clone()) - .expect("response transforms"); - - assert_native_fields_preserved(&response, &native_operation); - } - - #[test] - fn azure_document_intelligence_model_id_is_encoded() { - let url = complete_document_intelligence_url( - Some(ENDPOINT), - "prebuilt-layout?x=1#frag", - &Map::new(), - &|_| None, - ) - .expect("url builds"); - - assert_eq!( - url, - "https://example.cognitiveservices.azure.com/documentintelligence/documentModels/prebuilt-layout%3Fx%3D1%23frag:analyze?api-version=2024-11-30" - ); - } - - #[test] - fn azure_document_intelligence_dot_segment_model_id_is_rejected() { - let error = complete_document_intelligence_url( - Some(ENDPOINT), - "azure_ai/doc-intelligence/..", - &Map::new(), - &|_| None, - ) - .expect_err("dot segment must fail"); - - assert_eq!( - error, - Error::InvalidRequest("model_id cannot be a dot path segment".to_string()) - ); - } - - #[rstest] - fn document_intelligence_async_response_preserves_normalized_fields(native_operation: Value) { - let response = AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG - .transform_ocr_response( - "azure_ai/doc-intelligence/prebuilt-layout", - native_operation.clone(), - ) - .expect("response transforms"); - - assert_native_fields_preserved(&response, &native_operation); - } - - #[test] - fn document_intelligence_response_tolerates_missing_native_fields() { - let response = AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG - .transform_ocr_response( - "azure_ai/doc-intelligence/prebuilt-read", - json!({ - "status": "succeeded", - "analyzeResult": { - "pages": [{ - "pageNumber": 1, - "width": 8.5, - "height": 11, - "unit": "inch", - "lines": [{"content": "hello"}] - }] - } - }), - ) - .expect("missing optional fields are allowed"); - - assert_eq!(response.pages[0]["markdown"], "hello"); - assert_eq!(response.extra_fields["content"], Value::Null); - assert_eq!(response.extra_fields["tables"], Value::Null); - assert_eq!(response.extra_fields["keyValuePairs"], Value::Null); - } - - #[test] - fn document_intelligence_non_succeeded_status_is_rejected() { - let error = AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG - .transform_ocr_response( - "azure_ai/doc-intelligence/prebuilt-layout", - json!({"status": "failed"}), - ) - .expect_err("failed status must fail"); - - assert_eq!( - error, - Error::InvalidResponse( - "Azure Document Intelligence analysis failed with status: failed".to_string() - ) - ); - } - - #[test] - fn document_intelligence_supported_params_include_features() { - assert_eq!( - AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG.supported_ocr_params(), - &["pages", "features", "req_format"] - ); - } - - #[rstest] - fn document_intelligence_native_format_carries_raw_operation(native_operation: Value) { - let response = AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG - .transform_ocr_response_with_params( - "azure_ai/doc-intelligence/prebuilt-layout", - native_operation.clone(), - &Map::from_iter([("req_format".to_string(), json!("native"))]), - ) - .expect("native response transforms"); - - assert_eq!( - response.provider_native_response, - Some(native_operation.clone()) - ); - assert_native_fields_preserved(&response, &native_operation); - } - - #[rstest] - fn document_intelligence_async_native_format_carries_raw_operation(native_operation: Value) { - let response = AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG - .transform_ocr_response_with_params( - "azure_ai/doc-intelligence/prebuilt-layout", - native_operation.clone(), - &Map::from_iter([("req_format".to_string(), json!("native"))]), - ) - .expect("native response transforms"); - - assert_eq!( - response.provider_native_response, - Some(native_operation.clone()) - ); - assert_native_fields_preserved(&response, &native_operation); - } - - #[rstest] - #[case::default(Map::new())] - #[case::litellm(Map::from_iter([("req_format".to_string(), json!("litellm"))]))] - fn document_intelligence_default_format_omits_raw_operation( - #[case] optional_params: Map, - native_operation: Value, - ) { - let response = AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG - .transform_ocr_response_with_params( - "azure_ai/doc-intelligence/prebuilt-layout", - native_operation.clone(), - &optional_params, - ) - .expect("response transforms"); - - assert_eq!(response.provider_native_response, None); - assert_native_fields_preserved(&response, &native_operation); - } - - #[rstest] - #[case::native("native")] - #[case::litellm("litellm")] - fn document_intelligence_maps_req_format(#[case] req_format: &str) { - let mapped = map_document_intelligence_ocr_params(&Map::from_iter([( - "req_format".to_string(), - json!(req_format), - )])) - .expect("req_format maps"); - - assert_eq!( - mapped, - Map::from_iter([("req_format".to_string(), json!(req_format))]) - ); - } - - #[test] - fn document_intelligence_rejects_unknown_req_format() { - let error = map_document_intelligence_ocr_params(&Map::from_iter([( - "req_format".to_string(), - json!("azure"), - )])) - .expect_err("unknown req_format must fail"); - - assert!( - matches!(error, Error::InvalidRequest(message) if message.contains("Invalid `req_format`")) - ); - } - - #[test] - fn document_intelligence_url_omits_req_format() { - let url = complete_document_intelligence_url( - Some(ENDPOINT), - "prebuilt-layout", - &Map::from_iter([("req_format".to_string(), json!("native"))]), - &|_| None, - ) - .expect("url builds"); - - assert!(!url.contains("req_format")); - } - - #[test] - fn document_intelligence_validate_environment_uses_subscription_key() { - let headers = - validate_document_intelligence_environment(Vec::new(), Some("my-key"), None, &|_| None) - .expect("api key authenticates"); - - assert_eq!( - header_value(&headers, "Ocp-Apim-Subscription-Key"), - Some("my-key") - ); - } - - #[test] - fn document_intelligence_validate_environment_falls_back_to_entra_token() { - let headers = validate_document_intelligence_environment( - Vec::new(), - None, - Some("entra-token"), - &|_| None, - ) - .expect("Entra token authenticates"); - - assert_eq!( - header_value(&headers, "Authorization"), - Some("Bearer entra-token") - ); - assert_eq!(header_value(&headers, "Ocp-Apim-Subscription-Key"), None); - } - - #[test] - fn document_intelligence_supported_params_include_pages_features_and_req_format() { - assert_eq!( - AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG.supported_ocr_params(), - &["pages", "features", "req_format"] - ); - } - - #[test] - fn document_intelligence_maps_zero_based_page_list() { - let mapped = map_document_intelligence_ocr_params(&Map::from_iter([( - "pages".to_string(), - json!([0, 1, 2]), - )])) - .expect("pages map"); - - assert_eq!( - mapped, - Map::from_iter([("pages".to_string(), json!("1,2,3"))]) - ); - } - - #[test] - fn document_intelligence_page_mapping_dedupes_and_sorts() { - let mapped = map_document_intelligence_ocr_params(&Map::from_iter([( - "pages".to_string(), - json!([2, 0, 0, 1]), - )])) - .expect("pages map"); - - assert_eq!(mapped["pages"], "1,2,3"); - } - - #[test] - fn document_intelligence_page_mapping_omits_empty_list() { - let mapped = map_document_intelligence_ocr_params(&Map::from_iter([( - "pages".to_string(), - json!([]), - )])) - .expect("empty pages map"); - - assert!(mapped.is_empty()); - } - - #[test] - fn document_intelligence_page_mapping_accepts_native_range() { - let mapped = map_document_intelligence_ocr_params(&Map::from_iter([( - "pages".to_string(), - json!("3-9"), - )])) - .expect("range maps"); - - assert_eq!(mapped["pages"], "3-9"); - } - - #[test] - fn document_intelligence_page_mapping_strips_spaces() { - let mapped = map_document_intelligence_ocr_params(&Map::from_iter([( - "pages".to_string(), - json!("1-3, 5"), - )])) - .expect("range maps"); - - assert_eq!(mapped["pages"], "1-3,5"); - } - - #[test] - fn document_intelligence_page_mapping_accepts_string_tokens() { - let mapped = map_document_intelligence_ocr_params(&Map::from_iter([( - "pages".to_string(), - json!(["1", "3-5"]), - )])) - .expect("tokens map"); - - assert_eq!(mapped["pages"], "1,3-5"); - } - - #[test] - fn document_intelligence_page_mapping_rejects_invalid_string() { - let error = map_document_intelligence_ocr_params(&Map::from_iter([( - "pages".to_string(), - json!("a,b"), - )])) - .expect_err("invalid pages must fail"); - - assert!( - matches!(error, Error::InvalidRequest(message) if message.contains("Invalid `pages` string")) - ); - } - - #[test] - fn document_intelligence_page_mapping_rejects_negative_index() { - let error = map_document_intelligence_ocr_params(&Map::from_iter([( - "pages".to_string(), - json!([-1]), - )])) - .expect_err("negative pages must fail"); - - assert!( - matches!(error, Error::InvalidRequest(message) if message.contains("must be >= 0")) - ); - } - - #[test] - fn document_intelligence_page_mapping_rejects_bool_list() { - let error = map_document_intelligence_ocr_params(&Map::from_iter([( - "pages".to_string(), - json!([true, false]), - )])) - .expect_err("boolean pages must fail"); - - assert!( - matches!(error, Error::InvalidRequest(message) if message.contains("integers, not booleans")) - ); - } - - #[test] - fn document_intelligence_page_mapping_rejects_unsupported_type() { - let error = map_document_intelligence_ocr_params(&Map::from_iter([( - "pages".to_string(), - json!(5), - )])) - .expect_err("unsupported pages must fail"); - - assert!( - matches!(error, Error::InvalidRequest(message) if message.contains("Mistral-style")) - ); - } - - #[test] - fn document_intelligence_url_appends_pages_query() { - let url = complete_document_intelligence_url( - Some("https://example.cognitiveservices.azure.com/"), - "azure_ai/doc-intelligence/prebuilt-layout", - &Map::from_iter([("pages".to_string(), json!("1-3,5"))]), - &|_| None, - ) - .expect("url builds"); - - assert!(url.contains("api-version=2024-11-30")); - assert!(url.contains("pages=1-3,5")); - assert!(url.contains("/documentintelligence/documentModels/prebuilt-layout:analyze")); - } - - #[test] - fn document_intelligence_url_has_no_pages_when_params_are_empty() { - let url = complete_document_intelligence_url( - Some(ENDPOINT), - "prebuilt-layout", - &Map::new(), - &|_| None, - ) - .expect("url builds"); - - assert!(!url.contains("pages=")); - } - - #[rstest] - fn document_intelligence_request_keeps_pages_out_of_body( - document_intelligence_config: AzureDocumentIntelligenceOcrConfig, - ) { - let request = document_intelligence_config - .transform_ocr_request( - "prebuilt-layout", - json!({"type": "document_url", "document_url": "https://example.com/x.pdf"}), - Map::from_iter([("pages".to_string(), json!("1,2,3"))]), - ) - .expect("request transforms"); - - assert_eq!( - request.data, - json!({"urlSource": "https://example.com/x.pdf"}) - ); - } - - #[test] - fn document_intelligence_mistral_pages_flow_to_query_only() { - let mapped = map_document_intelligence_ocr_params(&Map::from_iter([( - "pages".to_string(), - json!([2, 3, 4, 5, 6, 7, 8]), - )])) - .expect("pages map"); - let url = - complete_document_intelligence_url(Some(ENDPOINT), "prebuilt-layout", &mapped, &|_| { - None - }) - .expect("url builds"); - let request = AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG - .transform_ocr_request( - "prebuilt-layout", - json!({"type": "document_url", "document_url": "https://example.com/x.pdf"}), - mapped, - ) - .expect("request transforms"); - - assert!(url.contains("pages=3,4,5,6,7,8,9")); - assert_eq!( - request.data, - json!({"urlSource": "https://example.com/x.pdf"}) - ); - } - - #[test] - fn document_intelligence_endpoint_ignores_generic_azure_ai_base() { - let resolved = resolve_document_intelligence_endpoint(None, &|name| match name { - AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT_ENV => Some(ENDPOINT.to_string()), - AZURE_AI_API_BASE_ENV => Some("https://generic.example.com".to_string()), - _ => None, - }) - .expect("endpoint resolves"); - - assert_eq!(resolved, ENDPOINT); - } - - #[test] - fn document_intelligence_endpoint_honors_explicit_api_base() { - let resolved = resolve_document_intelligence_endpoint( - Some("https://my-di.cognitiveservices.azure.com"), - &|name| match name { - AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT_ENV => Some(ENDPOINT.to_string()), - AZURE_AI_API_BASE_ENV => Some("https://generic.example.com".to_string()), - _ => None, - }, - ) - .expect("endpoint resolves"); - - assert_eq!(resolved, "https://my-di.cognitiveservices.azure.com"); - } - - #[test] - fn azure_ai_mistral_ocr_uses_generic_api_base() { - let resolved = resolve_azure_ai_api_base(None, &|name| match name { - AZURE_AI_API_BASE_ENV => Some("https://generic-azure-ai.example.com".to_string()), - AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT_ENV => Some(ENDPOINT.to_string()), - _ => None, - }) - .expect("api base resolves"); - - assert_eq!(resolved, "https://generic-azure-ai.example.com"); - } - - #[test] - fn azure_ai_ocr_authenticates_with_entra_token() { - let headers = - validate_azure_ai_environment(Vec::new(), None, Some("entra-token"), &|_| None) - .expect("Entra token authenticates"); - - assert_eq!( - header_value(&headers, "Authorization"), - Some("Bearer entra-token") - ); - } -} diff --git a/litellm-rust/crates/core/src/providers/mistral/auth.rs b/litellm-rust/crates/core/src/providers/mistral/auth.rs new file mode 100644 index 00000000000..f847152372e --- /dev/null +++ b/litellm-rust/crates/core/src/providers/mistral/auth.rs @@ -0,0 +1,100 @@ +use crate::auth::{ + CredentialPlacement, CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, + ProviderAuthPolicy, ResolvedCredential, SecretValue, +}; +use crate::error::AuthError; + +pub const MISTRAL_API_KEY_ENV: &str = "MISTRAL_API_KEY"; + +const AUTH_RULES: &[CredentialRule] = &[CredentialRule { + kind: CredentialPlanKind::Static, + placement: CredentialPlacement::Bearer, +}]; +const AUTH_POLICY: ProviderAuthPolicy = ProviderAuthPolicy { + rules: AUTH_RULES, + accepted_existing_headers: &["Authorization"], + existing_header_behavior: ExistingHeaderBehavior::Preserve, + scope: None, + audience: None, +}; + +pub fn resolve_api_key( + api_key: Option<&str>, + env_lookup: &dyn Fn(&str) -> Option, +) -> Result { + api_key + .map(str::trim) + .filter(|key| !key.is_empty()) + .map(str::to_string) + .or_else(|| env_lookup(MISTRAL_API_KEY_ENV).filter(|key| !key.trim().is_empty())) + .ok_or(AuthError::MissingApiKey { + provider: "Mistral", + }) +} + +pub fn validate_environment( + headers: Vec<(String, String)>, + api_key: Option<&str>, + env_lookup: &dyn Fn(&str) -> Option, +) -> Result, AuthError> { + if AUTH_POLICY.has_existing_credential(&headers) { + return Ok(headers); + } + let credential = + ResolvedCredential::Static(SecretValue::new(resolve_api_key(api_key, env_lookup)?)); + AUTH_POLICY.apply(headers, CredentialPlanKind::Static, &credential) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn explicit_key_precedes_environment_key() { + let env_lookup = |name: &str| (name == MISTRAL_API_KEY_ENV).then(|| "sk-env".to_string()); + + assert_eq!( + resolve_api_key(Some("sk-param"), &env_lookup).unwrap(), + "sk-param" + ); + } + + #[test] + fn blank_explicit_key_uses_environment_key() { + let env_lookup = |name: &str| (name == MISTRAL_API_KEY_ENV).then(|| "sk-env".to_string()); + + assert_eq!(resolve_api_key(Some(" "), &env_lookup).unwrap(), "sk-env"); + } + + #[test] + fn missing_key_is_typed() { + let error = resolve_api_key(None, &|_| None).expect_err("missing key should error"); + + assert_eq!( + error, + AuthError::MissingApiKey { + provider: "Mistral", + } + ); + } + + #[test] + fn static_key_is_applied_as_bearer() { + let headers = validate_environment(Vec::new(), Some("sk-param"), &|_| None).unwrap(); + + assert_eq!( + headers, + vec![("Authorization".to_string(), "Bearer sk-param".to_string())] + ); + } + + #[test] + fn existing_authorization_is_preserved_without_key_lookup() { + let headers = vec![("authorization".to_string(), "Bearer existing".to_string())]; + + assert_eq!( + validate_environment(headers.clone(), None, &|_| None).unwrap(), + headers + ); + } +} diff --git a/litellm-rust/crates/core/src/providers/mistral/mod.rs b/litellm-rust/crates/core/src/providers/mistral/mod.rs index 3621ff6a2fd..0e4a05d5978 100644 --- a/litellm-rust/crates/core/src/providers/mistral/mod.rs +++ b/litellm-rust/crates/core/src/providers/mistral/mod.rs @@ -1 +1 @@ -pub mod ocr; +pub mod auth; diff --git a/litellm-rust/crates/core/src/providers/mistral/ocr/mod.rs b/litellm-rust/crates/core/src/providers/mistral/ocr/mod.rs deleted file mode 100644 index f239b6921fa..00000000000 --- a/litellm-rust/crates/core/src/providers/mistral/ocr/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod transformation; diff --git a/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs deleted file mode 100644 index 11e8fe7db18..00000000000 --- a/litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs +++ /dev/null @@ -1,433 +0,0 @@ -use crate::error::{Error, json_type_name}; -use crate::ocr::transformation::OcrProviderConfig; -use crate::ocr::types::{OcrRequestData, OcrResponseData}; -use serde_json::{Map, Value}; - -const SUPPORTED_OCR_PARAMS: &[&str] = &[ - "pages", - "include_image_base64", - "image_limit", - "image_min_size", - "bbox_annotation_format", - "document_annotation_format", - "document_annotation_prompt", - "extract_header", - "extract_footer", - "table_format", - "confidence_scores_granularity", - "include_blocks", - "id", -]; - -/// Default Mistral API base, used when the caller does not override `api_base`. -pub const MISTRAL_DEFAULT_API_BASE: &str = "https://api.mistral.ai/v1"; - -/// Environment variable holding the Mistral API key. -pub const MISTRAL_API_KEY_ENV: &str = "MISTRAL_API_KEY"; - -/// Error message raised when no Mistral API key can be resolved. -pub const MISSING_KEY_MESSAGE: &str = "Missing Mistral API Key - A call is being made to Mistral but no key is set either in the environment variables or via params"; - -/// Build the complete OCR endpoint URL, de-duplicating a trailing `/v1`. -/// -/// Blank/whitespace `api_base` is treated as absent (guard at resolution time). -pub fn complete_url(api_base: Option<&str>) -> String { - let base = api_base - .map(str::trim) - .filter(|base| !base.is_empty()) - .unwrap_or(MISTRAL_DEFAULT_API_BASE) - .trim_end_matches('/'); - - if base.ends_with("/v1") { - format!("{base}/ocr") - } else { - format!("{base}/v1/ocr") - } -} - -/// Resolve the Mistral API key from the explicit param or the environment. -/// -/// Blank/whitespace values are treated as absent. Returns `Error::Auth` -/// when no usable key is available. -/// -/// Note: the env fallback only reads the process environment. Secret-manager -/// backends (AWS/Azure/GCP/Vault) are resolved on the Python side and passed in -/// via `api_key`; this fallback is a last resort for direct/standalone use. -pub fn resolve_api_key( - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result { - api_key - .map(str::trim) - .filter(|key| !key.is_empty()) - .map(str::to_string) - .or_else(|| env_lookup(MISTRAL_API_KEY_ENV).filter(|key| !key.trim().is_empty())) - .ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string())) -} - -pub struct MistralOcrConfig; - -pub const MISTRAL_OCR_CONFIG: MistralOcrConfig = MistralOcrConfig; - -impl OcrProviderConfig for MistralOcrConfig { - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn supported_ocr_params(&self) -> &'static [&'static str] { - SUPPORTED_OCR_PARAMS - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_request( - &self, - model: &str, - document: Value, - optional_params: Map, - ) -> Result { - if !document.is_object() { - return Err(Error::InvalidType { - expected: "object", - actual: json_type_name(&document), - }); - } - - let mut data = Map::new(); - data.insert("model".to_string(), Value::String(model.to_string())); - data.insert("document".to_string(), document); - for (param, value) in optional_params { - data.insert(param, value); - } - - Ok(OcrRequestData { - data: Value::Object(data), - files: None, - }) - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_response( - &self, - model: &str, - response_json: Value, - ) -> Result { - let response_object = response_json - .as_object() - .ok_or_else(|| Error::InvalidType { - expected: "object", - actual: json_type_name(&response_json), - })?; - - let pages = response_object - .get("pages") - .and_then(Value::as_array) - .cloned() - .unwrap_or_default(); - let model = response_object - .get("model") - .and_then(Value::as_str) - .unwrap_or(model) - .to_string(); - let document_annotation = response_object.get("document_annotation").cloned(); - let usage_info = response_object.get("usage_info").cloned(); - - Ok(OcrResponseData { - pages, - model, - document_annotation, - usage_info, - object: "ocr".to_string(), - extra_fields: Map::new(), - provider_native_response: None, - }) - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn complete_url( - &self, - api_base: Option<&str>, - _model: &str, - _optional_params: &Map, - _env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - Ok(complete_url(api_base)) - } - - fn resolve_api_key( - &self, - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - resolve_api_key(api_key, env_lookup) - } -} - -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -pub fn supported_ocr_params() -> &'static [&'static str] { - MISTRAL_OCR_CONFIG.supported_ocr_params() -} - -pub fn map_ocr_params(non_default_params: &Map) -> Map { - MISTRAL_OCR_CONFIG.map_ocr_params(non_default_params) -} - -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -pub fn transform_ocr_request( - model: &str, - document: Value, - optional_params: Map, -) -> Result { - MISTRAL_OCR_CONFIG.transform_ocr_request(model, document, optional_params) -} - -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -pub fn transform_ocr_response(model: &str, response_json: Value) -> Result { - MISTRAL_OCR_CONFIG.transform_ocr_response(model, response_json) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn extract_header_is_a_supported_ocr_param() { - assert!(supported_ocr_params().contains(&"extract_header")); - } - - #[test] - fn extract_footer_is_a_supported_ocr_param() { - assert!(supported_ocr_params().contains(&"extract_footer")); - } - - #[test] - fn existing_ocr_params_remain_supported() { - for param in [ - "pages", - "include_image_base64", - "image_limit", - "image_min_size", - "bbox_annotation_format", - "document_annotation_format", - ] { - assert!(supported_ocr_params().contains(¶m)); - } - } - - #[test] - fn map_ocr_params_forwards_extract_header() { - let params = json!({"extract_header": true}); - assert_eq!( - map_ocr_params(params.as_object().unwrap()), - params.as_object().unwrap().clone() - ); - } - - #[test] - fn map_ocr_params_forwards_extract_footer() { - let params = json!({"extract_footer": true}); - assert_eq!( - map_ocr_params(params.as_object().unwrap()), - params.as_object().unwrap().clone() - ); - } - - #[test] - fn map_ocr_params_forwards_extract_header_and_footer() { - let params = json!({"extract_header": true, "extract_footer": false}); - assert_eq!( - map_ocr_params(params.as_object().unwrap()), - params.as_object().unwrap().clone() - ); - } - - #[test] - fn map_ocr_params_drops_unknown_params() { - let params = json!({"extract_header": true, "unsupported_param": "value"}); - let mapped = map_ocr_params(params.as_object().unwrap()); - assert_eq!(mapped.get("extract_header"), Some(&json!(true))); - assert!(!mapped.contains_key("unsupported_param")); - } - - #[test] - fn new_ocr_params_are_supported() { - for param in [ - "table_format", - "confidence_scores_granularity", - "document_annotation_prompt", - "include_blocks", - "id", - ] { - assert!(supported_ocr_params().contains(¶m)); - } - } - - #[test] - fn map_ocr_params_forwards_new_ocr_params() { - for (param, value) in [ - ("table_format", json!("html")), - ("confidence_scores_granularity", json!("word")), - ( - "document_annotation_prompt", - json!("Extract all invoice line items"), - ), - ("include_blocks", json!(true)), - ("id", json!("req-123")), - ] { - let params = json!({param: value}); - assert_eq!( - map_ocr_params(params.as_object().unwrap()), - params.as_object().unwrap().clone() - ); - } - } - - #[test] - fn transform_ocr_request_includes_each_optional_param() { - let document = json!({ - "type": "document_url", - "document_url": "https://example.com/doc.pdf" - }); - for (param, value) in [ - ("table_format", json!("html")), - ("confidence_scores_granularity", json!("word")), - ( - "document_annotation_prompt", - json!("Extract all invoice line items"), - ), - ("id", json!("req-123")), - ("extract_header", json!(true)), - ("include_blocks", json!(true)), - ("pages", json!([0, 1])), - ] { - let result = transform_ocr_request( - "mistral-ocr-latest", - document.clone(), - json!({param: value}).as_object().unwrap().clone(), - ) - .expect("request should transform"); - assert_eq!(result.data.get(param), Some(&value)); - assert_eq!(result.data.get("model"), Some(&json!("mistral-ocr-latest"))); - assert_eq!(result.data.get("document"), Some(&document)); - assert_eq!(result.files, None); - } - } - - #[test] - fn transform_ocr_request_includes_multiple_new_params() { - let document = json!({ - "type": "document_url", - "document_url": "https://example.com/doc.pdf" - }); - let optional_params = json!({ - "table_format": "html", - "confidence_scores_granularity": "page", - "extract_header": true - }) - .as_object() - .unwrap() - .clone(); - let result = transform_ocr_request("mistral-ocr-latest", document, optional_params) - .expect("request should transform"); - assert_eq!(result.data.get("table_format"), Some(&json!("html"))); - assert_eq!( - result.data.get("confidence_scores_granularity"), - Some(&json!("page")) - ); - assert_eq!(result.data.get("extract_header"), Some(&json!(true))); - } - - #[test] - fn transform_ocr_response_preserves_blocks_and_confidence_scores() { - let blocks = json!([{"type": "title", "content": "Invoice"}]); - let confidence_scores = json!({"page": 0.98}); - let response = json!({ - "pages": [{"index": 0, "markdown": "# Invoice", "blocks": blocks, "confidence_scores": confidence_scores}], - "model": "mistral-ocr-4-0", - "usage_info": {"pages_processed": 1} - }); - let result = - transform_ocr_response("mistral-ocr-4-0", response).expect("response should transform"); - assert_eq!(result.pages[0].get("blocks"), Some(&blocks)); - assert_eq!( - result.pages[0].get("confidence_scores"), - Some(&confidence_scores) - ); - } - - #[test] - fn transform_ocr_response_preserves_ocr4_page_fields() { - let response = json!({ - "pages": [{"index": 0, "markdown": "table page", "tables": [{"rows": 2, "cols": 3}], "hyperlinks": ["https://example.com"], "header": "Acme Corp", "footer": "Page 1"}], - "model": "mistral-ocr-4-0", - "usage_info": {"pages_processed": 1} - }); - let result = transform_ocr_response("mistral-ocr-4-0", response.clone()) - .expect("response should transform"); - assert_eq!(result.pages[0], response["pages"][0]); - } - - #[test] - fn transform_ocr_request_rejects_non_object_document() { - let err = transform_ocr_request("mistral-ocr-latest", json!("bad"), Map::new()) - .expect_err("string document should be rejected"); - - assert_eq!( - err, - Error::InvalidType { - expected: "object", - actual: "string", - } - ); - } - - #[test] - fn transform_ocr_response_normalizes_mistral_json() { - let response = json!({ - "pages": [{"index": 0, "markdown": "hello"}], - "model": "mistral-ocr-2505-completion", - "document_annotation": null, - "usage_info": {"pages_processed": 1} - }); - - let result = transform_ocr_response("mistral-ocr-latest", response) - .expect("response should transform"); - - assert_eq!(result.pages, vec![json!({"index": 0, "markdown": "hello"})]); - assert_eq!(result.model, "mistral-ocr-2505-completion"); - assert_eq!(result.document_annotation, Some(Value::Null)); - assert_eq!(result.usage_info, Some(json!({"pages_processed": 1}))); - assert_eq!(result.object, "ocr"); - } - - #[test] - fn complete_url_defaults_and_dedupes_v1() { - assert_eq!(complete_url(None), "https://api.mistral.ai/v1/ocr"); - assert_eq!(complete_url(Some(" ")), "https://api.mistral.ai/v1/ocr"); - assert_eq!( - complete_url(Some("https://proxy.internal")), - "https://proxy.internal/v1/ocr" - ); - assert_eq!( - complete_url(Some("https://proxy.internal/v1/")), - "https://proxy.internal/v1/ocr" - ); - } - - #[test] - fn resolve_api_key_prefers_param_then_env() { - let no_env = |_: &str| None; - assert_eq!( - resolve_api_key(Some("sk-param"), &no_env).unwrap(), - "sk-param" - ); - - let with_env = |key: &str| (key == MISTRAL_API_KEY_ENV).then(|| "sk-env".to_string()); - assert_eq!(resolve_api_key(None, &with_env).unwrap(), "sk-env"); - // Blank param falls through to the environment. - assert_eq!(resolve_api_key(Some(" "), &with_env).unwrap(), "sk-env"); - } - - #[test] - fn resolve_api_key_errors_when_absent() { - let err = resolve_api_key(None, &|_| None).expect_err("missing key should error"); - assert_eq!(err, Error::Auth(MISSING_KEY_MESSAGE.to_string())); - } -} diff --git a/litellm-rust/crates/core/src/providers/reducto/auth.rs b/litellm-rust/crates/core/src/providers/reducto/auth.rs new file mode 100644 index 00000000000..3409e0ac4fd --- /dev/null +++ b/litellm-rust/crates/core/src/providers/reducto/auth.rs @@ -0,0 +1,92 @@ +use crate::auth::error::MissingCredential; +use crate::auth::{ + CredentialPlacement, CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, + ProviderAuthPolicy, ResolvedCredential, SecretValue, +}; +use crate::error::AuthError; + +pub const REDUCTO_API_KEY_ENV: &str = "REDUCTO_API_KEY"; + +const AUTH_RULES: &[CredentialRule] = &[CredentialRule { + kind: CredentialPlanKind::Static, + placement: CredentialPlacement::Bearer, +}]; +const AUTH_POLICY: ProviderAuthPolicy = ProviderAuthPolicy { + rules: AUTH_RULES, + accepted_existing_headers: &["Authorization"], + existing_header_behavior: ExistingHeaderBehavior::Preserve, + scope: None, + audience: None, +}; + +pub fn resolve_api_key( + api_key: Option<&str>, + env_lookup: &dyn Fn(&str) -> Option, +) -> Result { + api_key + .map(str::trim) + .filter(|key| !key.is_empty()) + .map(str::to_string) + .or_else(|| { + env_lookup(REDUCTO_API_KEY_ENV) + .map(|key| key.trim().to_string()) + .filter(|key| !key.is_empty()) + }) + .ok_or(AuthError::MissingCredential( + MissingCredential::ReductoApiKey, + )) +} + +pub fn validate_environment( + headers: Vec<(String, String)>, + api_key: Option<&str>, + env_lookup: &dyn Fn(&str) -> Option, +) -> Result, AuthError> { + if AUTH_POLICY.has_existing_credential(&headers) { + return Ok(headers); + } + let credential = + ResolvedCredential::Static(SecretValue::new(resolve_api_key(api_key, env_lookup)?)); + AUTH_POLICY.apply(headers, CredentialPlanKind::Static, &credential) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn explicit_key_precedes_environment_key() { + let key = resolve_api_key(Some("passed-key"), &|_| Some("env-key".to_string())) + .expect("explicit key resolves"); + + assert_eq!(key, "passed-key"); + } + + #[test] + fn blank_explicit_key_uses_environment_key() { + let env_lookup = + |name: &str| (name == REDUCTO_API_KEY_ENV).then(|| " env-key ".to_string()); + + assert_eq!(resolve_api_key(Some(" "), &env_lookup).unwrap(), "env-key"); + } + + #[test] + fn static_key_is_applied_as_bearer() { + let headers = validate_environment(Vec::new(), Some("passed-key"), &|_| None).unwrap(); + + assert_eq!( + headers, + vec![("Authorization".to_string(), "Bearer passed-key".to_string())] + ); + } + + #[test] + fn existing_authorization_is_preserved_without_key_lookup() { + let headers = vec![("authorization".to_string(), "Bearer existing".to_string())]; + + assert_eq!( + validate_environment(headers.clone(), None, &|_| None).unwrap(), + headers + ); + } +} diff --git a/litellm-rust/crates/core/src/providers/reducto/mod.rs b/litellm-rust/crates/core/src/providers/reducto/mod.rs index 3621ff6a2fd..0e4a05d5978 100644 --- a/litellm-rust/crates/core/src/providers/reducto/mod.rs +++ b/litellm-rust/crates/core/src/providers/reducto/mod.rs @@ -1 +1 @@ -pub mod ocr; +pub mod auth; diff --git a/litellm-rust/crates/core/src/providers/reducto/ocr/mod.rs b/litellm-rust/crates/core/src/providers/reducto/ocr/mod.rs deleted file mode 100644 index 8acee8f770c..00000000000 --- a/litellm-rust/crates/core/src/providers/reducto/ocr/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod transformation; - -#[cfg(test)] -mod tests; diff --git a/litellm-rust/crates/core/src/providers/reducto/ocr/tests.rs b/litellm-rust/crates/core/src/providers/reducto/ocr/tests.rs deleted file mode 100644 index 2b66d058b5d..00000000000 --- a/litellm-rust/crates/core/src/providers/reducto/ocr/tests.rs +++ /dev/null @@ -1,202 +0,0 @@ -use rstest::{fixture, rstest}; -use serde_json::{Value, json}; - -use super::transformation::*; -use crate::ocr::transformation::OcrProviderConfig; - -#[fixture] -fn parse_response() -> Value { - json!({ - "job_id": "job_123", - "usage": {"num_pages": 3, "credits": 3}, - "result": { - "chunks": [ - { - "content": "Page 1 block A", - "blocks": [{ - "content": "Page 1 block A", - "bbox": {"page": 1}, - "kind": "text", - }], - }, - { - "content": "Page 2 block A", - "blocks": [{ - "content": "Page 2 block A", - "bbox": {"page": 2}, - "kind": "table", - }], - }, - { - "content": "Page 1 block B", - "blocks": [{ - "content": "Page 1 block B", - "bbox": {"page": 1}, - "kind": "text", - }], - }, - { - "content": "Page 3 block A", - "blocks": [{ - "content": "Page 3 block A", - "bbox": {"page": 3}, - "kind": "figure", - }], - }, - ], - }, - }) -} - -#[rstest] -fn test_parse_v3_file_upload_and_response_mapping(parse_response: Value) { - let source = classify_document_source("data:application/pdf;base64,JVBERi0xLjQ=") - .expect("PDF data URI should be valid"); - let upload = build_upload_request( - source, - "Bearer test-key", - Some("https://platform.reducto.ai"), - ) - .expect("data URI should require upload"); - assert_eq!(upload.url, "https://platform.reducto.ai/upload"); - assert_eq!(upload.authorization, "Bearer test-key"); - assert_eq!(upload.file_name, "document"); - assert_eq!(upload.mime_type, "application/pdf"); - assert_eq!(upload.bytes, b"%PDF-1.4"); - - let optional_params = json!({ - "formatting": {"table_output_format": "html"}, - "retrieval": {"chunk_mode": "section"}, - "settings": {"ocr_system": "standard"}, - }) - .as_object() - .expect("params should be an object") - .clone(); - let request = build_parse_v3_request("reducto://uploaded.pdf", optional_params); - assert_eq!( - request.data, - json!({ - "input": "reducto://uploaded.pdf", - "formatting": {"table_output_format": "html"}, - "retrieval": {"chunk_mode": "section"}, - "settings": {"ocr_system": "standard"}, - }) - ); - - let transformed = transform_reducto_response("parse-v3", parse_response.clone()) - .expect("response should transform"); - assert_eq!( - transformed.usage_info, - Some(json!({"pages_processed": 3, "credits": 3})) - ); - assert_eq!(transformed.pages.len(), 3); - assert_eq!( - transformed.pages[0], - json!({ - "index": 0, - "markdown": "Page 1 block A\n\nPage 1 block B", - "blocks": [ - {"content": "Page 1 block A", "bbox": {"page": 1}, "kind": "text"}, - {"content": "Page 1 block B", "bbox": {"page": 1}, "kind": "text"}, - ], - }) - ); - assert_eq!(transformed.pages[1]["markdown"], "Page 2 block A"); - assert_eq!(transformed.pages[2]["markdown"], "Page 3 block A"); - assert_eq!(transformed.provider_native_response, Some(parse_response)); -} - -#[rstest] -fn test_parse_v3_reducto_id_passthrough_skips_upload(parse_response: Value) { - let document = json!({ - "type": "document_url", - "document_url": "reducto://already-uploaded.pdf", - }); - let source = extract_document_source(&document).expect("Reducto ID should be valid"); - assert!(build_upload_request(source.clone(), "Bearer test-key", None).is_none()); - assert_eq!( - source, - ReductoDocumentSource::FileId("reducto://already-uploaded.pdf".to_string()) - ); - - let request = REDUCTO_PARSE_V3_CONFIG - .transform_ocr_request( - "parse-v3", - document, - json!({"retrieval": {"chunk_mode": "section"}}) - .as_object() - .expect("params should be object") - .clone(), - ) - .expect("direct ID should transform"); - assert_eq!(request.data["input"], "reducto://already-uploaded.pdf"); - assert_eq!(request.data["retrieval"]["chunk_mode"], "section"); - - let response = REDUCTO_PARSE_V3_CONFIG - .transform_ocr_response("parse-v3", parse_response) - .expect("response should transform"); - assert!( - response.pages[0]["markdown"] - .as_str() - .expect("markdown should be string") - .starts_with("Page 1 block A") - ); -} - -#[rstest] -fn test_parse_legacy_wraps_enhance_under_options() { - let request = build_parse_legacy_request( - "reducto://legacy.pdf", - json!({"enhance": {"agentic": [{"type": "table"}]}}) - .as_object() - .expect("params should be object"), - ); - assert_eq!( - request.data, - json!({ - "document_url": "reducto://legacy.pdf", - "options": {"enhance": {"agentic": [{"type": "table"}]}}, - }) - ); -} - -#[rstest] -fn test_parse_v3_image_data_uri_upload_uses_image_mime() { - let source = classify_document_source("data:image/png;base64,iVBORw0KGgo=") - .expect("PNG data URI should be valid"); - let upload = build_upload_request( - source, - "Bearer programmatic-key", - Some("https://custom.reducto.test/"), - ) - .expect("data URI should require upload"); - assert_eq!(upload.url, "https://custom.reducto.test/upload"); - assert_eq!(upload.authorization, "Bearer programmatic-key"); - assert_eq!(upload.mime_type, "image/png"); - assert_eq!(upload.bytes, b"\x89PNG\r\n\x1a\n"); -} - -#[rstest] -#[case::http("http://example.com/document.pdf")] -#[case::https("https://example.com/document.pdf")] -fn test_parse_v3_rejects_plain_http_urls(#[case] source: &str) { - let error = classify_document_source(source).expect_err("plain URL should be rejected"); - assert!(error.to_string().contains("upload the file first")); -} - -#[rstest] -fn test_parse_v3_uses_programmatic_api_key_over_env() { - let key = resolve_api_key(Some("passed-key"), &|_| Some("env-reducto-key".to_string())) - .expect("explicit key should resolve"); - assert_eq!(key, "passed-key"); - - let headers = REDUCTO_PARSE_V3_CONFIG - .validate_environment(Vec::new(), Some("passed-key"), &|_| { - Some("env-reducto-key".to_string()) - }) - .expect("headers should validate"); - assert_eq!( - headers, - vec![("Authorization".to_string(), "Bearer passed-key".to_string())] - ); -} diff --git a/litellm-rust/crates/core/src/providers/reducto/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/reducto/ocr/transformation.rs deleted file mode 100644 index b8507541e18..00000000000 --- a/litellm-rust/crates/core/src/providers/reducto/ocr/transformation.rs +++ /dev/null @@ -1,407 +0,0 @@ -use std::collections::BTreeMap; - -use base64::Engine; -use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; -use serde_json::{Map, Value, json}; - -use crate::error::{Error, json_type_name}; -use crate::ocr::transformation::OcrProviderConfig; -use crate::ocr::types::{OcrRequestData, OcrResponseData}; - -pub const REDUCTO_API_BASE: &str = "https://platform.reducto.ai"; -pub const REDUCTO_API_KEY_ENV: &str = "REDUCTO_API_KEY"; -pub const REDUCTO_ID_PREFIX: &str = "reducto://"; - -const PARSE_V3_SUPPORTED_OCR_PARAMS: &[&str] = &["formatting", "retrieval", "settings"]; -const PARSE_LEGACY_SUPPORTED_OCR_PARAMS: &[&str] = &["enhance"]; -const MISSING_KEY_MESSAGE: &str = "Missing REDUCTO_API_KEY - set it in the environment or pass api_key to litellm.ocr()/litellm.aocr()"; -const DATA_URI_UPLOAD_REQUIRED: &str = - "Reducto data URI upload must complete before OCR request transformation"; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum ReductoDocumentSource { - FileId(String), - Upload { bytes: Vec, mime_type: String }, -} - -#[derive(Clone, PartialEq, Eq)] -pub struct ReductoUploadRequest { - pub url: String, - pub authorization: String, - pub file_name: &'static str, - pub bytes: Vec, - pub mime_type: String, -} - -pub struct ReductoParseV3Config; -pub struct ReductoParseLegacyConfig; - -pub const REDUCTO_PARSE_V3_CONFIG: ReductoParseV3Config = ReductoParseV3Config; -pub const REDUCTO_PARSE_LEGACY_CONFIG: ReductoParseLegacyConfig = ReductoParseLegacyConfig; - -pub fn config_for_model(model: &str) -> Option<&'static dyn OcrProviderConfig> { - match model { - "parse-v3" => Some(&REDUCTO_PARSE_V3_CONFIG), - "parse-legacy" => Some(&REDUCTO_PARSE_LEGACY_CONFIG), - _ => None, - } -} - -pub fn normalize_api_base(api_base: Option<&str>) -> String { - api_base - .map(str::trim) - .filter(|base| !base.is_empty()) - .unwrap_or(REDUCTO_API_BASE) - .trim_end_matches('/') - .to_string() -} - -pub fn parse_url(api_base: Option<&str>) -> String { - format!("{}/parse", normalize_api_base(api_base)) -} - -pub fn upload_url(api_base: Option<&str>) -> String { - format!("{}/upload", normalize_api_base(api_base)) -} - -pub fn resolve_api_key( - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result { - api_key - .map(str::trim) - .filter(|key| !key.is_empty()) - .map(str::to_string) - .or_else(|| { - env_lookup(REDUCTO_API_KEY_ENV) - .map(|key| key.trim().to_string()) - .filter(|key| !key.is_empty()) - }) - .ok_or_else(|| Error::Auth(MISSING_KEY_MESSAGE.to_string())) -} - -pub fn extract_document_source(document: &Value) -> Result { - let document = document.as_object().ok_or_else(|| Error::InvalidType { - expected: "object", - actual: json_type_name(document), - })?; - let source = document - .get("document_url") - .and_then(Value::as_str) - .filter(|source| !source.is_empty()) - .or_else(|| document.get("image_url").and_then(Value::as_str)) - .ok_or_else(|| { - Error::InvalidRequest( - "Reducto expected OCR preprocessing to produce document_url or image_url" - .to_string(), - ) - })?; - classify_document_source(source) -} - -pub fn classify_document_source(source: &str) -> Result { - if source.starts_with(REDUCTO_ID_PREFIX) { - return Ok(ReductoDocumentSource::FileId(source.to_string())); - } - if source.starts_with("http://") || source.starts_with("https://") { - return Err(Error::InvalidRequest( - "Reducto requires type='file' (auto-uploaded) or a reducto:// id. Plain http(s) URLs are not supported; upload the file first." - .to_string(), - )); - } - if !source.starts_with("data:") { - return Err(Error::InvalidRequest( - "Reducto requires a reducto:// id or a base64 data URI after OCR preprocessing." - .to_string(), - )); - } - - let (header, encoded) = source - .split_once(',') - .ok_or_else(|| Error::InvalidRequest("Invalid Reducto data URI provided.".to_string()))?; - if !header.split(';').any(|part| part == "base64") { - return Err(Error::InvalidRequest( - "Reducto only supports base64-encoded data URIs.".to_string(), - )); - } - - let mime_type = header - .strip_prefix("data:") - .and_then(|header| header.split(';').next()) - .filter(|mime| !mime.is_empty()) - .unwrap_or("application/octet-stream") - .to_string(); - let bytes = BASE64_STANDARD.decode(encoded).map_err(|_| { - Error::InvalidRequest("Invalid Reducto base64 payload provided.".to_string()) - })?; - - Ok(ReductoDocumentSource::Upload { bytes, mime_type }) -} - -pub fn build_upload_request( - source: ReductoDocumentSource, - authorization: &str, - api_base: Option<&str>, -) -> Option { - let ReductoDocumentSource::Upload { bytes, mime_type } = source else { - return None; - }; - - Some(ReductoUploadRequest { - url: upload_url(api_base), - authorization: authorization.to_string(), - file_name: "document", - bytes, - mime_type, - }) -} - -pub fn extract_upload_file_id(response_json: &Value) -> Result<&str, Error> { - response_json - .as_object() - .and_then(|response| response.get("file_id")) - .and_then(Value::as_str) - .filter(|file_id| !file_id.is_empty()) - .ok_or_else(|| { - Error::InvalidResponse(format!( - "Reducto /upload returned 200 without a file_id; got payload={response_json}" - )) - }) -} - -pub fn build_parse_v3_request( - file_id: &str, - optional_params: Map, -) -> OcrRequestData { - let data = std::iter::once(("input".to_string(), Value::String(file_id.to_string()))) - .chain(optional_params) - .collect(); - OcrRequestData { - data: Value::Object(data), - files: None, - } -} - -pub fn build_parse_legacy_request( - file_id: &str, - optional_params: &Map, -) -> OcrRequestData { - let options = optional_params - .get("enhance") - .filter(|enhance| !enhance.is_null()) - .map(|enhance| json!({"options": {"enhance": enhance}})); - let data = match options { - Some(Value::Object(options)) => std::iter::once(( - "document_url".to_string(), - Value::String(file_id.to_string()), - )) - .chain(options) - .collect(), - _ => Map::from_iter([( - "document_url".to_string(), - Value::String(file_id.to_string()), - )]), - }; - OcrRequestData { - data: Value::Object(data), - files: None, - } -} - -fn source_file_id(document: &Value) -> Result { - match extract_document_source(document)? { - ReductoDocumentSource::FileId(file_id) => Ok(file_id), - ReductoDocumentSource::Upload { .. } => Err(Error::Unsupported(DATA_URI_UPLOAD_REQUIRED)), - } -} - -fn page_number(block: &Map) -> Option { - let page = block.get("bbox")?.as_object()?.get("page")?; - page.as_i64() - .or_else(|| page.as_u64().and_then(|page| i64::try_from(page).ok())) - .or_else(|| page.as_str().and_then(|page| page.parse().ok())) -} - -fn chunks(result: &Map) -> &[Value] { - result - .get("chunks") - .and_then(Value::as_array) - .map(Vec::as_slice) - .unwrap_or_default() -} - -fn build_pages(result: &Map) -> Vec { - let blocks_by_page = chunks(result) - .iter() - .filter_map(Value::as_object) - .filter_map(|chunk| chunk.get("blocks").and_then(Value::as_array)) - .flatten() - .filter_map(|block| block.as_object().map(|object| (block, object))) - .filter_map(|(block, object)| page_number(object).map(|page| (page, block.clone()))) - .fold( - BTreeMap::>::new(), - |mut pages, (page, block)| { - pages.entry(page).or_default().push(block); - pages - }, - ); - - if blocks_by_page.is_empty() { - let markdown = chunks(result) - .iter() - .filter_map(Value::as_object) - .filter_map(|chunk| chunk.get("content").and_then(Value::as_str)) - .filter(|content| !content.is_empty()) - .collect::>() - .join("\n\n"); - return if markdown.is_empty() { - Vec::new() - } else { - vec![json!({"index": 0, "markdown": markdown})] - }; - } - - blocks_by_page - .into_iter() - .map(|(page, blocks)| { - let markdown = blocks - .iter() - .filter_map(Value::as_object) - .filter_map(|block| block.get("content").and_then(Value::as_str)) - .filter(|content| !content.is_empty()) - .collect::>() - .join("\n\n"); - json!({ - "index": page.saturating_sub(1).max(0), - "markdown": markdown, - "blocks": blocks, - }) - }) - .collect() -} - -pub fn transform_reducto_response( - model: &str, - response_json: Value, -) -> Result { - let response = response_json - .as_object() - .ok_or_else(|| Error::InvalidType { - expected: "object", - actual: json_type_name(&response_json), - })?; - let empty_result = Map::new(); - let result = match response.get("result") { - Some(Value::Object(result)) => result, - Some(Value::Null) => &empty_result, - Some(_) => { - return Err(Error::InvalidResponse( - "Reducto result must be an object".to_string(), - )); - } - None => response, - }; - let usage = response - .get("usage") - .and_then(Value::as_object) - .cloned() - .unwrap_or_default(); - let usage_info = Some(json!({ - "pages_processed": usage.get("num_pages").cloned().unwrap_or(Value::Null), - "credits": usage.get("credits").cloned().unwrap_or(Value::Null), - })); - - Ok(OcrResponseData { - pages: build_pages(result), - model: model.to_string(), - document_annotation: None, - usage_info, - object: "ocr".to_string(), - extra_fields: Map::new(), - provider_native_response: Some(response_json), - }) -} - -impl OcrProviderConfig for ReductoParseV3Config { - fn supported_ocr_params(&self) -> &'static [&'static str] { - PARSE_V3_SUPPORTED_OCR_PARAMS - } - - fn transform_ocr_request( - &self, - _model: &str, - document: Value, - optional_params: Map, - ) -> Result { - let file_id = source_file_id(&document)?; - Ok(build_parse_v3_request(&file_id, optional_params)) - } - - fn transform_ocr_response( - &self, - model: &str, - response_json: Value, - ) -> Result { - transform_reducto_response(model, response_json) - } - - fn complete_url( - &self, - api_base: Option<&str>, - _model: &str, - _optional_params: &Map, - _env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - Ok(parse_url(api_base)) - } - - fn resolve_api_key( - &self, - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - resolve_api_key(api_key, env_lookup) - } -} - -impl OcrProviderConfig for ReductoParseLegacyConfig { - fn supported_ocr_params(&self) -> &'static [&'static str] { - PARSE_LEGACY_SUPPORTED_OCR_PARAMS - } - - fn transform_ocr_request( - &self, - _model: &str, - document: Value, - optional_params: Map, - ) -> Result { - let file_id = source_file_id(&document)?; - Ok(build_parse_legacy_request(&file_id, &optional_params)) - } - - fn transform_ocr_response( - &self, - model: &str, - response_json: Value, - ) -> Result { - transform_reducto_response(model, response_json) - } - - fn complete_url( - &self, - api_base: Option<&str>, - _model: &str, - _optional_params: &Map, - _env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - Ok(parse_url(api_base)) - } - - fn resolve_api_key( - &self, - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - resolve_api_key(api_key, env_lookup) - } -} diff --git a/litellm-rust/crates/core/src/providers/vertex_ai/mod.rs b/litellm-rust/crates/core/src/providers/vertex_ai/mod.rs index 3621ff6a2fd..8b137891791 100644 --- a/litellm-rust/crates/core/src/providers/vertex_ai/mod.rs +++ b/litellm-rust/crates/core/src/providers/vertex_ai/mod.rs @@ -1 +1 @@ -pub mod ocr; + diff --git a/litellm-rust/crates/core/src/url_utils.rs b/litellm-rust/crates/core/src/url_utils.rs index f66351ef724..3d5c2dfad97 100644 --- a/litellm-rust/crates/core/src/url_utils.rs +++ b/litellm-rust/crates/core/src/url_utils.rs @@ -1,8 +1,3 @@ -#![allow( - dead_code, - reason = "used by the OCR architecture in the next stacked PR" -)] - use std::marker::PhantomData; use thiserror::Error; @@ -14,8 +9,6 @@ pub(crate) enum ApiUrlError { Parse(#[from] url::ParseError), #[error("URL cannot be used as a base")] CannotBeBase, - #[error("unsupported URL scheme: {0}")] - Scheme(String), } pub(crate) struct Base; @@ -34,47 +27,6 @@ impl ApiUrl { }) } - pub(crate) fn parse_with_default_scheme( - value: &str, - default_scheme: &str, - ) -> Result { - let value = value.trim(); - if value.contains("://") { - return Self::parse(value); - } - Self::parse(&format!("{default_scheme}://{value}")) - } - - pub(crate) fn with_scheme(mut self, scheme: &str) -> Result { - self.url - .set_scheme(scheme) - .map_err(|()| ApiUrlError::Scheme(scheme.to_string()))?; - Ok(self) - } - - pub(crate) fn scheme(&self) -> &str { - self.url.scheme() - } - - pub(crate) fn truncate_path_after(mut self, segment: &str) -> Result { - let segments: Vec = self - .url - .path_segments() - .ok_or(ApiUrlError::CannotBeBase)? - .filter(|value| !value.is_empty()) - .map(str::to_string) - .collect(); - let Some(index) = segments.iter().position(|value| value == segment) else { - return Ok(self); - }; - self.url - .path_segments_mut() - .map_err(|()| ApiUrlError::CannotBeBase)? - .clear() - .extend(segments[..=index].iter().map(String::as_str)); - Ok(self) - } - pub(crate) fn complete_path(self, target: &[&str]) -> Result, ApiUrlError> { self.complete_path_with_aliases(target, &[]) } @@ -125,15 +77,6 @@ impl ApiUrl { } impl ApiUrl { - pub(crate) fn has_query_key(&self, key: &str) -> bool { - self.url.query_pairs().any(|(name, _)| name == key) - } - - pub(crate) fn append_query_pair(mut self, key: &str, value: &str) -> Self { - self.url.query_pairs_mut().append_pair(key, value); - self - } - pub(crate) fn append_query_pairs<'a>( mut self, pairs: impl IntoIterator, @@ -165,19 +108,4 @@ mod tests { assert_eq!(actual, expected); } } - - #[test] - fn completion_places_paths_before_queries_and_encodes_query_values() { - let actual = ApiUrl::parse("https://example.test/v1?tenant=a") - .and_then(|url| url.complete_path(&["v1", "ocr"])) - .map(|url| { - url.append_query_pair("model", "name with spaces") - .into_string() - }) - .expect("url builds"); - assert_eq!( - actual, - "https://example.test/v1/ocr?tenant=a&model=name+with+spaces" - ); - } } diff --git a/litellm-rust/crates/core/tests/azure_ai_ocr.rs b/litellm-rust/crates/core/tests/azure_ai_ocr.rs new file mode 100644 index 00000000000..ab6b02c41a2 --- /dev/null +++ b/litellm-rust/crates/core/tests/azure_ai_ocr.rs @@ -0,0 +1,39 @@ +use crate::ocr::integrations::OcrIntegration; +use crate::ocr::integrations::{AzureMistral as AZURE_MISTRAL, MistralDirect as MISTRAL}; +use crate::ocr::test_support::body; +use crate::ocr::types::OcrConnection; +use serde_json::json; + +#[tokio::test] +async fn azure_ai_reuses_mistral_body_transform() { + let document = json!({"type":"document_url","document_url":"data:application/pdf;base64,YWJj"}); + let params = json!({"include_image_base64":true}); + assert_eq!( + body(&AZURE_MISTRAL, "model", document.clone(), params.clone()) + .await + .unwrap(), + body(&MISTRAL, "model", document, params).await.unwrap() + ); +} +#[tokio::test] +async fn azure_ai_mistral_ocr_uses_generic_api_base() { + let connection = OcrConnection { + api_key: Some("key".into()), + api_base: Some("https://example.com/".into()), + ..Default::default() + }; + let prepared = AZURE_MISTRAL + .prepare( + &connection, + &Default::default(), + "model", + &Default::default(), + &|_| None, + ) + .await + .unwrap(); + assert_eq!( + prepared.url, + "https://example.com/providers/mistral/azure/ocr" + ); +} diff --git a/litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs b/litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs new file mode 100644 index 00000000000..e42a51542ed --- /dev/null +++ b/litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs @@ -0,0 +1,415 @@ +use crate::ocr::formats::OcrFormat; +use crate::ocr::integrations::AzureDocumentIntelligence as AZURE_DOCUMENT_INTELLIGENCE; +use crate::ocr::integrations::OcrIntegration; +use crate::ocr::prepare::OcrIntegrationKind; +use crate::ocr::registry::decode_integration_request; +use crate::ocr::test_support::perform_ocr; +use crate::ocr::test_support::{MockResponse, body, mock_server, params, transform, wire_request}; +use crate::ocr::types::OcrConnection; +use rstest::{fixture, rstest}; +use serde_json::{Value, json}; + +fn connection() -> OcrConnection { + OcrConnection { + api_base: Some("https://example.com".into()), + api_key: Some("key".into()), + ..Default::default() + } +} +#[fixture] +fn operation() -> Value { + json!({"status":"succeeded","operationExtension":42,"analyzeResult":{ + "content":"A\n\nB","tables":[{"cells":[]}],"keyValuePairs":[{"key":{"content":"A"}}], + "pages":[{"pageNumber":1,"width":8.5,"height":11,"unit":"inch","lines":[{"content":"A"},{"content":null},{"content":"B"}],"words":[{"content":"A","confidence":0.99}]}]}}) +} + +async fn prepared_url( + model: &str, + params: &::MappedParams, +) -> Result { + AZURE_DOCUMENT_INTELLIGENCE + .prepare(&connection(), &Default::default(), model, params, &|_| None) + .await + .map(|prepared| prepared.url) +} + +fn query_value(url: &str, key: &str) -> Option { + url::Url::parse(url) + .expect("prepared URL parses") + .query_pairs() + .find_map(|(name, value)| (name == key).then(|| value.into_owned())) +} + +#[rstest] +#[case(json!([2,0,0,1]), "1,2,3")] +#[case(json!("1-3, 5"), "1-3,5")] +#[case(json!(["1","3-5"]), "1,3-5")] +#[case(json!("0,3-1"), "0,3-1")] +#[tokio::test] +async fn document_intelligence_url_normalizes_zero_based_pages( + #[case] pages: Value, + #[case] expected: &str, +) { + let mapped = params(&AZURE_DOCUMENT_INTELLIGENCE, json!({"pages":pages})); + let url = prepared_url("prebuilt-read", &mapped).await.unwrap(); + assert_eq!(query_value(&url, "pages").as_deref(), Some(expected)); +} +#[rstest] +#[case(json!([true]))] +#[case(json!([1,"2"]))] +#[case(json!([18446744073709551615u64]))] +fn invalid_page_types_return_typed_errors(#[case] pages: Value) { + assert!( + decode_integration_request( + OcrIntegrationKind::AzureDocumentIntelligence, + json!({"pages":pages}).as_object().unwrap().clone() + ) + .is_err() + ); +} +#[rstest] +#[case(json!([-1]))] +#[case(json!([i64::MAX]))] +#[case(json!("1&&features=bad"))] +#[case(json!(""))] +#[case(json!(["1-2-3"]))] +fn invalid_page_values_never_panic(#[case] pages: Value) { + let input = serde_json::from_value(json!({"pages":pages})).unwrap(); + assert!( + crate::ocr::formats::document_intelligence::AzureDocumentIntelligenceOcrFormat::map_params( + input + ) + .is_err() + ); +} +#[tokio::test] +async fn document_intelligence_page_mapping_omits_empty_list() { + let mapped = params( + &AZURE_DOCUMENT_INTELLIGENCE, + json!({"pages":[],"features":[]}), + ); + assert!( + !prepared_url("prebuilt-read", &mapped) + .await + .unwrap() + .contains("&pages") + ); +} +#[rstest] +#[case(json!(["keyValuePairs","languages"]), "keyValuePairs,languages")] +#[case(json!("keyValuePairs, languages"), "keyValuePairs,languages")] +#[tokio::test] +async fn document_intelligence_maps_features(#[case] features: Value, #[case] expected: &str) { + let mapped = params(&AZURE_DOCUMENT_INTELLIGENCE, json!({"features":features})); + let url = prepared_url("prebuilt-read", &mapped).await.unwrap(); + assert_eq!(query_value(&url, "features").as_deref(), Some(expected)); +} +#[rstest] +#[case(json!(["languages,ocr"]))] +#[case(json!("languages&pages=1"))] +#[case(json!(""))] +fn document_intelligence_rejects_invalid_features(#[case] features: Value) { + assert!( + crate::ocr::formats::document_intelligence::AzureDocumentIntelligenceOcrFormat::map_params( + serde_json::from_value(json!({"features":features})).unwrap() + ) + .is_err() + ); +} +#[tokio::test] +async fn document_intelligence_mistral_pages_flow_to_query_only() { + let result = body( + &AZURE_DOCUMENT_INTELLIGENCE, + "model", + json!({"type":"document_url","document_url":"https://example.com/doc.pdf"}), + json!({"pages":[0,1],"features":"languages"}), + ) + .await + .unwrap(); + assert_eq!(result, json!({"urlSource":"https://example.com/doc.pdf"})); +} +#[rstest] +#[case("data:application/pdf;base64,YWJj", "YWJj")] +#[case("data:application/pdf,abc", "YWJj")] +#[case("data:application/pdf;base64,", "")] +#[case("data:application/pdf;version=1.7;base64,YWJj", "YWJj")] +#[case("DATA:;BASE64,YQ%3D%3D#page", "YQ==")] +#[tokio::test] +async fn document_intelligence_request_uses_base64_source_for_data_uri( + #[case] source: &str, + #[case] expected: &str, +) { + assert_eq!( + body( + &AZURE_DOCUMENT_INTELLIGENCE, + "model", + json!({"type":"image_url","image_url":source}), + json!({}) + ) + .await + .unwrap(), + json!({"base64Source":expected}) + ); +} +#[tokio::test] +async fn azure_document_intelligence_model_id_is_encoded() { + let mapped = params(&AZURE_DOCUMENT_INTELLIGENCE, json!({})); + assert!( + prepared_url("azure_ai/doc-intelligence/a ?#é", &mapped) + .await + .unwrap() + .contains("a%20%3F%23%C3%A9:analyze") + ); +} +#[tokio::test] +async fn azure_document_intelligence_dot_segment_model_id_is_rejected() { + let mapped = params(&AZURE_DOCUMENT_INTELLIGENCE, json!({})); + for model in [".", "..", "azure_ai/doc-intelligence/.."] { + assert!(prepared_url(model, &mapped).await.is_err()); + } +} +#[rstest] +fn document_intelligence_response_normalizes_pages(operation: Value) { + let result = transform( + &AZURE_DOCUMENT_INTELLIGENCE, + "model", + operation.clone(), + json!({}), + ) + .unwrap(); + assert_eq!(result["pages"][0]["markdown"], "A\n\nB"); + assert_eq!( + result["pages"][0]["dimensions"], + json!({"width":816,"height":1056,"dpi":96}) + ); + assert_eq!(result["usage_info"]["pages_processed"], 1); + assert_eq!(result["tables"], operation["analyzeResult"]["tables"]); +} +#[test] +fn document_intelligence_response_tolerates_missing_native_fields() { + for value in [ + json!({"status":"succeeded"}), + json!({"status":"succeeded","analyzeResult":null}), + ] { + let result = transform(&AZURE_DOCUMENT_INTELLIGENCE, "model", value, json!({})).unwrap(); + assert_eq!(result["pages"], json!([])); + assert_eq!(result["tables"], Value::Null); + } + let result = transform( + &AZURE_DOCUMENT_INTELLIGENCE, + "model", + json!({"status":"succeeded","analyzeResult":{"pages":[{}]}}), + json!({}), + ) + .unwrap(); + assert_eq!(result["pages"][0]["dimensions"]["width"], 816); +} +#[rstest] +#[case(json!({"pages":null}), "pages")] +#[case(json!({"pages":[null]}), "pages[0]")] +#[case(json!({"pages":[{"lines":null}]}), "lines")] +#[case(json!({"pages":[{"lines":[{"content":42}]}]}), "content")] +#[case(json!({"pages":[{"width":"bad"}]}), "width")] +fn malformed_pages_are_rejected_with_paths(#[case] analysis: Value, #[case] path: &str) { + let error = transform( + &AZURE_DOCUMENT_INTELLIGENCE, + "model", + json!({"status":"succeeded","analyzeResult":analysis}), + json!({}), + ) + .unwrap_err(); + assert!(error.to_string().contains(path), "{error}"); +} +#[test] +fn page_coercions_and_overflow_are_explicit() { + let result = transform(&AZURE_DOCUMENT_INTELLIGENCE,"model",json!({"status":"succeeded","analyzeResult":{"pages":[{"pageNumber":"2","width":"8.5","height":11.0}]}}),json!({})).unwrap(); + assert_eq!(result["pages"][0]["index"], 1); + for page in [json!({"pageNumber":i64::MIN}), json!({"width":1e100})] { + let error = transform( + &AZURE_DOCUMENT_INTELLIGENCE, + "model", + json!({"status":"succeeded","analyzeResult":{"pages":[page]}}), + json!({}), + ) + .expect_err("provider page overflows"); + let error = crate::Error::from(error); + assert_eq!(error.kind(), crate::error::ErrorKind::InvalidResponse); + assert!(error.to_string().contains("numeric value")); + } +} +#[test] +fn document_intelligence_non_succeeded_status_is_rejected() { + for status in [ + json!("failed"), + json!("running"), + json!("unknown"), + Value::Null, + ] { + assert!( + transform( + &AZURE_DOCUMENT_INTELLIGENCE, + "model", + json!({"status":status}), + json!({}) + ) + .is_err() + ); + } +} +#[rstest] +fn document_intelligence_native_format_carries_raw_operation(operation: Value) { + let response = transform( + &AZURE_DOCUMENT_INTELLIGENCE, + "model", + operation.clone(), + json!({"req_format":"native"}), + ) + .unwrap(); + assert_eq!(response["provider_native_response"], operation); + assert!( + transform( + &AZURE_DOCUMENT_INTELLIGENCE, + "model", + operation.clone(), + json!({"req_format":"litellm"}) + ) + .unwrap() + .get("provider_native_response") + .is_none() + ); +} +#[tokio::test] +async fn document_intelligence_url_omits_req_format() { + let mapped = params(&AZURE_DOCUMENT_INTELLIGENCE, json!({"req_format":"native"})); + assert!( + !prepared_url("model", &mapped) + .await + .unwrap() + .contains("req_format") + ); +} +#[test] +fn document_intelligence_rejects_unknown_req_format() { + assert!( + decode_integration_request( + OcrIntegrationKind::AzureDocumentIntelligence, + json!({"req_format":"azure"}).as_object().unwrap().clone() + ) + .is_err() + ); +} +#[rstest] +#[case(false)] +#[case(true)] +#[tokio::test] +async fn polling_forwards_subscription_or_bearer_and_preserves_native( + operation: Value, + #[case] bearer: bool, +) { + let (base, requests, server) = mock_server(vec![ + MockResponse { + status: 202, + headers: vec![("Operation-Location", "{base}/operation".into())], + body: json!({}), + }, + MockResponse { + status: 200, + headers: vec![("Retry-After", "0".into())], + body: json!({"status":"running"}), + }, + MockResponse::json(operation.clone()), + ]) + .await; + let mut request = wire_request( + "azure_ai/doc-intelligence/prebuilt-read", + &base, + json!({"req_format":"native","pages":[0,2]}), + ); + if bearer { + request.connection.api_key = None; + request.connection.extra_headers = vec![("Authorization".into(), "Bearer token".into())]; + } + request + .connection + .extra_headers + .push(("X-Trace".into(), "initial-request-only".into())); + let result = perform_ocr(request).await.unwrap(); + server.await.unwrap(); + assert_eq!( + result.provider_native_response, + Some(operation.as_object().unwrap().clone()) + ); + let seen = requests.lock().unwrap(); + assert!(seen[0].contains("&pages=1%2C3")); + assert!(seen[0].contains("x-trace: initial-request-only")); + for poll in &seen[1..] { + assert!(!poll.to_ascii_lowercase().contains("x-trace:")); + assert!(poll.to_ascii_lowercase().contains(if bearer { + "authorization: bearer token" + } else { + "ocp-apim-subscription-key: test-key" + })); + } +} +#[tokio::test] +async fn polling_rejects_cross_origin_location() { + let (base, requests, server) = mock_server(vec![MockResponse { + status: 202, + headers: vec![("Operation-Location", "http://example.com/operation".into())], + body: json!({}), + }]) + .await; + let error = perform_ocr(wire_request( + "azure_ai/doc-intelligence/prebuilt-read", + &base, + json!({}), + )) + .await + .unwrap_err(); + server.await.unwrap(); + assert!(error.to_string().contains("cross-origin")); + assert_eq!(requests.lock().unwrap().len(), 1); +} +#[tokio::test] +async fn polling_deadline_bounds_retry_after() { + let (base, _, server) = mock_server(vec![ + MockResponse { + status: 202, + headers: vec![("Operation-Location", "{base}/operation".into())], + body: json!({}), + }, + MockResponse { + status: 200, + headers: vec![("Retry-After", "9999".into())], + body: json!({"status":"notStarted"}), + }, + ]) + .await; + let mut request = wire_request("azure_ai/doc-intelligence/prebuilt-read", &base, json!({})); + request.connection.poll_timeout = std::time::Duration::from_millis(100); + let error = tokio::time::timeout(std::time::Duration::from_secs(1), perform_ocr(request)) + .await + .unwrap() + .unwrap_err(); + server.await.unwrap(); + assert!(error.to_string().contains("timed out")); +} + +#[rstest] +#[case("data:application/pdf;base64")] +#[case("data:application/pdf;base64,INVALID!")] +#[tokio::test] +async fn document_intelligence_rejects_invalid_data_uri(#[case] source: &str) { + let error = body( + &AZURE_DOCUMENT_INTELLIGENCE, + "model", + json!({"type":"document_url","document_url":source}), + json!({}), + ) + .await + .unwrap_err(); + assert!(matches!( + error, + crate::ocr::error::OcrError::Request(crate::ocr::error::OcrRequestError::InvalidDataUri) + )); +} diff --git a/litellm-rust/crates/core/tests/mistral_ocr.rs b/litellm-rust/crates/core/tests/mistral_ocr.rs new file mode 100644 index 00000000000..7182c1e038a --- /dev/null +++ b/litellm-rust/crates/core/tests/mistral_ocr.rs @@ -0,0 +1,100 @@ +use crate::ocr::backends::mistral::complete_url; +use crate::ocr::formats::mistral::MistralOcrFormat; +use crate::ocr::test_support::{format_body, transform_format}; +use crate::ocr::types::OcrDocument; +use rstest::rstest; +use serde_json::json; + +#[rstest] +#[case(json!({"extract_header":true}))] +#[case(json!({"extract_footer":false}))] +#[case(json!({"pages":[0,2],"include_image_base64":true,"image_limit":2,"image_min_size":100}))] +#[case(json!({"table_format":"html","confidence_scores_granularity":"word","include_blocks":true,"id":"req-123"}))] +#[case(json!({"bbox_annotation_format":{"type":"json_schema","schema":{"type":"object"}},"document_annotation_format":{"type":"json_schema"},"document_annotation_prompt":"extract"}))] +fn transform_ocr_request_includes_each_optional_param(#[case] options: serde_json::Value) { + let document = json!({"type":"document_url","document_url":"https://example.com/a.pdf"}); + let result = format_body::( + "model", + serde_json::from_value::(document.clone()).unwrap(), + options.clone(), + ) + .unwrap(); + assert_eq!(result["model"], "model"); + assert_eq!(result["document"], document); + for (name, value) in options.as_object().unwrap() { + assert_eq!(&result[name], value); + } +} + +#[test] +fn map_ocr_params_drops_unknown_params() { + let result = format_body::( + "model", + serde_json::from_value::( + json!({"type":"image_url","image_url":"data:image/png;base64,YWJj"}), + ) + .unwrap(), + json!({"unknown":{},"extract_header":true}), + ) + .unwrap(); + assert!(result.get("unknown").is_none()); + assert_eq!(result["extract_header"], true); +} + +#[test] +fn transform_ocr_response_preserves_ocr4_page_fields() { + let page = json!({"index":0,"markdown":"hello","header":"head","footer":"foot","confidence_scores":{"mean":0.99},"blocks":[{"type":"text","content":"hello"}],"images":[{"id":"img-1","image_base64":"YWJj","top_left_x":12}],"dimensions":{"width":100,"height":200,"dpi":96}}); + let response = transform_format::("model", json!({"pages":[page.clone()],"model":"returned-model","usage_info":{"pages_processed":1,"future_counter":5}}), json!({}), false).unwrap(); + for (name, value) in page.as_object().unwrap() { + if name != "images" { + assert_eq!(&response["pages"][0][name], value); + } + } + assert_eq!(response["pages"][0]["images"][0]["id"], "img-1"); + assert_eq!(response["usage_info"]["future_counter"], 5); + assert_eq!(response["model"], "returned-model"); +} + +#[test] +fn transform_ocr_response_normalizes_mistral_json() { + let result = transform_format::( + "model", + json!({"pages":[{"index":"0","markdown":"hello"}],"usage_info":{"pages_processed":"1"}}), + json!({}), + false, + ) + .unwrap(); + assert_eq!(result["pages"][0]["index"], 0); + assert_eq!(result["usage_info"]["pages_processed"], 1); + assert_eq!(result["pages"][0]["images"], serde_json::Value::Null); + assert!(result.get("provider_native_response").is_none()); + assert!( + transform_format::("model", json!({"pages":null}), json!({}), false) + .is_err() + ); +} + +#[test] +fn complete_url_defaults_and_dedupes_v1() { + assert_eq!(complete_url(None).unwrap(), "https://api.mistral.ai/v1/ocr"); + assert_eq!( + complete_url(Some(" ")).unwrap(), + "https://api.mistral.ai/v1/ocr" + ); + assert_eq!( + complete_url(Some("https://example.com")).unwrap(), + "https://example.com/v1/ocr" + ); + assert_eq!( + complete_url(Some("https://example.com/v1/")).unwrap(), + "https://example.com/v1/ocr" + ); + assert_eq!( + complete_url(Some("https://example.com/v1/ocr?tenant=a")).unwrap(), + "https://example.com/v1/ocr?tenant=a" + ); + assert_eq!( + complete_url(Some("https://example.com/v1?tenant=a")).unwrap(), + "https://example.com/v1/ocr?tenant=a" + ); +} diff --git a/litellm-rust/crates/core/tests/ocr.rs b/litellm-rust/crates/core/tests/ocr.rs new file mode 100644 index 00000000000..a9176a11d7b --- /dev/null +++ b/litellm-rust/crates/core/tests/ocr.rs @@ -0,0 +1,194 @@ +use std::sync::{Arc, Mutex}; + +use serde_json::{Value, json}; + +use super::OcrClient; +use super::test_support::{MockResponse, mock_server, perform_ocr, wire_request}; +use super::wire::{OcrWireRequest, decode_request, decode_response}; + +#[test] +fn provider_options_are_decoded_only_for_the_selected_backend() { + let mistral = decode_request(OcrWireRequest { + model: "mistral/model".into(), + document: json!({"type":"document_url","document_url":"data:application/pdf;base64,YWJj"}), + api_key: Some("key".into()), + api_base: None, + custom_llm_provider: None, + extra_headers: None, + optional_params: json!({"vertex_project":42}).as_object().unwrap().clone(), + timeout_seconds: None, + }); + assert!(mistral.is_ok()); +} + +#[rstest::rstest] +#[case("mistral/model", "/v1/ocr")] +#[case("azure_ai/model", "/providers/mistral/azure/ocr")] +#[tokio::test] +async fn performs_mistral_ocr_across_backends(#[case] model: &str, #[case] path: &str) { + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[{"index":0,"markdown":"hello","custom":"preserved"}],"usage_info":{"pages_processed":1}}))]).await; + let result = perform_ocr(wire_request( + model, + &base, + json!({"extract_header":true,"unknown":"ignored"}), + )) + .await + .unwrap(); + server.await.unwrap(); + assert_eq!(result.pages[0].markdown, "hello"); + assert_eq!(result.pages[0].extra_fields["custom"], "preserved"); + let request = seen.lock().unwrap(); + assert!(request[0].starts_with(&format!("POST {path} "))); + assert!( + request[0] + .to_ascii_lowercase() + .contains("authorization: bearer test-key\r\n") + ); + let body: Value = serde_json::from_str(request[0].split_once("\r\n\r\n").unwrap().1).unwrap(); + assert_eq!( + body, + json!({ + "model": "model", + "document": {"type":"document_url", "document_url":"data:application/pdf;base64,YWJj"}, + "extract_header": true + }) + ); + assert_eq!(result.model, "model"); + assert_eq!(result.usage_info.unwrap().pages_processed, Some(1)); +} + +#[tokio::test] +async fn uses_the_host_injected_http_client() { + let (base, seen, server) = mock_server(vec![MockResponse::json( + json!({"pages":[],"usage_info":{"pages_processed":0}}), + )]) + .await; + let mut default_headers = reqwest::header::HeaderMap::new(); + default_headers.insert( + "x-transport-owner", + reqwest::header::HeaderValue::from_static("python-sdk"), + ); + let provider_http = reqwest::Client::builder() + .default_headers(default_headers) + .build() + .expect("test HTTP client builds"); + let client = OcrClient::new(provider_http).expect("test OCR client builds"); + + client + .perform(wire_request("mistral/model", &base, json!({}))) + .await + .expect("OCR request succeeds"); + + server.await.expect("mock server completes"); + assert!(seen.lock().unwrap()[0].contains("x-transport-owner: python-sdk")); +} + +#[tokio::test] +async fn invalid_pages_fail_before_network_and_invoke_failure_hook() { + use crate::call_lifecycle::{CallLifecycleContext, CallLifecycleTiming}; + use crate::ocr::hooks::{OcrHooks, OcrLogFuture}; + struct Failures(Arc>>); + impl OcrHooks for Failures { + fn failure<'a>( + &'a self, + _context: &'a CallLifecycleContext, + error: &'a crate::Error, + _timing: &'a CallLifecycleTiming, + ) -> OcrLogFuture<'a> { + Box::pin(async move { + self.0.lock().unwrap().push(error.to_string()); + }) + } + } + let failures = Arc::new(Mutex::new(Vec::new())); + let mut request = wire_request( + "azure_ai/doc-intelligence/prebuilt-read", + "http://127.0.0.1:1", + json!({"pages":[-1]}), + ); + request.hooks = Arc::new(Failures(failures.clone())); + assert_eq!( + perform_ocr(request).await.unwrap_err().kind(), + crate::error::ErrorKind::InvalidRequest + ); + assert_eq!(failures.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn pre_call_hook_params_use_the_same_validation_as_wire_params() { + use crate::ocr::hooks::{OcrHookFuture, OcrHooks, OcrPreCallRequest}; + + struct InvalidPages; + impl OcrHooks for InvalidPages { + fn has_guardrails(&self) -> bool { + true + } + + fn pre_call(&self, request: OcrPreCallRequest) -> OcrHookFuture<'_, OcrPreCallRequest> { + Box::pin(async move { + Ok(OcrPreCallRequest { + optional_params: json!({"pages":[true]}), + ..request + }) + }) + } + } + + let mut request = wire_request( + "azure_ai/doc-intelligence/prebuilt-read", + "http://127.0.0.1:1", + json!({}), + ); + request.hooks = Arc::new(InvalidPages); + let error = perform_ocr(request).await.unwrap_err(); + assert_eq!(error.kind(), crate::error::ErrorKind::InvalidRequest); + assert!(error.to_string().contains("integers, not booleans")); +} + +#[test] +fn typed_response_rejects_malformed_pages_with_field_path() { + let error = decode_response::( + br#"{"pages":[{"index":0,"markdown":42}]}"#, + false, + ) + .unwrap_err(); + assert!(error.to_string().contains("pages[0].markdown")); + assert!(!error.to_string().contains("42")); +} + +#[rstest::rstest] +#[case("mistral/model", json!({"model":"model","document":42}), "guardrail.body.document")] +#[case("azure_ai/model", json!({"model":"model","document":{"type":"document_url","document_url":"https://example.com/doc.pdf"}}), "data URI")] +#[tokio::test] +async fn rejects_invalid_guardrail_body_before_sending( + #[case] model: &str, + #[case] body: Value, + #[case] expected: &str, +) { + use crate::ocr::hooks::{OcrDuringCallRequest, OcrHookFuture, OcrHooks}; + + struct Rewrite(Value); + impl OcrHooks for Rewrite { + fn has_guardrails(&self) -> bool { + true + } + + fn during_call( + &self, + request: OcrDuringCallRequest, + ) -> OcrHookFuture<'_, OcrDuringCallRequest> { + Box::pin(async move { + Ok(OcrDuringCallRequest { + body: self.0.clone(), + ..request + }) + }) + } + } + + let mut request = wire_request(model, "http://127.0.0.1:1", json!({})); + request.hooks = Arc::new(Rewrite(body)); + let error = perform_ocr(request).await.unwrap_err(); + assert_eq!(error.kind(), crate::error::ErrorKind::InvalidRequest); + assert!(error.to_string().contains(expected), "{error}"); +} diff --git a/litellm-rust/crates/core/tests/ocr/support.rs b/litellm-rust/crates/core/tests/ocr/support.rs new file mode 100644 index 00000000000..6d0bb638e10 --- /dev/null +++ b/litellm-rust/crates/core/tests/ocr/support.rs @@ -0,0 +1,185 @@ +use std::sync::{Arc, Mutex}; + +use serde_json::{Value, json}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpListener; + +use crate::ocr::error::OcrError; +use crate::ocr::formats::OcrFormat; +use crate::ocr::integrations::{MappedParams, OcrIntegration}; +use crate::ocr::types::OcrConnection; +use crate::ocr::wire::{OcrWireRequest, decode_request, decode_response}; +use crate::ocr::{OcrClient, OcrRequest, OcrResponseData}; + +pub(crate) fn ocr_client() -> OcrClient { + let document_http = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("test document client builds"); + OcrClient::for_test(reqwest::Client::new(), document_http) +} + +pub(crate) async fn perform_ocr(request: OcrRequest) -> Result { + ocr_client().perform(request).await +} + +pub(crate) fn params(_integration: &I, value: Value) -> MappedParams +where + I: OcrIntegration, +{ + I::Format::map_params(serde_json::from_value(value).unwrap()).unwrap() +} + +pub(crate) fn transform( + integration: &I, + model: &str, + value: Value, + options: Value, +) -> Result +where + I: OcrIntegration, +{ + let params = I::Format::map_params(serde_json::from_value(options.clone()).unwrap())?; + let preserve_native_response = integration.preserve_native_response(¶ms); + transform_format::(model, value, options, preserve_native_response) +} + +pub(crate) fn transform_format( + model: &str, + value: Value, + options: Value, + preserve_native_response: bool, +) -> Result +where + F: OcrFormat, +{ + let params = F::map_params(serde_json::from_value(options).unwrap())?; + let decoded = decode_response( + &serde_json::to_vec(&value).unwrap(), + preserve_native_response, + )?; + let mut response = F::transform_response(model, decoded.data, ¶ms)?; + response.provider_native_response = decoded.native; + Ok(response.into_json()) +} + +pub(crate) async fn body( + integration: &I, + model: &str, + document: Value, + options: Value, +) -> Result +where + I: OcrIntegration, +{ + let client = ocr_client(); + let document = integration + .prepare_document( + &client, + serde_json::from_value(document).unwrap(), + &OcrConnection::default(), + &[], + ) + .await?; + format_body::(model, document.into(), options) +} + +pub(crate) fn format_body( + model: &str, + document: F::PreparedDocument, + options: Value, +) -> Result +where + F: OcrFormat, +{ + let params = F::map_params(serde_json::from_value(options).unwrap())?; + Ok(serde_json::to_value(F::transform_request(model, document, ¶ms)?).unwrap()) +} + +pub(crate) fn wire_request(model: &str, base: &str, options: Value) -> OcrRequest { + decode_request(OcrWireRequest { + model: model.into(), + document: json!({"type":"document_url","document_url":"data:application/pdf;base64,YWJj"}), + api_key: Some("test-key".into()), + api_base: Some(base.into()), + custom_llm_provider: None, + extra_headers: None, + optional_params: options.as_object().unwrap().clone(), + timeout_seconds: Some(2.0), + }) + .unwrap() +} + +pub(crate) struct MockResponse { + pub status: u16, + pub headers: Vec<(&'static str, String)>, + pub body: Value, +} + +impl MockResponse { + pub fn json(body: Value) -> Self { + Self { + status: 200, + headers: vec![], + body, + } + } +} + +pub(crate) async fn mock_server( + responses: Vec, +) -> (String, Arc>>, tokio::task::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let requests = Arc::new(Mutex::new(Vec::new())); + let seen = requests.clone(); + let server_base = base.clone(); + let task = tokio::spawn(async move { + for response in responses { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut bytes = Vec::new(); + let mut buffer = [0u8; 4096]; + let header_end = loop { + let n = socket.read(&mut buffer).await.unwrap(); + assert!(n > 0); + bytes.extend_from_slice(&buffer[..n]); + if let Some(index) = bytes.windows(4).position(|s| s == b"\r\n\r\n") { + break index + 4; + } + }; + let length = String::from_utf8_lossy(&bytes[..header_end]) + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().unwrap()) + }) + .unwrap_or(0); + while bytes.len() < header_end + length { + let n = socket.read(&mut buffer).await.unwrap(); + assert!(n > 0); + bytes.extend_from_slice(&buffer[..n]); + } + seen.lock() + .unwrap() + .push(String::from_utf8_lossy(&bytes).into_owned()); + let body = serde_json::to_vec(&response.body).unwrap(); + let headers = response + .headers + .into_iter() + .map(|(name, value)| { + format!("{name}: {}\r\n", value.replace("{base}", &server_base)) + }) + .collect::(); + let head = format!( + "HTTP/1.1 {} OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n{}\r\n", + response.status, + body.len(), + headers + ); + socket.write_all(head.as_bytes()).await.unwrap(); + socket.write_all(&body).await.unwrap(); + } + }); + (base, requests, task) +} diff --git a/litellm-rust/crates/core/tests/ocr_prepare.rs b/litellm-rust/crates/core/tests/ocr_prepare.rs new file mode 100644 index 00000000000..dc262d1b4a9 --- /dev/null +++ b/litellm-rust/crates/core/tests/ocr_prepare.rs @@ -0,0 +1,54 @@ +use super::*; +use rstest::rstest; + +#[rstest] +#[case::mistral_default("mistral", "future-model", OcrIntegrationKind::Mistral)] +#[case::azure_document_intelligence( + "azure_ai", + "doc-intelligence/prebuilt-read", + OcrIntegrationKind::AzureDocumentIntelligence +)] +#[case::azure_mistral("azure_ai", "future-model", OcrIntegrationKind::AzureMistral)] +#[case::reducto_legacy("reducto", "parse-legacy", OcrIntegrationKind::ReductoLegacy)] +#[case::reducto_v3("reducto", "parse-v3", OcrIntegrationKind::ReductoV3)] +#[case::reducto_default("reducto", "future-model", OcrIntegrationKind::ReductoV3)] +fn typed_provider_and_model_select_the_request_contract( + #[case] provider: &str, + #[case] model: &str, + #[case] expected: OcrIntegrationKind, +) { + let provider = provider.parse().expect("known provider"); + let model = OcrModel::from(model); + assert_eq!(resolve_ocr_integration(provider, &model), expected); +} + +#[rstest] +#[case::mistral(OcrIntegrationKind::Mistral, OcrProvider::Mistral, "mistral")] +#[case::azure_mistral(OcrIntegrationKind::AzureMistral, OcrProvider::AzureAi, "azure_ai")] +#[case::azure_document_intelligence( + OcrIntegrationKind::AzureDocumentIntelligence, + OcrProvider::AzureAi, + "azure_ai" +)] +#[case::reducto_v3(OcrIntegrationKind::ReductoV3, OcrProvider::Reducto, "reducto")] +#[case::reducto_legacy(OcrIntegrationKind::ReductoLegacy, OcrProvider::Reducto, "reducto")] +fn integration_kind_exposes_its_routing_provider( + #[case] integration: OcrIntegrationKind, + #[case] expected_provider: OcrProvider, + #[case] expected_name: &str, +) { + assert_eq!(integration.provider(), expected_provider); + assert_eq!(integration.provider().as_str(), expected_name); +} + +#[test] +fn unknown_models_are_preserved_for_passthrough() { + let model = OcrModel::from("future-ocr-v9"); + assert_eq!(model, OcrModel::Passthrough("future-ocr-v9".to_string())); + assert_eq!(model.as_str(), "future-ocr-v9"); +} + +#[test] +fn unknown_providers_are_rejected_by_the_typed_boundary() { + assert!("future_provider".parse::().is_err()); +} diff --git a/litellm-rust/crates/core/tests/reducto_ocr.rs b/litellm-rust/crates/core/tests/reducto_ocr.rs new file mode 100644 index 00000000000..90fc9995de3 --- /dev/null +++ b/litellm-rust/crates/core/tests/reducto_ocr.rs @@ -0,0 +1,181 @@ +use crate::ocr::integrations::{ReductoLegacy as REDUCTO_LEGACY, ReductoV3 as REDUCTO_V3}; +use crate::ocr::test_support::{ + MockResponse, body, mock_server, perform_ocr, transform, wire_request, +}; +use rstest::rstest; +use serde_json::json; + +#[tokio::test] +async fn test_parse_v3_reducto_id_passthrough_skips_upload() { + let result=body(&REDUCTO_V3,"parse-v3",json!({"type":"document_url","document_url":"reducto://already.pdf"}),json!({"formatting":{"table_output_format":"html"},"retrieval":{"chunk_mode":"section"},"settings":{"ocr_system":"standard"}})).await.unwrap(); + assert_eq!(result["input"], "reducto://already.pdf"); + assert_eq!(result["formatting"]["table_output_format"], "html"); +} +#[tokio::test] +async fn test_parse_legacy_wraps_enhance_under_options() { + let doc = json!({"type":"document_url","document_url":"reducto://legacy.pdf"}); + let result = body( + &REDUCTO_LEGACY, + "parse-legacy", + doc.clone(), + json!({"enhance":{"agentic":[{"type":"table"}]}}), + ) + .await + .unwrap(); + assert_eq!( + result, + json!({"document_url":"reducto://legacy.pdf","options":{"enhance":{"agentic":[{"type":"table"}]}}}) + ); + assert!( + body(&REDUCTO_LEGACY, "parse-legacy", doc, json!({})) + .await + .unwrap() + .get("options") + .is_none() + ); +} +#[rstest] +#[case("http://example.com/a.pdf")] +#[case("https://example.com/a.pdf")] +#[case("data:application/pdf;base64")] +#[case("data:application/pdf;base64,INVALID!")] +#[tokio::test] +async fn test_parse_v3_rejects_plain_http_urls(#[case] source: &str) { + assert!( + body( + &REDUCTO_V3, + "parse-v3", + json!({"type":"document_url","document_url":source}), + json!({}) + ) + .await + .is_err() + ); +} +#[test] +fn reducto_groups_blocks_and_preserves_native_response() { + let raw = json!({"job_id":"job-1","usage":{"num_pages":2,"credits":3},"result":{"chunks":[ + {"blocks":[{"content":"B","bbox":{"page":2},"kind":"table"}]}, + {"blocks":[{"content":"A","bbox":{"page":1},"kind":"text"},{"content":"C","bbox":{"page":1},"kind":"text"}]}]}}); + let response = transform(&REDUCTO_V3, "parse-v3", raw.clone(), json!({})).unwrap(); + assert_eq!(response["pages"][0]["markdown"], "A\n\nC"); + assert_eq!(response["pages"][1]["markdown"], "B"); + assert_eq!(response["pages"][1]["blocks"][0]["kind"], "table"); + assert_eq!(response["provider_native_response"], raw); + assert_eq!(response["usage_info"]["credits"], 3.0); +} +#[test] +fn reducto_missing_result_and_null_result_are_distinct() { + let flat = transform( + &REDUCTO_V3, + "parse-v3", + json!({"chunks":[{"content":"text"}]}), + json!({}), + ) + .unwrap(); + assert_eq!(flat["pages"][0]["markdown"], "text"); + let null = transform( + &REDUCTO_V3, + "parse-v3", + json!({"result":null,"chunks":[{"content":"ignored"}]}), + json!({}), + ) + .unwrap(); + assert_eq!(null["pages"], json!([])); +} +#[rstest] +#[case("parse-v3")] +#[case("parse-legacy")] +#[tokio::test] +async fn test_parse_v3_file_upload_and_response_mapping( + #[case] model: &str, + #[values( + "data:application/pdf;base64,YWJj", + "data:application/pdf,abc", + "DATA:application/pdf;BASE64,YWJj#page" + )] + source: &str, +) { + let (base, seen, server) = mock_server(vec![ + MockResponse::json(json!({"file_id":"reducto://uploaded.pdf"})), + MockResponse::json( + json!({"result":{"chunks":[{"content":"hello"}]},"usage":{"num_pages":1}}), + ), + ]) + .await; + let mut request = wire_request(&format!("reducto/{model}"), &base, json!({})); + request.document = request.document.with_source(source.to_string()); + request.connection.extra_headers = vec![ + ("Content-Type".into(), "application/json".into()), + ("X-Trace".into(), "upload-test".into()), + ]; + let response = perform_ocr(request).await.unwrap(); + server.await.unwrap(); + assert_eq!(response.pages[0].markdown, "hello"); + let seen = seen.lock().unwrap(); + assert_eq!(seen.len(), 2); + assert!(seen[0].starts_with("POST /upload ")); + assert!(seen[0].contains("content-type: multipart/form-data; boundary=")); + assert!(seen[0].contains("x-trace: upload-test")); + assert!(seen[0].contains("application/pdf")); + assert!(seen[0].contains("abc")); + assert!(seen[1].starts_with("POST /parse ")); + assert!(seen[1].contains("reducto://uploaded.pdf")); +} + +#[rstest] +#[case("parse-v3")] +#[case("parse-legacy")] +#[tokio::test] +async fn upload_rejects_empty_file_id_before_parsing(#[case] model: &str) { + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"file_id":""}))]).await; + let error = perform_ocr(wire_request(&format!("reducto/{model}"), &base, json!({}))) + .await + .unwrap_err(); + server.await.unwrap(); + assert_eq!(error.kind(), crate::error::ErrorKind::InvalidResponse); + assert!(error.to_string().contains("file_id")); + assert_eq!(seen.lock().unwrap().len(), 1); +} + +#[rstest] +#[case("parse-v3")] +#[case("parse-legacy")] +#[tokio::test] +async fn guardrail_rewrites_document_before_upload(#[case] model: &str) { + use crate::ocr::hooks::{OcrDuringCallRequest, OcrHookFuture, OcrHooks}; + + struct RewriteDocument; + impl OcrHooks for RewriteDocument { + fn has_guardrails(&self) -> bool { + true + } + + fn during_call( + &self, + request: OcrDuringCallRequest, + ) -> OcrHookFuture<'_, OcrDuringCallRequest> { + Box::pin(async move { + assert_eq!( + request.body["document_url"], + "data:application/pdf;base64,YWJj" + ); + Ok(OcrDuringCallRequest { + body: json!({"type":"document_url","document_url":"reducto://guarded.pdf"}), + ..request + }) + }) + } + } + + let (base, seen, server) = + mock_server(vec![MockResponse::json(json!({"result":{"chunks":[]}}))]).await; + let mut request = wire_request(&format!("reducto/{model}"), &base, json!({})); + request.hooks = std::sync::Arc::new(RewriteDocument); + perform_ocr(request).await.unwrap(); + server.await.unwrap(); + let seen = seen.lock().unwrap(); + assert_eq!(seen.len(), 1); + assert!(seen[0].starts_with("POST /parse ")); + assert!(seen[0].contains("reducto://guarded.pdf")); +} diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index bda09a7d840..672efbb569d 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -28,6 +28,7 @@ litellm-ai-gateway = { workspace = true, default-features = false } litellm-python-interop.workspace = true pyo3.workspace = true pyo3-async-runtimes.workspace = true +reqwest.workspace = true serde.workspace = true serde_json.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/errors.rs b/litellm-rust/crates/python-bridge/src/errors.rs index 76c298abf89..b5b2d9f3fdc 100644 --- a/litellm-rust/crates/python-bridge/src/errors.rs +++ b/litellm-rust/crates/python-bridge/src/errors.rs @@ -1,4 +1,4 @@ -use litellm_core::error::Error; +use litellm_core::error::{Error, ErrorKind}; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; @@ -17,13 +17,18 @@ pyo3::create_exception!( ); pub(crate) fn core_error_to_pyerr(err: Error) -> PyErr { - match err { - Error::Auth(message) => PyValueError::new_err(message), - Error::InvalidProvider(_) - | Error::InvalidRequest(_) - | Error::InvalidType { .. } - | Error::MissingField(_) => PyValueError::new_err(err.to_string()), - other => PyRuntimeError::new_err(other.to_string()), + match err.kind() { + ErrorKind::Auth + | ErrorKind::InvalidProvider + | ErrorKind::InvalidRequest + | ErrorKind::InvalidType + | ErrorKind::MissingField => PyValueError::new_err(err.to_string()), + ErrorKind::InvalidResponse + | ErrorKind::Http + | ErrorKind::Network + | ErrorKind::Connect + | ErrorKind::Routing + | ErrorKind::Unsupported => PyRuntimeError::new_err(err.to_string()), } } @@ -34,22 +39,20 @@ pub(crate) fn core_error_to_pyerr(err: Error) -> PyErr { /// on its own path; anything after it is not, because the provider has already /// done the work and billed for it. pub(crate) fn chat_completions_error_to_pyerr(err: Error) -> PyErr { - match err { - Error::Unsupported(_) - | Error::Auth(_) - | Error::InvalidProvider(_) - | Error::InvalidRequest(_) - | Error::InvalidType { .. } - | Error::MissingField(_) - | Error::Routing(_) - // Nothing reached the provider, so serving it on Python cannot double - // bill and is the only way the caller gets an answer at all. - | Error::Connect(_) => RustBridgeDeclined::new_err(err.to_string()), - Error::Http { status, body } => { - RustUpstreamError::new_err((status, format!("{status}: {body}"))) - } - Error::Network(message) | Error::InvalidResponse(message) => { - RustUpstreamError::new_err((0u16, message)) + if let Error::Http { status, body } = err { + return RustUpstreamError::new_err((status, format!("{status}: {body}"))); + } + match err.kind() { + ErrorKind::Unsupported + | ErrorKind::Auth + | ErrorKind::InvalidProvider + | ErrorKind::InvalidRequest + | ErrorKind::InvalidType + | ErrorKind::MissingField + | ErrorKind::Routing + | ErrorKind::Connect => RustBridgeDeclined::new_err(err.to_string()), + ErrorKind::Http | ErrorKind::Network | ErrorKind::InvalidResponse => { + RustUpstreamError::new_err((0u16, err.to_string())) } } } @@ -73,6 +76,7 @@ pub(crate) fn ocr_error_to_pyerr(err: Error) -> PyErr { #[cfg(test)] mod ocr_error_tests { use super::*; + use litellm_core::error::TransportError; #[test] fn ocr_errors_preserve_python_validation_and_provider_details() { @@ -83,10 +87,13 @@ mod ocr_error_tests { assert!(mapped.is_instance_of::(py)); assert_eq!(mapped.value(py).to_string(), "Document URL is required"); } - let mapped = ocr_error_to_pyerr(Error::Http { - status: 429, - body: r#"{"message":"rate limited"}"#.to_string(), - }); + let mapped = ocr_error_to_pyerr( + TransportError::Http { + status: 429, + body: r#"{"message":"rate limited"}"#.to_string(), + } + .into(), + ); assert!(mapped.is_instance_of::(py)); let args: (u16, String) = mapped .value(py) diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 384f0be5a1b..6ef1fcb28e6 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -5,6 +5,7 @@ mod execution; mod function_trace; mod marshal; mod routes; +mod transport; use litellm_ai_gateway::io::responses_ws::ResponsesWebSocketConnection as RustResponsesWebSocketConnection; use pyo3::prelude::*; diff --git a/litellm-rust/crates/python-bridge/src/routes/definition.rs b/litellm-rust/crates/python-bridge/src/routes/definition.rs index bc51647cbad..e14b44c8732 100644 --- a/litellm-rust/crates/python-bridge/src/routes/definition.rs +++ b/litellm-rust/crates/python-bridge/src/routes/definition.rs @@ -222,11 +222,7 @@ mod tests { let module = PyModule::new(py, "routes").expect("module should be created"); crate::routes::register(&module).expect("routes should register"); let routes = [ - ( - "ocr", - "aocr", - "(model, document, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None)", - ), + ("ocr", "aocr", "(request)"), ( "transcription", "atranscription", @@ -311,7 +307,27 @@ mod tests { .expect("kwargs should accept extra_headers"); let document = PyDict::new(py); - for (sync_name, async_name) in [("ocr", "aocr"), ("transcription", "atranscription")] { + let ocr_request = PyDict::new(py); + ocr_request.set_item("model", "model").expect("sets model"); + ocr_request + .set_item("document", &document) + .expect("sets document"); + ocr_request + .set_item("extra_headers", &invalid_headers) + .expect("sets headers"); + let sync_ocr_error = module + .getattr("ocr") + .and_then(|function| function.call1((&ocr_request,))) + .expect_err("sync OCR should reject non-dict extra_headers"); + let async_ocr_error = module + .getattr("aocr") + .and_then(|function| function.call1((&ocr_request,))) + .expect_err("async OCR should reject non-dict extra_headers"); + assert!(sync_ocr_error.to_string().contains("invalid OCR request")); + assert_eq!(async_ocr_error.to_string(), sync_ocr_error.to_string()); + + { + let (sync_name, async_name) = ("transcription", "atranscription"); let sync_error = module .getattr(sync_name) .and_then(|function| function.call(("model", &document), Some(&kwargs))) diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr.rs b/litellm-rust/crates/python-bridge/src/routes/ocr.rs index cc2f8e43cea..38d88ab225f 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr.rs @@ -1,51 +1,29 @@ -use litellm_core::Error; use std::future::Future; -use litellm_ai_gateway::io::ocr::{OcrRequest, ocr as run_ocr}; +use crate::errors::ocr_error_to_pyerr; +use litellm_core::Error; +use litellm_core::ocr::wire::{OcrWireRequest, decode_request}; use pyo3::prelude::*; use serde_json::Value; -use crate::errors::ocr_error_to_pyerr; -use crate::marshal::{RouteOptions, RouteOptionsInputs, object_or_empty}; - fn prepare_ocr( inputs: OcrInputs, ) -> PyResult> + Send + 'static> { - let document = inputs.document; - let options = RouteOptions::from_python(RouteOptionsInputs { - model: inputs.model, - api_key: inputs.api_key, - api_base: inputs.api_base, - custom_llm_provider: inputs.custom_llm_provider, - extra_headers: inputs.extra_headers, - timeout_seconds: inputs.timeout_seconds, - })?; - let optional_params = object_or_empty("optional_params", inputs.optional_params)?; - + let client = crate::transport::ocr_client().map_err(ocr_error_to_pyerr)?; + let wire: OcrWireRequest = + litellm_core::ocr::wire::decode_request_value(inputs.request, "request") + .map_err(|error| ocr_error_to_pyerr(error.into()))?; + let mut request = decode_request(wire).map_err(ocr_error_to_pyerr)?; + request.connection.max_download_bytes = litellm_core::ocr::types::download_limit_bytes( + std::env::var("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB") + .ok() + .as_deref(), + ); Ok(async move { - let RouteOptions { - model, - api_key, - api_base, - custom_llm_provider, - extra_headers, - timeout, - } = options; - run_ocr(OcrRequest { - model: &model, - document, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - optional_params, - timeout, - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: Default::default(), - litellm_call_id: None, - }) - .await + client + .perform(request) + .await + .map(|response| response.into_json()) }) } @@ -54,20 +32,10 @@ bridge_route! { asynchronous = aocr, inputs = OcrInputs, required = { - model: String, #[pyo3(from_py_with = litellm_python_interop::from_py)] - document: serde_json::Value, - }, - optional = { - api_key: Option, - api_base: Option, - custom_llm_provider: Option, - #[pyo3(from_py_with = litellm_python_interop::from_py)] - extra_headers: Option, - #[pyo3(from_py_with = litellm_python_interop::from_py)] - optional_params: Option, - timeout_seconds: Option, + request: serde_json::Value, }, + optional = {}, prepare = prepare_ocr, errors = ocr_error_to_pyerr, } diff --git a/litellm-rust/crates/python-bridge/src/transport.rs b/litellm-rust/crates/python-bridge/src/transport.rs new file mode 100644 index 00000000000..fed8eb8a5ef --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/transport.rs @@ -0,0 +1,17 @@ +use std::sync::OnceLock; + +use litellm_core::error::{Error, TransportError}; +use litellm_core::ocr::OcrClient; + +pub(crate) fn ocr_client() -> Result { + static CLIENT: OnceLock> = OnceLock::new(); + CLIENT + .get_or_init(|| { + reqwest::Client::builder() + .build() + .map_err(TransportError::from) + .and_then(OcrClient::new) + }) + .clone() + .map_err(Error::from) +} diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index df3f9d2096b..7a3c4419de3 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -31,6 +31,7 @@ from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.rust_bridge import ocr as rust_ocr_bridge from litellm.rust_bridge.bindings import native_exception_types from litellm.rust_bridge.configuration import rust_enabled +from litellm.rust_bridge.timeouts import timeout_to_seconds from litellm.types.router import GenericLiteLLMParams from litellm.utils import ProviderConfigManager, client @@ -54,20 +55,22 @@ class _PreparedOCRRequest: litellm_logging_obj: LiteLLMLoggingObj -@dataclass -class _PreparedRustOCRCall: - api_key: str | None - api_base: str | None - headers: dict[str, object] - optional_params: dict[str, object] - - _RUST_OCR_PROVIDERS: Final = { "mistral", "azure_ai", - "vertex_ai", } +_AZURE_RUST_AUTH_PARAMS: Final = ( + "azure_ad_token", + "tenant_id", + "client_id", + "client_secret", + "azure_scope", + "azure_authority_host", + "azure_credential", + "azure_federated_token_file", +) + def _prepare_ocr_request( model: str, @@ -195,33 +198,38 @@ def _rust_ocr_supported(prepared_request: _PreparedOCRRequest) -> bool: return False if not prepared_request.provider_config.supports_rust_bridge(): return False + if prepared_request.custom_llm_provider == "azure_ai" and ( + callable(prepared_request.litellm_params.get("azure_ad_token_provider")) + or prepared_request.litellm_params.get("azure_username") is not None + or prepared_request.litellm_params.get("azure_password") is not None + ): + return False return prepared_request.custom_llm_provider in _RUST_OCR_PROVIDERS def _rust_bridge_optional_params( prepared_request: _PreparedOCRRequest, - resolve_secret: Callable[[str], str | None], + raw_optional_params: Mapping[str, object], ) -> dict[str, object]: - optional_params: Final = dict(prepared_request.optional_params) - if prepared_request.custom_llm_provider == "vertex_ai": - vertex_project: Final = ( - prepared_request.litellm_params.get("vertex_project") - or prepared_request.litellm_params.get("vertex_ai_project") - or litellm.vertex_project - or resolve_secret("VERTEXAI_PROJECT") - ) - vertex_location: Final = ( - prepared_request.litellm_params.get("vertex_location") - or prepared_request.litellm_params.get("vertex_ai_location") - or litellm.vertex_location - or resolve_secret("VERTEXAI_LOCATION") - or resolve_secret("VERTEX_LOCATION") - ) - if vertex_project is not None: - optional_params["vertex_project"] = vertex_project - if vertex_location is not None: - optional_params["vertex_location"] = vertex_location - return optional_params + supported_params: Final = cast( + list[str], + prepared_request.provider_config.get_supported_ocr_params(model=prepared_request.model), + ) + provider_params: Final = { + name: raw_optional_params[name] for name in supported_params if name in raw_optional_params + } + azure_auth_params: Final = ( + { + **{name: raw_optional_params[name] for name in _AZURE_RUST_AUTH_PARAMS if name in raw_optional_params}, + **({"enable_azure_ad_token_refresh": True} if litellm.enable_azure_ad_token_refresh is True else {}), + } + if prepared_request.custom_llm_provider == "azure_ai" + else {} + ) + return { + **provider_params, + **azure_auth_params, + } def _rust_bridge_api_base( @@ -239,47 +247,39 @@ def _rust_bridge_api_base( def _prepare_rust_ocr_call( prepared_request: _PreparedOCRRequest, + raw_optional_params: Mapping[str, object], + caller_api_key: str | None, + caller_api_base: str | None, resolve_api_key: Callable[[str], str | None], -) -> _PreparedRustOCRCall: - provider_config: Final = prepared_request.provider_config - api_key_env_var: Final = provider_config.get_api_key_env_var() - resolved_api_key: Final = prepared_request.api_key or ( - resolve_api_key(api_key_env_var) if api_key_env_var is not None else None +) -> dict[str, object]: + logging_api_base: Final = _rust_bridge_api_base(prepared_request, resolve_api_key) + rust_optional_params: Final = _rust_bridge_optional_params( + prepared_request, + raw_optional_params, ) - resolved_headers: Final = provider_config.validate_environment( - headers=prepared_request.extra_headers or {}, - model=prepared_request.model, - api_key=resolved_api_key, - api_base=prepared_request.api_base, - litellm_params=prepared_request.litellm_params, - ) - resolved_complete_url: Final = provider_config.get_complete_url( - api_base=prepared_request.api_base, - model=prepared_request.model, - optional_params=prepared_request.optional_params, - litellm_params=prepared_request.litellm_params, - ) - rust_api_base: Final = _rust_bridge_api_base(prepared_request, resolve_api_key) - rust_optional_params: Final = _rust_bridge_optional_params(prepared_request, resolve_api_key) prepared_request.litellm_logging_obj.pre_call( input="OCR document processing", - api_key=resolved_api_key, + api_key=prepared_request.api_key, additional_args={ "complete_input_dict": { "model": prepared_request.model, "document": prepared_request.document, - **rust_optional_params, + **prepared_request.optional_params, }, - "api_base": resolved_complete_url, - "headers": resolved_headers, + "api_base": logging_api_base or "", + "headers": prepared_request.extra_headers or {}, }, ) - return _PreparedRustOCRCall( - api_key=resolved_api_key, - api_base=rust_api_base, - headers=cast(dict[str, object], resolved_headers), - optional_params=rust_optional_params, - ) + return { + "model": prepared_request.model, + "document": prepared_request.document, + "api_key": prepared_request.api_key if caller_api_key is not None else None, + "api_base": prepared_request.api_base if caller_api_base is not None else None, + "custom_llm_provider": prepared_request.custom_llm_provider, + "extra_headers": prepared_request.extra_headers, + "optional_params": rust_optional_params, + "timeout_seconds": timeout_to_seconds(prepared_request.effective_timeout), + } def _map_rust_ocr_error( @@ -311,25 +311,22 @@ def _map_rust_ocr_error( def _run_rust_ocr( prepared_request: _PreparedOCRRequest, + raw_optional_params: Mapping[str, object], + caller_api_key: str | None, + caller_api_base: str | None, resolve_api_key: Callable[[str], str | None], ) -> OCRResponse | None: if rust_ocr_bridge.load_rust_ocr() is None: return None prepared: Final = _prepare_rust_ocr_call( prepared_request=prepared_request, + raw_optional_params=raw_optional_params, + caller_api_key=caller_api_key, + caller_api_base=caller_api_base, resolve_api_key=resolve_api_key, ) try: - rust_response: Final = rust_ocr_bridge.ocr( - model=prepared_request.model, - document=prepared_request.document, - api_key=prepared.api_key, - api_base=prepared.api_base, - custom_llm_provider=prepared_request.custom_llm_provider, - extra_headers=prepared.headers, - optional_params=prepared.optional_params, - timeout=prepared_request.effective_timeout, - ) + rust_response: Final = rust_ocr_bridge.ocr(request=prepared) except Exception as error: raise _map_rust_ocr_error(error, prepared_request, native_exception_types()) from error if rust_response is None: @@ -339,25 +336,22 @@ def _run_rust_ocr( async def _run_rust_aocr( prepared_request: _PreparedOCRRequest, + raw_optional_params: Mapping[str, object], + caller_api_key: str | None, + caller_api_base: str | None, resolve_api_key: Callable[[str], str | None], ) -> OCRResponse | None: if rust_ocr_bridge.load_rust_aocr() is None: return None prepared: Final = _prepare_rust_ocr_call( prepared_request=prepared_request, + raw_optional_params=raw_optional_params, + caller_api_key=caller_api_key, + caller_api_base=caller_api_base, resolve_api_key=resolve_api_key, ) try: - rust_response: Final = await rust_ocr_bridge.aocr( - model=prepared_request.model, - document=prepared_request.document, - api_key=prepared.api_key, - api_base=prepared.api_base, - custom_llm_provider=prepared_request.custom_llm_provider, - extra_headers=prepared.headers, - optional_params=prepared.optional_params, - timeout=prepared_request.effective_timeout, - ) + rust_response: Final = await rust_ocr_bridge.aocr(request=prepared) except Exception as error: raise _map_rust_ocr_error(error, prepared_request, native_exception_types()) from error if rust_response is None: @@ -445,6 +439,7 @@ async def aocr( "kwargs": kwargs, } try: + raw_optional_params: Final = dict(kwargs) prepared: Final = _prepare_ocr_request( model=model, document=document, @@ -464,6 +459,9 @@ async def aocr( rust_response: Final = await _run_rust_aocr( prepared_request=prepared, + raw_optional_params=raw_optional_params, + caller_api_key=api_key, + caller_api_base=api_base, resolve_api_key=get_secret_str, ) if rust_response is None: @@ -716,6 +714,7 @@ def ocr( } try: _is_async: Final = kwargs.pop("aocr", False) is True + raw_optional_params: Final = dict(kwargs) completion_kwargs["aocr"] = _is_async prepared: Final = _prepare_ocr_request( model=model, @@ -736,6 +735,9 @@ def ocr( rust_response: Final = _run_rust_ocr( prepared_request=prepared, + raw_optional_params=raw_optional_params, + caller_api_key=api_key, + caller_api_base=api_base, resolve_api_key=get_secret_str, ) if rust_response is None: diff --git a/litellm/rust_bridge/ocr.py b/litellm/rust_bridge/ocr.py index b7fdb5a98ef..c967081ab8b 100644 --- a/litellm/rust_bridge/ocr.py +++ b/litellm/rust_bridge/ocr.py @@ -5,39 +5,16 @@ from __future__ import annotations from collections.abc import Awaitable from typing import Final, Protocol, cast # noqa: TID251 # native extension exposes dynamically typed callables -import httpx - from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.timeouts import timeout_to_seconds as _timeout_to_seconds class RustOcr(Protocol): - def __call__( - self, - model: str, - document: dict[str, object], - api_key: str | None, - api_base: str | None, - custom_llm_provider: str | None, - extra_headers: dict[str, object] | None, - optional_params: dict[str, object], - timeout_seconds: float | None, - ) -> dict[str, object]: + def __call__(self, request: dict[str, object]) -> dict[str, object]: raise NotImplementedError class RustAocr(Protocol): - def __call__( - self, - model: str, - document: dict[str, object], - api_key: str | None, - api_base: str | None, - custom_llm_provider: str | None, - extra_headers: dict[str, object] | None, - optional_params: dict[str, object], - timeout_seconds: float | None, - ) -> Awaitable[dict[str, object]]: + def __call__(self, request: dict[str, object]) -> Awaitable[dict[str, object]]: raise NotImplementedError @@ -61,53 +38,11 @@ def load_rust_aocr() -> RustAocr | None: return _AOCR.load() -def ocr( - *, - model: str, - document: dict[str, object], - api_key: str | None, - api_base: str | None, - custom_llm_provider: str | None, - extra_headers: dict[str, object] | None, - optional_params: dict[str, object], - timeout: float | httpx.Timeout | None, -) -> dict[str, object] | None: +def ocr(*, request: dict[str, object]) -> dict[str, object] | None: rust_ocr: Final = load_rust_ocr() - if rust_ocr is None: - return None - return rust_ocr( - model=model, - document=document, - api_key=api_key, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - extra_headers=extra_headers, - optional_params=optional_params, - timeout_seconds=_timeout_to_seconds(timeout), - ) + return None if rust_ocr is None else rust_ocr(request=request) -async def aocr( - *, - model: str, - document: dict[str, object], - api_key: str | None, - api_base: str | None, - custom_llm_provider: str | None, - extra_headers: dict[str, object] | None, - optional_params: dict[str, object], - timeout: float | httpx.Timeout | None, -) -> dict[str, object] | None: +async def aocr(*, request: dict[str, object]) -> dict[str, object] | None: rust_aocr: Final = load_rust_aocr() - if rust_aocr is None: - return None - return await rust_aocr( - model=model, - document=document, - api_key=api_key, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - extra_headers=extra_headers, - optional_params=optional_params, - timeout_seconds=_timeout_to_seconds(timeout), - ) + return None if rust_aocr is None else await rust_aocr(request=request) diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py index c34833221cc..2f9293bfd48 100644 --- a/tests/test_litellm/ocr/test_rust_bridge.py +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -50,29 +50,8 @@ class RecordingBridge: def __init__(self) -> None: self.calls: list[dict[str, object]] = [] - def __call__( - self, - model: str, - document: dict[str, object], - api_key: str | None, - api_base: str | None, - custom_llm_provider: str | None, - extra_headers: dict[str, object] | None, - optional_params: dict[str, object], - timeout_seconds: float | None, - ) -> dict[str, object]: - self.calls.append( - { - "model": model, - "document": document, - "api_key": api_key, - "api_base": api_base, - "custom_llm_provider": custom_llm_provider, - "extra_headers": extra_headers, - "optional_params": optional_params, - "timeout_seconds": timeout_seconds, - } - ) + def __call__(self, request: dict[str, object]) -> dict[str, object]: + self.calls.append(request) return dict(FAKE_OCR_RESPONSE) @@ -82,59 +61,18 @@ class RecordingAsyncBridge: def __init__(self) -> None: self.calls: list[dict[str, object]] = [] - async def __call__( - self, - model: str, - document: dict[str, object], - api_key: str | None, - api_base: str | None, - custom_llm_provider: str | None, - extra_headers: dict[str, object] | None, - optional_params: dict[str, object], - timeout_seconds: float | None, - ) -> dict[str, object]: - self.calls.append( - { - "model": model, - "document": document, - "api_key": api_key, - "api_base": api_base, - "custom_llm_provider": custom_llm_provider, - "extra_headers": extra_headers, - "optional_params": optional_params, - "timeout_seconds": timeout_seconds, - } - ) + async def __call__(self, request: dict[str, object]) -> dict[str, object]: + self.calls.append(request) return dict(FAKE_OCR_RESPONSE) class RaisingBridge: - def __call__( - self, - model: str, - document: dict[str, object], - api_key: str | None, - api_base: str | None, - custom_llm_provider: str | None, - extra_headers: dict[str, object] | None, - optional_params: dict[str, object], - timeout_seconds: float | None, - ) -> dict[str, object]: + def __call__(self, request: dict[str, object]) -> dict[str, object]: raise RuntimeError("bridge failed") class RaisingAsyncBridge: - async def __call__( - self, - model: str, - document: dict[str, object], - api_key: str | None, - api_base: str | None, - custom_llm_provider: str | None, - extra_headers: dict[str, object] | None, - optional_params: dict[str, object], - timeout_seconds: float | None, - ) -> dict[str, object]: + async def __call__(self, request: dict[str, object]) -> dict[str, object]: raise RuntimeError("bridge failed") @@ -167,6 +105,12 @@ class FakeOCRConfig: def get_api_key_env_var(self) -> str: return self.api_key_env_var + def supports_rust_bridge(self) -> bool: + return True + + def get_supported_ocr_params(self, model: str) -> list[str]: + return ["include_image_base64", "pages"] + def validate_environment( self, *, @@ -390,9 +334,9 @@ def test_load_rust_ocr_uses_compiled_extension(monkeypatch): def test_timeout_to_seconds_handles_float_timeout_and_none(): - assert rust_bridge._timeout_to_seconds(12.5) == 12.5 - assert rust_bridge._timeout_to_seconds(None) is None - assert rust_bridge._timeout_to_seconds(httpx.Timeout(30.0, read=42.0)) == 42.0 + assert ocr_main.timeout_to_seconds(12.5) == 12.5 + assert ocr_main.timeout_to_seconds(None) is None + assert ocr_main.timeout_to_seconds(httpx.Timeout(30.0, read=42.0)) == 42.0 def test_bridge_wrapper_forwards_prepared_args_and_wraps_response(): @@ -401,32 +345,20 @@ def test_bridge_wrapper_forwards_prepared_args_and_wraps_response(): litellm.rust(True) rust_bridge._OCR.override(bridge) - response = rust_bridge.ocr( - model="mistral-ocr-latest", - document=DOCUMENT, - api_key="sk-test", - api_base="https://proxy.internal", - custom_llm_provider="mistral", - extra_headers={"Authorization": "Bearer sk-test", "x-trace-id": "trace-1"}, - optional_params={"include_image_base64": True, "pages": [0]}, - timeout=12.5, - ) - - assert response == FAKE_OCR_RESPONSE - call = bridge.calls[0] - assert call == { + request: dict[str, object] = { "model": "mistral-ocr-latest", "document": DOCUMENT, "api_key": "sk-test", "api_base": "https://proxy.internal", "custom_llm_provider": "mistral", - "extra_headers": { - "Authorization": "Bearer sk-test", - "x-trace-id": "trace-1", - }, + "extra_headers": {"x-trace-id": "trace-1"}, "optional_params": {"include_image_base64": True, "pages": [0]}, "timeout_seconds": 12.5, } + response = rust_bridge.ocr(request=request) + + assert response == FAKE_OCR_RESPONSE + assert bridge.calls == [request] @pytest.mark.asyncio @@ -436,19 +368,7 @@ async def test_bridge_wrapper_forwards_prepared_async_args_and_wraps_response(): litellm.rust(True) rust_bridge._AOCR.override(bridge) - response = await rust_bridge.aocr( - model="mistral-ocr-maas", - document=DOCUMENT, - api_key=None, - api_base=None, - custom_llm_provider="vertex_ai", - extra_headers=None, - optional_params={"vertex_project": "project-1"}, - timeout=httpx.Timeout(30.0, read=42.0), - ) - - assert response == FAKE_OCR_RESPONSE - assert bridge.calls[0] == { + request: dict[str, object] = { "model": "mistral-ocr-maas", "document": DOCUMENT, "api_key": None, @@ -458,6 +378,10 @@ async def test_bridge_wrapper_forwards_prepared_async_args_and_wraps_response(): "optional_params": {"vertex_project": "project-1"}, "timeout_seconds": 42.0, } + response = await rust_bridge.aocr(request=request) + + assert response == FAKE_OCR_RESPONSE + assert bridge.calls == [request] def test_run_rust_ocr_prepares_request_and_wraps_response(): @@ -474,6 +398,9 @@ def test_run_rust_ocr_prepares_request_and_wraps_response(): optional_params={"include_image_base64": True}, timeout=12.5, ), + raw_optional_params={"include_image_base64": True}, + caller_api_key="sk-test", + caller_api_base="https://proxy.internal", resolve_api_key=lambda _name: None, ) @@ -485,10 +412,7 @@ def test_run_rust_ocr_prepares_request_and_wraps_response(): "api_key": "sk-test", "api_base": "https://proxy.internal", "custom_llm_provider": "mistral", - "extra_headers": { - "Authorization": "Bearer sk-test", - "x-trace-id": "trace-1", - }, + "extra_headers": {"x-trace-id": "trace-1"}, "optional_params": {"include_image_base64": True}, "timeout_seconds": 12.5, } @@ -508,17 +432,20 @@ def test_rust_upstream_error_uses_ocr_provider_error_mapping(): assert mapped.message == '{"message":"invalid model"}' -def test_run_rust_ocr_resolves_key_via_secret_manager_when_missing(): +def test_run_rust_ocr_leaves_environment_key_resolution_to_rust(): bridge = RecordingBridge() litellm.rust(True) rust_bridge._OCR.override(bridge) ocr_main._run_rust_ocr( prepared_request=build_prepared_request(api_key=None, timeout=None), + raw_optional_params={}, + caller_api_key=None, + caller_api_base=None, resolve_api_key=lambda name: "sk-from-vault" if name == "MISTRAL_API_KEY" else None, ) - assert bridge.calls[0]["api_key"] == "sk-from-vault" + assert bridge.calls[0]["api_key"] is None def test_run_rust_ocr_prefers_explicit_key_over_resolver(): @@ -534,13 +461,16 @@ def test_run_rust_ocr_prefers_explicit_key_over_resolver(): api_key="sk-explicit", timeout=None, ), + raw_optional_params={}, + caller_api_key="sk-explicit", + caller_api_base=None, resolve_api_key=_resolver, ) assert bridge.calls[0]["api_key"] == "sk-explicit" -def test_run_rust_ocr_uses_provider_api_key_env_var(): +def test_run_rust_ocr_does_not_resolve_provider_api_key_in_python(): bridge = RecordingBridge() resolver_calls = [] litellm.rust(True) @@ -557,62 +487,14 @@ def test_run_rust_ocr_uses_provider_api_key_env_var(): api_key=None, timeout=None, ), + raw_optional_params={}, + caller_api_key=None, + caller_api_base=None, resolve_api_key=_resolver, ) - assert resolver_calls == ["PROVIDER_OCR_API_KEY"] - assert bridge.calls[0]["api_key"] == "sk-provider-env" - - -def test_prepare_rust_ocr_call_forwards_vertex_routing_metadata(): - bridge = RecordingBridge() - litellm.rust(True) - rust_bridge._OCR.override(bridge) - - ocr_main._run_rust_ocr( - prepared_request=build_prepared_request( - custom_llm_provider="vertex_ai", - model="mistral-ocr-maas", - litellm_params={ - "vertex_project": "project-1", - "vertex_location": "us-central1", - "vertex_credentials": "redacted", - }, - optional_params={"include_image_base64": True}, - timeout=None, - ), - resolve_api_key=lambda _name: None, - ) - - assert bridge.calls[0]["optional_params"] == { - "include_image_base64": True, - "vertex_project": "project-1", - "vertex_location": "us-central1", - } - - -def test_prepare_rust_ocr_call_resolves_vertex_routing_metadata_from_secret_manager(): - bridge = RecordingBridge() - litellm.rust(True) - rust_bridge._OCR.override(bridge) - - def _resolver(name: str) -> str | None: - return { - "VERTEXAI_PROJECT": "project-from-secret", - "VERTEXAI_LOCATION": "us-east5", - }.get(name) - - ocr_main._run_rust_ocr( - prepared_request=build_prepared_request( - custom_llm_provider="vertex_ai", - model="mistral-ocr-maas", - timeout=None, - ), - resolve_api_key=_resolver, - ) - - assert bridge.calls[0]["optional_params"]["vertex_project"] == "project-from-secret" - assert bridge.calls[0]["optional_params"]["vertex_location"] == "us-east5" + assert resolver_calls == [] + assert bridge.calls[0]["api_key"] is None def test_prepare_rust_ocr_call_resolves_azure_ai_api_base_from_secret_manager(): @@ -627,10 +509,13 @@ def test_prepare_rust_ocr_call_resolves_azure_ai_api_base_from_secret_manager(): api_base=None, timeout=None, ), + raw_optional_params={}, + caller_api_key="sk-test", + caller_api_base=None, resolve_api_key=lambda name: "https://azure.example.com" if name == "AZURE_AI_API_BASE" else None, ) - assert bridge.calls[0]["api_base"] == "https://azure.example.com" + assert bridge.calls[0]["api_base"] is None def test_prepare_rust_ocr_call_resolves_document_intelligence_endpoint(): @@ -645,12 +530,78 @@ def test_prepare_rust_ocr_call_resolves_document_intelligence_endpoint(): api_base=None, timeout=None, ), + raw_optional_params={}, + caller_api_key="sk-test", + caller_api_base=None, resolve_api_key=lambda name: ( "https://document-intelligence.example.com" if name == "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT" else None ), ) - assert bridge.calls[0]["api_base"] == "https://document-intelligence.example.com" + assert bridge.calls[0]["api_base"] is None + + +def test_prepare_rust_ocr_call_delegates_azure_authentication_to_rust(): + bridge = RecordingBridge() + litellm.rust(True) + rust_bridge._OCR.override(bridge) + + ocr_main._run_rust_ocr( + prepared_request=build_prepared_request( + custom_llm_provider="azure_ai", + model="pixtral-12b-2409", + api_key=None, + api_base="https://example.services.ai.azure.com", + extra_headers={"x-trace-id": "trace-1"}, + litellm_params={ + "azure_ad_token": "entra-token", + "tenant_id": "tenant", + "client_id": "client", + "client_secret": "secret", + }, + timeout=None, + ), + raw_optional_params={ + "azure_ad_token": "entra-token", + "tenant_id": "tenant", + "client_id": "client", + "client_secret": "secret", + }, + caller_api_key=None, + caller_api_base="https://example.services.ai.azure.com", + resolve_api_key=lambda _name: None, + ) + + assert bridge.calls[0]["extra_headers"] == {"x-trace-id": "trace-1"} + assert bridge.calls[0]["optional_params"] == { + "azure_ad_token": "entra-token", + "tenant_id": "tenant", + "client_id": "client", + "client_secret": "secret", + } + + +def test_prepare_rust_ocr_call_forwards_global_azure_refresh(monkeypatch: pytest.MonkeyPatch): + bridge = RecordingBridge() + litellm.rust(True) + rust_bridge._OCR.override(bridge) + monkeypatch.setattr(litellm, "enable_azure_ad_token_refresh", True) + + ocr_main._run_rust_ocr( + prepared_request=build_prepared_request( + custom_llm_provider="azure_ai", + model="pixtral-12b-2409", + api_key=None, + api_base="https://example.services.ai.azure.com", + timeout=None, + ), + raw_optional_params={}, + caller_api_key=None, + caller_api_base="https://example.services.ai.azure.com", + resolve_api_key=lambda _name: None, + ) + + assert bridge.calls[0]["optional_params"] == {"enable_azure_ad_token_refresh": True} def test_run_rust_ocr_runs_pre_call_logging(): @@ -667,6 +618,9 @@ def test_run_rust_ocr_runs_pre_call_logging(): optional_params={"include_image_base64": True}, timeout=None, ), + raw_optional_params={"include_image_base64": True}, + caller_api_key="sk-test", + caller_api_base="https://api.mistral.ai/v1", resolve_api_key=lambda _name: None, ) @@ -676,11 +630,8 @@ def test_run_rust_ocr_runs_pre_call_logging(): complete_input = additional_args["complete_input_dict"] assert complete_input["document"] == DOCUMENT assert complete_input["include_image_base64"] is True - assert additional_args["api_base"] == "https://api.mistral.ai/v1/ocr" - assert additional_args["headers"] == { - "Authorization": "Bearer sk-test", - "x-trace-id": "trace-1", - } + assert additional_args["api_base"] == "https://api.mistral.ai/v1" + assert additional_args["headers"] == {"x-trace-id": "trace-1"} def test_ocr_routes_to_rust_when_enabled(fake_bridge): @@ -700,10 +651,7 @@ def test_ocr_routes_to_rust_when_enabled(fake_bridge): assert call["document"] == DOCUMENT assert call["api_key"] == "sk-test" assert call["custom_llm_provider"] == "mistral" - assert call["extra_headers"] == { - "Authorization": "Bearer sk-test", - "x-trace-id": "trace-1", - } + assert call["extra_headers"] == {"x-trace-id": "trace-1"} assert call["optional_params"].get("include_image_base64") is True @@ -719,6 +667,27 @@ def test_ocr_routes_azure_ai_to_rust_when_enabled(fake_bridge): assert len(fake_bridge.calls) == 1 assert fake_bridge.calls[0]["model"] == "pixtral-12b-2409" assert fake_bridge.calls[0]["custom_llm_provider"] == "azure_ai" + assert fake_bridge.calls[0]["extra_headers"] is None + + +def test_ocr_routes_azure_entra_inputs_to_rust_without_python_auth(fake_bridge): + response = litellm.ocr( + model="azure_ai/pixtral-12b-2409", + document=DOCUMENT, + api_base="https://example.services.ai.azure.com", + azure_ad_token="entra-token", + tenant_id="tenant", + client_id="client", + ) + + assert isinstance(response, OCRResponse) + assert fake_bridge.calls[0]["api_key"] is None + assert fake_bridge.calls[0]["extra_headers"] is None + assert fake_bridge.calls[0]["optional_params"] == { + "azure_ad_token": "entra-token", + "tenant_id": "tenant", + "client_id": "client", + } def test_ocr_rust_path_converts_file_document_before_bridge(fake_bridge): @@ -772,10 +741,7 @@ async def test_aocr_routes_to_async_rust_when_enabled(fake_async_bridge): assert call["document"] == DOCUMENT assert call["api_key"] == "sk-test" assert call["custom_llm_provider"] == "mistral" - assert call["extra_headers"] == { - "Authorization": "Bearer sk-test", - "x-trace-id": "trace-1", - } + assert call["extra_headers"] == {"x-trace-id": "trace-1"} assert call["optional_params"].get("include_image_base64") is True @@ -826,6 +792,12 @@ def test_ocr_does_not_route_to_rust_when_disabled(): assert bridge.calls == [] +def test_vertex_ocr_stays_on_python_until_its_stack_boundary(): + prepared = build_prepared_request(custom_llm_provider="vertex_ai") + + assert ocr_main._rust_ocr_supported(prepared) is False + + def test_ocr_falls_back_to_python_when_bridge_unavailable(monkeypatch): """Rust enabled but no bridge available (no injected impl, no compiled wheel): ocr() must degrade to the Python HTTP handler instead of raising."""