From 7107f2ebbdace71225d73ce00d8285998ecd7539 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 9 Sep 2026 16:50:56 -0700 Subject: [PATCH] refactor(ocr): encode integration and preparation contracts in types --- .../crates/core/src/ocr/backends/azure_ai.rs | 54 +++++--- .../crates/core/src/ocr/backends/mistral.rs | 28 ++-- .../crates/core/src/ocr/backends/mod.rs | 59 ++++++-- .../crates/core/src/ocr/backends/reducto.rs | 30 ++-- .../crates/core/src/ocr/backends/vertex_ai.rs | 52 ++++--- litellm-rust/crates/core/src/ocr/document.rs | 51 ++++++- .../crates/core/src/ocr/formats/mod.rs | 2 +- litellm-rust/crates/core/src/ocr/handler.rs | 119 +++++++++++----- litellm-rust/crates/core/src/ocr/hooks.rs | 34 ++--- litellm-rust/crates/core/src/ocr/prepare.rs | 32 ++--- litellm-rust/crates/core/src/ocr/registry.rs | 130 +++++++----------- .../crates/core/tests/azure_ai_ocr.rs | 3 +- .../tests/azure_document_intelligence_ocr.rs | 7 +- litellm-rust/crates/core/tests/ocr.rs | 88 ++++++++---- .../crates/core/tests/vertex_ai_ocr.rs | 11 +- 15 files changed, 443 insertions(+), 257 deletions(-) diff --git a/litellm-rust/crates/core/src/ocr/backends/azure_ai.rs b/litellm-rust/crates/core/src/ocr/backends/azure_ai.rs index 805d7d13600..14f8c29df21 100644 --- a/litellm-rust/crates/core/src/ocr/backends/azure_ai.rs +++ b/litellm-rust/crates/core/src/ocr/backends/azure_ai.rs @@ -1,6 +1,6 @@ use crate::auth::azure::AzureAuthInputs; use crate::constants::{AZURE_AI_OCR_PATH, AZURE_DI_API_VERSION}; -use crate::ocr::backends::{OcrBackend, PreparedOcrBackend}; +use crate::ocr::backends::{HostConfig, OcrHost, OcrIntegration, PreparedOcrBackend}; use crate::ocr::error::{OcrError, OcrRequestError}; use crate::ocr::formats::document_intelligence::{ AzureDocumentIntelligenceOcrFormat, @@ -25,20 +25,20 @@ fn encode_model_id(model: &str) -> Result { Ok(utf8_percent_encode(model, PATH_SEGMENT).to_string()) } -pub struct AzureMistralOcrBackend; +#[derive(Clone, Debug)] +pub struct AzureMistral; -impl OcrBackend for AzureMistralOcrBackend { - type Config = AzureAuthInputs; - - fn provider_name(&self) -> &'static str { - "azure_ai" - } +impl OcrIntegration for AzureMistral { + type Host = AzureHost; + type Format = MistralOcrFormat; + type PreparedDocument = crate::ocr::document::InlineOcrDocument; + const FORMAT: Self::Format = MistralOcrFormat; #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] async fn prepare( &self, connection: &OcrConnection, - config: &Self::Config, + config: &HostConfig, _model: &str, _params: &MistralOcrParams, env_lookup: &(dyn Fn(&str) -> Option + Sync), @@ -57,13 +57,21 @@ impl OcrBackend for AzureMistralOcrBackend { }) } + fn validate_request_body( + &self, + body: &crate::ocr::formats::mistral::types::MistralOcrRequest, + ) -> Result<(), crate::ocr::error::OcrRequestError> { + crate::ocr::document::InlineOcrDocument::try_from(body.document.clone())?; + Ok(()) + } + async fn prepare_document( &self, client: &crate::ocr::OcrClient, document: OcrDocument, connection: &OcrConnection, _headers: &[(String, String)], - ) -> Result { + ) -> Result { crate::ocr::document::inline_remote_document( client.document_fetcher(), document, @@ -73,19 +81,19 @@ impl OcrBackend for AzureMistralOcrBackend { } } -pub struct AzureDocumentIntelligenceOcrBackend; +#[derive(Clone, Debug)] +pub struct AzureDocumentIntelligence; -impl OcrBackend for AzureDocumentIntelligenceOcrBackend { - type Config = AzureAuthInputs; - - fn provider_name(&self) -> &'static str { - "azure_ai" - } +impl OcrIntegration for AzureDocumentIntelligence { + type Host = AzureHost; + type Format = AzureDocumentIntelligenceOcrFormat; + type PreparedDocument = OcrDocument; + const FORMAT: Self::Format = AzureDocumentIntelligenceOcrFormat; async fn prepare( &self, connection: &OcrConnection, - config: &Self::Config, + config: &HostConfig, model: &str, params: &DocumentIntelligenceParams, env_lookup: &(dyn Fn(&str) -> Option + Sync), @@ -124,7 +132,7 @@ impl OcrBackend for AzureDocumentIntelligenc document: OcrDocument, _connection: &OcrConnection, _headers: &[(String, String)], - ) -> Result { + ) -> Result { Ok(document) } @@ -156,3 +164,11 @@ impl OcrBackend for AzureDocumentIntelligenc .await } } + +#[derive(Clone, Debug)] +pub struct AzureHost; + +impl OcrHost for AzureHost { + type Config = AzureAuthInputs; + const PROVIDER: crate::ocr::registry::OcrProvider = crate::ocr::registry::OcrProvider::AzureAi; +} diff --git a/litellm-rust/crates/core/src/ocr/backends/mistral.rs b/litellm-rust/crates/core/src/ocr/backends/mistral.rs index 0bc0d202b8f..2266c9bb7c7 100644 --- a/litellm-rust/crates/core/src/ocr/backends/mistral.rs +++ b/litellm-rust/crates/core/src/ocr/backends/mistral.rs @@ -1,12 +1,10 @@ use crate::constants::MISTRAL_OCR_API_BASE; -use crate::ocr::backends::{OcrBackend, PreparedOcrBackend}; +use crate::ocr::backends::{HostConfig, OcrHost, OcrIntegration, PreparedOcrBackend}; use crate::ocr::error::OcrError; use crate::ocr::formats::mistral::{MistralOcrFormat, types::MistralOcrParams}; use crate::ocr::types::{OcrConnection, OcrDocument}; use crate::providers::mistral::auth; -pub struct MistralOcrBackend; - pub fn complete_url(api_base: Option<&str>) -> String { let base = api_base .map(str::trim) @@ -20,17 +18,19 @@ pub fn complete_url(api_base: Option<&str>) -> String { } } -impl OcrBackend for MistralOcrBackend { - type Config = (); +#[derive(Clone, Debug)] +pub struct MistralDirect; - fn provider_name(&self) -> &'static str { - "mistral" - } +impl OcrIntegration for MistralDirect { + type Host = MistralHost; + type Format = MistralOcrFormat; + type PreparedDocument = OcrDocument; + const FORMAT: Self::Format = MistralOcrFormat; async fn prepare( &self, connection: &OcrConnection, - _config: &Self::Config, + _config: &HostConfig, _model: &str, _params: &MistralOcrParams, env_lookup: &(dyn Fn(&str) -> Option + Sync), @@ -52,7 +52,15 @@ impl OcrBackend for MistralOcrBackend { document: OcrDocument, _connection: &OcrConnection, _headers: &[(String, String)], - ) -> Result { + ) -> Result { Ok(document) } } + +#[derive(Clone, Debug)] +pub struct MistralHost; + +impl OcrHost for MistralHost { + type Config = (); + const PROVIDER: crate::ocr::registry::OcrProvider = crate::ocr::registry::OcrProvider::Mistral; +} diff --git a/litellm-rust/crates/core/src/ocr/backends/mod.rs b/litellm-rust/crates/core/src/ocr/backends/mod.rs index dd5cb5dbc53..d8c518137d3 100644 --- a/litellm-rust/crates/core/src/ocr/backends/mod.rs +++ b/litellm-rust/crates/core/src/ocr/backends/mod.rs @@ -17,17 +17,40 @@ pub struct PreparedOcrBackend { pub headers: Vec<(String, String)>, } -pub trait OcrBackend: Send + Sync + Sized + 'static { - type Config: Clone + Send + Sync + 'static; +pub trait OcrIntegration: Send + Sync + Sized + 'static { + type Host: OcrHost; + type Format: OcrFormat; + type PreparedDocument: Into<::PreparedDocument> + Send; + const FORMAT: Self::Format; - fn provider_name(&self) -> &'static str; + fn provider_name(&self) -> &'static str { + Self::Host::PROVIDER.as_str() + } + + fn format(&self) -> Self::Format { + Self::FORMAT + } + + fn decode_input_params( + &self, + params: serde_json::Map, + prefix: &str, + ) -> Result, super::error::OcrRequestError> { + super::registry::validate_request_format( + ¶ms, + self.supports_native_request_format(), + self.provider_name(), + )?; + Self::Format::validate_input_params(¶ms)?; + super::wire::decode_request_value(serde_json::Value::Object(params), prefix) + } fn prepare( &self, connection: &OcrConnection, - config: &Self::Config, + config: &HostConfig, model: &str, - params: &F::MappedParams, + params: &MappedParams, env_lookup: &(dyn Fn(&str) -> Option + Sync), ) -> impl Future> + Send; @@ -37,9 +60,16 @@ pub trait OcrBackend: Send + Sync + Sized + 'static { document: OcrDocument, connection: &OcrConnection, headers: &[(String, String)], - ) -> impl Future> + Send; + ) -> impl Future> + Send; - fn preserve_native_response(&self, _params: &F::MappedParams) -> bool { + fn validate_request_body( + &self, + _body: &::RequestBody, + ) -> Result<(), super::error::OcrRequestError> { + Ok(()) + } + + fn preserve_native_response(&self, _params: &MappedParams) -> bool { false } @@ -54,8 +84,10 @@ pub trait OcrBackend: Send + Sync + Sized + 'static { _url: &str, _headers: &[(String, String)], _connection: &OcrConnection, - params: &F::MappedParams, - ) -> impl Future, OcrError>> + Send { + params: &MappedParams, + ) -> impl Future< + Output = Result::ResponseBody>, OcrError>, + > + Send { super::client::read_json_response(response, self.preserve_native_response(params)) } @@ -63,3 +95,12 @@ pub trait OcrBackend: Send + Sync + Sized + 'static { false } } + +pub trait OcrHost: Send + Sync + 'static { + type Config: Clone + std::fmt::Debug + Send + Sync + 'static; + const PROVIDER: super::registry::OcrProvider; +} + +pub type InputParams = <::Format as OcrFormat>::InputParams; +pub type MappedParams = <::Format as OcrFormat>::MappedParams; +pub type HostConfig = <::Host as OcrHost>::Config; diff --git a/litellm-rust/crates/core/src/ocr/backends/reducto.rs b/litellm-rust/crates/core/src/ocr/backends/reducto.rs index c1c2f25e820..b627f476366 100644 --- a/litellm-rust/crates/core/src/ocr/backends/reducto.rs +++ b/litellm-rust/crates/core/src/ocr/backends/reducto.rs @@ -1,5 +1,5 @@ use crate::constants::{REDUCTO_ID_PREFIX, REDUCTO_OCR_API_BASE}; -use crate::ocr::backends::{OcrBackend, PreparedOcrBackend}; +use crate::ocr::backends::{HostConfig, OcrHost, OcrIntegration, PreparedOcrBackend}; use crate::ocr::document::InlineDocument; use crate::ocr::error::{OcrError, OcrRequestError, OcrResponseError}; use crate::ocr::formats::reducto::{ @@ -9,7 +9,13 @@ use crate::ocr::formats::reducto::{ use crate::ocr::types::{OcrConnection, OcrDocument}; use crate::providers::reducto::auth; -pub struct ReductoOcrBackend; +#[derive(Clone, Debug)] +pub struct ReductoHost; + +impl OcrHost for ReductoHost { + type Config = (); + const PROVIDER: crate::ocr::registry::OcrProvider = crate::ocr::registry::OcrProvider::Reducto; +} pub fn normalize_api_base(api_base: Option<&str>) -> &str { api_base @@ -67,18 +73,20 @@ async fn prepare_reducto_document( } macro_rules! impl_reducto_backend { - ($format:ty, $params:ty) => { - impl OcrBackend<$format> for ReductoOcrBackend { - type Config = (); + ($integration:ident, $format:ident, $params:ty) => { + #[derive(Clone, Debug)] + pub struct $integration; - fn provider_name(&self) -> &'static str { - "reducto" - } + impl OcrIntegration for $integration { + type Host = ReductoHost; + type Format = $format; + type PreparedDocument = ReductoFileId; + const FORMAT: Self::Format = $format; async fn prepare( &self, connection: &OcrConnection, - _config: &Self::Config, + _config: &HostConfig, _model: &str, _params: &$params, env_lookup: &(dyn Fn(&str) -> Option + Sync), @@ -118,5 +126,5 @@ macro_rules! impl_reducto_backend { }; } -impl_reducto_backend!(ReductoParseV3Format, ReductoV3Params); -impl_reducto_backend!(ReductoParseLegacyFormat, ReductoLegacyParams); +impl_reducto_backend!(ReductoV3, ReductoParseV3Format, ReductoV3Params); +impl_reducto_backend!(ReductoLegacy, ReductoParseLegacyFormat, ReductoLegacyParams); diff --git a/litellm-rust/crates/core/src/ocr/backends/vertex_ai.rs b/litellm-rust/crates/core/src/ocr/backends/vertex_ai.rs index b61804ed0ca..daaeb9c5710 100644 --- a/litellm-rust/crates/core/src/ocr/backends/vertex_ai.rs +++ b/litellm-rust/crates/core/src/ocr/backends/vertex_ai.rs @@ -1,13 +1,11 @@ use crate::constants::{VERTEX_DEEPSEEK_API_BASE, VERTEX_OCR_DEFAULT_LOCATION}; -use crate::ocr::backends::{OcrBackend, PreparedOcrBackend}; +use crate::ocr::backends::{HostConfig, OcrHost, OcrIntegration, PreparedOcrBackend}; use crate::ocr::error::OcrError; use crate::ocr::formats::deepseek::{DeepSeekOcrFormat, types::DeepSeekOcrParams}; use crate::ocr::formats::mistral::{MistralOcrFormat, types::MistralOcrParams}; use crate::ocr::types::{OcrConnection, OcrDocument}; use crate::providers::vertex_ai::auth::{self, VertexAuthInputs}; -pub struct VertexAiOcrBackend; - async fn authenticate( connection: &OcrConnection, config: &VertexAuthInputs, @@ -28,18 +26,20 @@ fn location(config: &VertexAuthInputs, env_lookup: &dyn Fn(&str) -> Option for VertexAiOcrBackend { - type Config = VertexAuthInputs; +#[derive(Clone, Debug)] +pub struct VertexMistral; - fn provider_name(&self) -> &'static str { - "vertex_ai" - } +impl OcrIntegration for VertexMistral { + type Host = VertexHost; + type Format = MistralOcrFormat; + type PreparedDocument = crate::ocr::document::InlineOcrDocument; + const FORMAT: Self::Format = MistralOcrFormat; #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] async fn prepare( &self, connection: &OcrConnection, - config: &Self::Config, + config: &HostConfig, model: &str, _params: &MistralOcrParams, env_lookup: &(dyn Fn(&str) -> Option + Sync), @@ -61,13 +61,21 @@ impl OcrBackend for VertexAiOcrBackend { }) } + fn validate_request_body( + &self, + body: &crate::ocr::formats::mistral::types::MistralOcrRequest, + ) -> Result<(), crate::ocr::error::OcrRequestError> { + crate::ocr::document::InlineOcrDocument::try_from(body.document.clone())?; + Ok(()) + } + async fn prepare_document( &self, client: &crate::ocr::OcrClient, document: OcrDocument, connection: &OcrConnection, _headers: &[(String, String)], - ) -> Result { + ) -> Result { crate::ocr::document::inline_remote_document( client.document_fetcher(), document, @@ -77,17 +85,19 @@ impl OcrBackend for VertexAiOcrBackend { } } -impl OcrBackend for VertexAiOcrBackend { - type Config = VertexAuthInputs; +#[derive(Clone, Debug)] +pub struct VertexDeepSeek; - fn provider_name(&self) -> &'static str { - "vertex_ai" - } +impl OcrIntegration for VertexDeepSeek { + type Host = VertexHost; + type Format = DeepSeekOcrFormat; + type PreparedDocument = OcrDocument; + const FORMAT: Self::Format = DeepSeekOcrFormat; async fn prepare( &self, connection: &OcrConnection, - config: &Self::Config, + config: &HostConfig, _model: &str, _params: &DeepSeekOcrParams, env_lookup: &(dyn Fn(&str) -> Option + Sync), @@ -114,7 +124,15 @@ impl OcrBackend for VertexAiOcrBackend { document: OcrDocument, _connection: &OcrConnection, _headers: &[(String, String)], - ) -> Result { + ) -> Result { Ok(document) } } + +#[derive(Clone, Debug)] +pub struct VertexHost; + +impl OcrHost for VertexHost { + type Config = VertexAuthInputs; + const PROVIDER: crate::ocr::registry::OcrProvider = crate::ocr::registry::OcrProvider::VertexAi; +} diff --git a/litellm-rust/crates/core/src/ocr/document.rs b/litellm-rust/crates/core/src/ocr/document.rs index 8771201e7bb..b865eb96671 100644 --- a/litellm-rust/crates/core/src/ocr/document.rs +++ b/litellm-rust/crates/core/src/ocr/document.rs @@ -41,14 +41,34 @@ impl<'a> InlineDocument<'a> { } } +#[derive(Debug)] +pub struct InlineOcrDocument(OcrDocument); + +impl TryFrom for InlineOcrDocument { + type Error = OcrRequestError; + + fn try_from(document: OcrDocument) -> Result { + let inline = + InlineDocument::parse(document.source())?.ok_or(OcrRequestError::InvalidDataUri)?; + inline.decode(crate::constants::OCR_INLINE_MAX_BYTES)?; + Ok(Self(document)) + } +} + +impl From for OcrDocument { + fn from(document: InlineOcrDocument) -> Self { + document.0 + } +} + pub(crate) async fn inline_remote_document( fetcher: &MediaFetcher, document: OcrDocument, connection: &OcrConnection, -) -> Result { +) -> Result { let source = document.source(); if !source.starts_with("http://") && !source.starts_with("https://") { - return Ok(document); + return Ok(InlineOcrDocument::try_from(document)?); } let url = Url::parse(source).map_err(|_| OcrRequestError::RequestField { path: "document URL".into(), @@ -64,11 +84,11 @@ pub(crate) async fn inline_remote_document( ) .await .map_err(map_media_error)?; - Ok(document.with_source(format!( + Ok(InlineOcrDocument::try_from(document.with_source(format!( "data:{};base64,{}", downloaded.content_type, STANDARD.encode(downloaded.bytes) - ))) + )))?) } fn map_media_error(error: MediaError) -> OcrError { @@ -119,6 +139,27 @@ mod tests { } } + #[rstest] + #[case("https://example.com/document.pdf")] + #[case("reducto://document")] + #[case("data:application/pdf;base64,INVALID!")] + #[case("")] + fn inline_document_rejects_unprepared_sources(#[case] source: &str) { + let document = OcrDocument::DocumentUrl { + document_url: source.into(), + }; + assert!(super::InlineOcrDocument::try_from(document).is_err()); + } + + #[test] + fn inline_document_preserves_valid_image_source() { + let document = OcrDocument::ImageUrl { + image_url: "data:image/png;base64,YWJj".into(), + }; + let prepared = super::InlineOcrDocument::try_from(document.clone()).unwrap(); + assert_eq!(OcrDocument::from(prepared), document); + } + #[test] fn preserves_mime_parameters_and_uses_standard_default() { let document = InlineDocument::parse("data:application/pdf;version=1.7;base64,YQ==") @@ -216,7 +257,7 @@ mod tests { let request = server.await.expect("server completes"); assert_eq!( - prepared.source(), + OcrDocument::from(prepared).source(), "data:application/pdf;base64,YWJj" ); assert!(!request.to_ascii_lowercase().contains("authorization")); diff --git a/litellm-rust/crates/core/src/ocr/formats/mod.rs b/litellm-rust/crates/core/src/ocr/formats/mod.rs index 7c9e0d27e05..8ca047c53f7 100644 --- a/litellm-rust/crates/core/src/ocr/formats/mod.rs +++ b/litellm-rust/crates/core/src/ocr/formats/mod.rs @@ -12,7 +12,7 @@ pub mod mistral; pub mod reducto; pub trait OcrFormat: Send + Sync + Sized + 'static { - type InputParams: Clone + Serialize + DeserializeOwned + Send + Sync; + type InputParams: std::fmt::Debug + Clone + Serialize + DeserializeOwned + Send + Sync; type MappedParams: Clone + Serialize + Send + Sync + Into; type PreparedDocument: Send; type RequestBody: Serialize + DeserializeOwned + Send + Sync; diff --git a/litellm-rust/crates/core/src/ocr/handler.rs b/litellm-rust/crates/core/src/ocr/handler.rs index 3bfb606895d..62ebf53a16b 100644 --- a/litellm-rust/crates/core/src/ocr/handler.rs +++ b/litellm-rust/crates/core/src/ocr/handler.rs @@ -1,10 +1,10 @@ use std::sync::Arc; use super::OcrClient; -use super::backends::OcrBackend; +use super::backends::{MappedParams, OcrIntegration}; use super::formats::{OcrFormat, request_error}; use super::hooks::{OcrDuringCallRequest, OcrHooks, OcrLifecycleHooks}; -use super::prepare::{PreparedOcrRequest, prepare_ocr_call}; +use super::prepare::{MappedOcrRequest, prepare_ocr_call}; use super::registry::{ AZURE_DOCUMENT_INTELLIGENCE, AZURE_MISTRAL, MISTRAL, OcrIntegrationInput, OcrIntegrationRequest, REDUCTO_LEGACY, REDUCTO_V3, VERTEX_DEEPSEEK, VERTEX_MISTRAL, @@ -56,14 +56,13 @@ pub(crate) async fn perform_ocr_request( } impl OcrExecution<'_> { - async fn run( + async fn run( self, - integration: super::registry::OcrIntegration, - input: OcrIntegrationInput, + integration: I, + input: OcrIntegrationInput, ) -> Result where - F: OcrFormat, - B: OcrBackend, + I: OcrIntegration, { let Self { client, @@ -100,18 +99,17 @@ impl OcrExecution<'_> { } #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -async fn execute_ocr_provider_call( +async fn execute_ocr_provider_call( client: &OcrClient, - request: PreparedOcrRequest, + request: MappedOcrRequest, hooks: &dyn OcrHooks, provider_name: &str, ) -> Result where - F: OcrFormat, - B: OcrBackend, + I: OcrIntegration, { - let backend = &request.integration.backend; - let format = &request.integration.format; + let backend = &request.integration; + let format = &request.integration.format(); let prepared_backend = backend .prepare( &request.connection, @@ -140,30 +138,94 @@ where let document = backend .prepare_document(client, document, &request.connection, &headers) .await?; - let body = serde_json::to_value(format.transform_ocr_request( - &request.model, - document, - &request.params, - )?) - .map_err(|_| request_error("body"))?; + let body = format.transform_ocr_request(&request.model, document.into(), &request.params)?; let body = if !backend.guard_document_before_preparation() && hooks.has_guardrails() { let guarded = hooks .during_call(OcrDuringCallRequest { model: request.model.clone(), custom_llm_provider: provider_name.into(), url: url.clone(), - body, + body: serde_json::to_value(body).map_err(|_| request_error("body"))?, }) .await?; - guarded.body - } else { + let body: OcrWireBody<::RequestBody> = + OcrWireBody::decode(guarded.body)?; + backend.validate_request_body(&body.body)?; body + } else { + OcrWireBody { + body, + extra: serde_json::Map::new(), + } }; + send_ocr_call( + client, + ReadyOcrCall { + integration: request.integration, + model: request.model, + params: request.params, + connection: request.connection, + url, + headers, + body, + }, + ) + .await +} + +#[derive(serde::Serialize)] +struct OcrWireBody { + #[serde(flatten)] + body: B, + #[serde(flatten)] + extra: serde_json::Map, +} + +impl OcrWireBody { + fn decode(value: serde_json::Value) -> Result { + let body: B = super::wire::decode_request_value(value.clone(), "guardrail.body")?; + let serde_json::Value::Object(fields) = value else { + return Err(request_error("guardrail.body")); + }; + let known = serde_json::to_value(&body).map_err(|_| request_error("guardrail.body"))?; + let extra = fields + .into_iter() + .filter(|(key, _)| known.get(key).is_none()) + .collect(); + Ok(Self { body, extra }) + } +} + +struct ReadyOcrCall { + integration: I, + model: String, + params: MappedParams, + connection: OcrConnection, + url: String, + headers: Vec<(String, String)>, + body: OcrWireBody<::RequestBody>, +} + +async fn send_ocr_call( + client: &OcrClient, + request: ReadyOcrCall, +) -> Result { + let ReadyOcrCall { + integration, + model, + params, + connection, + url, + headers, + body, + } = request; + let backend = &integration; + let format = integration.format(); let mut builder = client .provider_http() .post(&url) .json(&body) - .timeout(request.connection.timeout); + .timeout(connection.timeout); for (name, value) in &headers { builder = builder.header(name, value); } @@ -171,16 +233,9 @@ where .await .map_err(crate::error::TransportError::from)?; let decoded = backend - .read_response( - client, - response, - &url, - &headers, - &request.connection, - &request.params, - ) + .read_response(client, response, &url, &headers, &connection, ¶ms) .await?; - let response = format.transform_ocr_response(&request.model, decoded.data, &request.params)?; + let response = format.transform_ocr_response(&model, decoded.data, ¶ms)?; Ok(OcrResponseData { provider_native_response: decoded.native, ..response diff --git a/litellm-rust/crates/core/src/ocr/hooks.rs b/litellm-rust/crates/core/src/ocr/hooks.rs index e10f6234847..895ed54acfc 100644 --- a/litellm-rust/crates/core/src/ocr/hooks.rs +++ b/litellm-rust/crates/core/src/ocr/hooks.rs @@ -2,9 +2,9 @@ use std::future::Future; use std::pin::Pin; use std::sync::Arc; -use super::backends::OcrBackend; +use super::backends::OcrIntegration; use super::formats::OcrFormat; -use super::prepare::PreparedOcrRequest; +use super::prepare::MappedOcrRequest; use super::types::{OcrDocument, OcrResponseData}; use crate::Error; use crate::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; @@ -64,35 +64,29 @@ pub trait OcrHooks: Send + Sync { pub struct NoopOcrHooks; impl OcrHooks for NoopOcrHooks {} -pub(crate) struct OcrLifecycleHooks +pub(crate) struct OcrLifecycleHooks where - F: OcrFormat, - B: OcrBackend, + I: OcrIntegration, { pub hooks: Arc, pub provider_name: String, - pub marker: std::marker::PhantomData<(F, B)>, + pub marker: std::marker::PhantomData, } -impl - CallLifecycleHooks< - Result, Error>, - PreparedOcrRequest, - OcrResponseData, - > for OcrLifecycleHooks +impl CallLifecycleHooks, Error>, MappedOcrRequest, OcrResponseData> + for OcrLifecycleHooks where - F: OcrFormat, - B: OcrBackend, + I: OcrIntegration, { - type PreCallFuture<'a> = OcrHookFuture<'a, Result, Error>>; - type DuringCallFuture<'a> = OcrHookFuture<'a, PreparedOcrRequest>; + type PreCallFuture<'a> = OcrHookFuture<'a, Result, Error>>; + type DuringCallFuture<'a> = OcrHookFuture<'a, MappedOcrRequest>; type SuccessFuture<'a> = OcrLogFuture<'a>; type FailureFuture<'a> = OcrLogFuture<'a>; fn async_pre_call_hook<'a>( &'a self, _context: &'a CallLifecycleContext, - request: Result, Error>, + request: Result, Error>, ) -> Self::PreCallFuture<'a> { Box::pin(async move { let request = request?; @@ -124,8 +118,8 @@ where })?, "guardrail.optional_params", ) - .and_then(|params| request.integration.format.map_ocr_params(params))?; - Ok(Ok(PreparedOcrRequest { + .and_then(|params| request.integration.format().map_ocr_params(params))?; + Ok(Ok(MappedOcrRequest { document: changed.document, params, ..request @@ -136,7 +130,7 @@ where fn async_during_call_hook<'a>( &'a self, _context: &'a CallLifecycleContext, - request: Result, Error>, + request: Result, Error>, ) -> Self::DuringCallFuture<'a> { Box::pin(async move { request }) } diff --git a/litellm-rust/crates/core/src/ocr/prepare.rs b/litellm-rust/crates/core/src/ocr/prepare.rs index e63415cd575..c52e52365cf 100644 --- a/litellm-rust/crates/core/src/ocr/prepare.rs +++ b/litellm-rust/crates/core/src/ocr/prepare.rs @@ -1,6 +1,6 @@ -use super::backends::OcrBackend; +use super::backends::{HostConfig, InputParams, MappedParams, OcrIntegration}; use super::formats::OcrFormat; -use super::registry::OcrIntegration; + use super::types::{OcrConnection, OcrDocument}; use crate::Error; @@ -8,34 +8,32 @@ pub use super::registry::{ OcrIntegrationKind, OcrIntegrationRequest, OcrModel, OcrProvider, resolve_ocr_integration, }; -pub(crate) struct PreparedOcrRequest +pub(crate) struct MappedOcrRequest where - F: OcrFormat, - B: OcrBackend, + I: OcrIntegration, { - pub integration: OcrIntegration, + pub integration: I, pub model: String, pub document: OcrDocument, - pub params: F::MappedParams, - pub backend_config: B::Config, + pub params: MappedParams, + pub backend_config: HostConfig, pub connection: OcrConnection, } #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -pub(crate) fn prepare_ocr_call( - integration: OcrIntegration, +pub(crate) fn prepare_ocr_call( + integration: I, model: String, document: OcrDocument, - params: F::InputParams, - backend_config: B::Config, + params: InputParams, + backend_config: HostConfig, connection: OcrConnection, -) -> Result, Error> +) -> Result, Error> where - F: OcrFormat, - B: OcrBackend, + I: OcrIntegration, { - let params = integration.format.map_ocr_params(params)?; - Ok(PreparedOcrRequest { + let params = integration.format().map_ocr_params(params)?; + Ok(MappedOcrRequest { integration, model, document, diff --git a/litellm-rust/crates/core/src/ocr/registry.rs b/litellm-rust/crates/core/src/ocr/registry.rs index c6e23add60b..95c38b2b91b 100644 --- a/litellm-rust/crates/core/src/ocr/registry.rs +++ b/litellm-rust/crates/core/src/ocr/registry.rs @@ -6,57 +6,15 @@ use crate::auth::azure::AzureAuthInputs; use crate::providers::vertex_ai::auth::VertexAuthInputs; use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; -use super::backends::OcrBackend; -use super::backends::azure_ai::{AzureDocumentIntelligenceOcrBackend, AzureMistralOcrBackend}; -use super::backends::mistral::MistralOcrBackend; -use super::backends::reducto::ReductoOcrBackend; -use super::backends::vertex_ai::VertexAiOcrBackend; -use super::formats::OcrFormat; -use super::formats::deepseek::{DeepSeekOcrFormat, types::DeepSeekOcrParams}; -use super::formats::document_intelligence::{ - AzureDocumentIntelligenceOcrFormat, types::DocumentIntelligenceInputParams, -}; -use super::formats::mistral::{MistralOcrFormat, types::MistralOcrParams}; -use super::formats::reducto::{ - ReductoParseLegacyFormat, ReductoParseV3Format, - types::{ReductoLegacyParams, ReductoV3Params}, -}; +pub use super::backends::azure_ai::{AzureDocumentIntelligence, AzureMistral}; +pub use super::backends::mistral::MistralDirect; +pub use super::backends::reducto::{ReductoLegacy, ReductoV3}; +pub use super::backends::vertex_ai::{VertexDeepSeek, VertexMistral}; +use super::backends::{HostConfig, InputParams}; +pub use super::backends::{OcrHost, OcrIntegration}; use super::types::OcrRequestFormat; -pub(crate) struct OcrIntegration -where - F: OcrFormat, - B: OcrBackend, -{ - pub(crate) format: F, - pub(crate) backend: B, -} - -impl OcrIntegration -where - F: OcrFormat, - B: OcrBackend, -{ - const fn new(format: F, backend: B) -> Self { - Self { format, backend } - } - - pub(crate) fn decode_input_params( - &self, - params: Map, - prefix: &str, - ) -> Result { - validate_request_format( - ¶ms, - self.backend.supports_native_request_format(), - self.backend.provider_name(), - )?; - F::validate_input_params(¶ms)?; - super::wire::decode_request_value(Value::Object(params), prefix) - } -} - -fn validate_request_format( +pub(crate) fn validate_request_format( params: &Map, supports_native: bool, provider: &'static str, @@ -72,43 +30,55 @@ fn validate_request_format( Ok(()) } -pub(crate) const MISTRAL: OcrIntegration = - OcrIntegration::new(MistralOcrFormat, MistralOcrBackend); -pub(crate) const AZURE_MISTRAL: OcrIntegration = - OcrIntegration::new(MistralOcrFormat, AzureMistralOcrBackend); -pub(crate) const AZURE_DOCUMENT_INTELLIGENCE: OcrIntegration< - AzureDocumentIntelligenceOcrFormat, - AzureDocumentIntelligenceOcrBackend, -> = OcrIntegration::new( - AzureDocumentIntelligenceOcrFormat, - AzureDocumentIntelligenceOcrBackend, -); -pub(crate) const VERTEX_MISTRAL: OcrIntegration = - OcrIntegration::new(MistralOcrFormat, VertexAiOcrBackend); -pub(crate) const VERTEX_DEEPSEEK: OcrIntegration = - OcrIntegration::new(DeepSeekOcrFormat, VertexAiOcrBackend); -pub(crate) const REDUCTO_V3: OcrIntegration = - OcrIntegration::new(ReductoParseV3Format, ReductoOcrBackend); -pub(crate) const REDUCTO_LEGACY: OcrIntegration = - OcrIntegration::new(ReductoParseLegacyFormat, ReductoOcrBackend); +pub(crate) const MISTRAL: MistralDirect = MistralDirect; +pub(crate) const AZURE_MISTRAL: AzureMistral = AzureMistral; +pub(crate) const AZURE_DOCUMENT_INTELLIGENCE: AzureDocumentIntelligence = AzureDocumentIntelligence; +pub(crate) const VERTEX_MISTRAL: VertexMistral = VertexMistral; +pub(crate) const VERTEX_DEEPSEEK: VertexDeepSeek = VertexDeepSeek; +pub(crate) const REDUCTO_V3: ReductoV3 = ReductoV3; +pub(crate) const REDUCTO_LEGACY: ReductoLegacy = ReductoLegacy; +/// ``` +/// use litellm_core::ocr::registry::{MistralDirect, OcrIntegrationInput}; +/// use litellm_core::ocr::formats::mistral::types::MistralOcrParams; +/// let input = OcrIntegrationInput:: { +/// params: MistralOcrParams::default(), +/// backend_config: (), +/// }; +/// ``` +/// +/// ```compile_fail +/// use litellm_core::ocr::registry::{MistralDirect, OcrIntegrationInput}; +/// use litellm_core::ocr::formats::deepseek::types::DeepSeekOcrParams; +/// let input = OcrIntegrationInput:: { +/// params: DeepSeekOcrParams::default(), +/// backend_config: (), +/// }; +/// ``` +/// +/// ```compile_fail +/// use litellm_core::ocr::registry::{VertexMistral, OcrIntegrationInput}; +/// use litellm_core::ocr::formats::mistral::types::MistralOcrParams; +/// let input = OcrIntegrationInput:: { +/// params: MistralOcrParams::default(), +/// backend_config: (), +/// }; +/// ``` #[derive(Clone, Debug)] -pub struct OcrIntegrationInput { - pub params: P, - pub backend_config: C, +pub struct OcrIntegrationInput { + pub params: InputParams, + pub backend_config: HostConfig, } #[derive(Clone, Debug)] pub enum OcrIntegrationRequest { - Mistral(OcrIntegrationInput), - AzureMistral(OcrIntegrationInput), - AzureDocumentIntelligence( - OcrIntegrationInput, - ), - VertexMistral(OcrIntegrationInput), - VertexDeepSeek(OcrIntegrationInput), - ReductoV3(OcrIntegrationInput), - ReductoLegacy(OcrIntegrationInput), + Mistral(OcrIntegrationInput), + AzureMistral(OcrIntegrationInput), + AzureDocumentIntelligence(OcrIntegrationInput), + VertexMistral(OcrIntegrationInput), + VertexDeepSeek(OcrIntegrationInput), + ReductoV3(OcrIntegrationInput), + ReductoLegacy(OcrIntegrationInput), } #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/litellm-rust/crates/core/tests/azure_ai_ocr.rs b/litellm-rust/crates/core/tests/azure_ai_ocr.rs index 89e60b862c3..20755436056 100644 --- a/litellm-rust/crates/core/tests/azure_ai_ocr.rs +++ b/litellm-rust/crates/core/tests/azure_ai_ocr.rs @@ -1,4 +1,4 @@ -use crate::ocr::backends::OcrBackend; +use crate::ocr::backends::OcrIntegration; use crate::ocr::registry::{AZURE_MISTRAL, MISTRAL}; use crate::ocr::tests::body; use crate::ocr::types::OcrConnection; @@ -23,7 +23,6 @@ async fn azure_ai_mistral_ocr_uses_generic_api_base() { ..Default::default() }; let prepared = AZURE_MISTRAL - .backend .prepare( &connection, &Default::default(), diff --git a/litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs b/litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs index 8991b8ab843..a635ba18a59 100644 --- a/litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs +++ b/litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs @@ -1,4 +1,4 @@ -use crate::ocr::backends::OcrBackend; +use crate::ocr::backends::OcrIntegration; use crate::ocr::formats::OcrFormat; use crate::ocr::prepare::OcrIntegrationKind; use crate::ocr::registry::{AZURE_DOCUMENT_INTELLIGENCE, decode_integration_request}; @@ -26,7 +26,6 @@ async fn prepared_url( params: &::MappedParams, ) -> Result { AZURE_DOCUMENT_INTELLIGENCE - .backend .prepare(&connection(), &Default::default(), model, params, &|_| None) .await .map(|prepared| prepared.url) @@ -69,7 +68,7 @@ fn invalid_page_values_never_panic(#[case] pages: Value) { let input = serde_json::from_value(json!({"pages":pages})).unwrap(); assert!( AZURE_DOCUMENT_INTELLIGENCE - .format + .format() .map_ocr_params(input) .is_err() ); @@ -107,7 +106,7 @@ async fn document_intelligence_maps_features(#[case] features: Value, #[case] ex fn document_intelligence_rejects_invalid_features(#[case] features: Value) { assert!( AZURE_DOCUMENT_INTELLIGENCE - .format + .format() .map_ocr_params(serde_json::from_value(json!({"features":features})).unwrap()) .is_err() ); diff --git a/litellm-rust/crates/core/tests/ocr.rs b/litellm-rust/crates/core/tests/ocr.rs index 0a2774629e4..91eb60d843f 100644 --- a/litellm-rust/crates/core/tests/ocr.rs +++ b/litellm-rust/crates/core/tests/ocr.rs @@ -3,9 +3,9 @@ use std::sync::{Arc, Mutex}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; -use super::backends::OcrBackend; +use super::backends::{MappedParams, OcrIntegration}; use super::formats::OcrFormat; -use super::registry::OcrIntegration; + use super::types::OcrConnection; use super::wire::{OcrWireRequest, decode_request, decode_response}; use super::{OcrClient, OcrRequest}; @@ -25,55 +25,51 @@ pub(crate) async fn perform_ocr( ocr_client().perform(request).await } -pub(crate) fn params(integration: &OcrIntegration, value: Value) -> F::MappedParams +pub(crate) fn params(integration: &I, value: Value) -> MappedParams where - F: OcrFormat, - B: OcrBackend, + I: OcrIntegration, { integration - .format + .format() .map_ocr_params(serde_json::from_value(value).unwrap()) .unwrap() } -pub(crate) fn transform( - integration: &OcrIntegration, +pub(crate) fn transform( + integration: &I, model: &str, value: Value, options: Value, ) -> Result where - F: OcrFormat, - B: OcrBackend, + I: OcrIntegration, { let params = integration - .format + .format() .map_ocr_params(serde_json::from_value(options).unwrap())?; let decoded = decode_response( &serde_json::to_vec(&value).unwrap(), - integration.backend.preserve_native_response(¶ms), + integration.preserve_native_response(¶ms), )?; let mut response = integration - .format + .format() .transform_ocr_response(model, decoded.data, ¶ms)?; response.provider_native_response = decoded.native; Ok(response.into_json()) } -pub(crate) async fn body( - integration: &OcrIntegration, +pub(crate) async fn body( + integration: &I, model: &str, document: Value, options: Value, ) -> Result where - F: OcrFormat, - B: OcrBackend, + I: OcrIntegration, { let params = integration - .format + .format() .map_ocr_params(serde_json::from_value(options).unwrap())?; let client = ocr_client(); let document = integration - .backend .prepare_document( &client, serde_json::from_value(document).unwrap(), @@ -81,12 +77,14 @@ where &[], ) .await?; - Ok(serde_json::to_value( - integration - .format - .transform_ocr_request(model, document, ¶ms)?, + Ok( + serde_json::to_value(integration.format().transform_ocr_request( + model, + document.into(), + ¶ms, + )?) + .unwrap(), ) - .unwrap()) } pub(crate) fn wire_request(model: &str, base: &str, options: Value) -> OcrRequest { decode_request(OcrWireRequest { @@ -343,3 +341,45 @@ fn typed_response_rejects_malformed_pages_with_field_path() { assert!(error.to_string().contains("pages[0].markdown")); assert!(!error.to_string().contains("42")); } + +#[rstest::rstest] +#[case("mistral/model", json!({"model":"model","document":42}), "guardrail.body.document")] +#[case("azure_ai/model", json!({"model":"model","document":{"type":"document_url","document_url":"https://example.com/doc.pdf"}}), "data URI")] +#[case("vertex_ai/model", json!({"model":"model","document":{"type":"document_url","document_url":"data:application/pdf;base64,INVALID!"}}), "data URI")] +#[tokio::test] +async fn rejects_invalid_guardrail_body_before_sending( + #[case] model: &str, + #[case] body: Value, + #[case] expected: &str, +) { + use crate::ocr::hooks::{OcrDuringCallRequest, OcrHookFuture, OcrHooks}; + + struct Rewrite(Value); + impl OcrHooks for Rewrite { + fn has_guardrails(&self) -> bool { + true + } + + fn during_call( + &self, + request: OcrDuringCallRequest, + ) -> OcrHookFuture<'_, OcrDuringCallRequest> { + Box::pin(async move { + Ok(OcrDuringCallRequest { + body: self.0.clone(), + ..request + }) + }) + } + } + + let mut request = wire_request( + model, + "http://127.0.0.1:1", + json!({"vertex_project":"test-project"}), + ); + request.hooks = Arc::new(Rewrite(body)); + let error = perform_ocr(request).await.unwrap_err(); + assert_eq!(error.kind(), crate::error::ErrorKind::InvalidRequest); + assert!(error.to_string().contains(expected), "{error}"); +} diff --git a/litellm-rust/crates/core/tests/vertex_ai_ocr.rs b/litellm-rust/crates/core/tests/vertex_ai_ocr.rs index c362a3bd3f6..8dc4aafce85 100644 --- a/litellm-rust/crates/core/tests/vertex_ai_ocr.rs +++ b/litellm-rust/crates/core/tests/vertex_ai_ocr.rs @@ -1,5 +1,4 @@ -use crate::ocr::backends::OcrBackend; -use crate::ocr::formats::mistral::MistralOcrFormat; +use crate::ocr::backends::OcrIntegration; use crate::ocr::registry::{MISTRAL, VERTEX_MISTRAL}; use crate::ocr::tests::body; use crate::ocr::types::OcrConnection; @@ -35,8 +34,8 @@ async fn vertex_mistral_url_uses_project_location_and_model() { .unwrap(), ) .unwrap(); - let prepared = OcrBackend::::prepare( - &VERTEX_MISTRAL.backend, + let prepared = OcrIntegration::prepare( + &VERTEX_MISTRAL, &connection, &config, "mistral-ocr-maas", @@ -57,8 +56,8 @@ async fn vertex_mistral_resolves_provider_environment_in_the_backend() { api_key: Some("token".into()), ..Default::default() }; - let prepared = OcrBackend::::prepare( - &VERTEX_MISTRAL.backend, + let prepared = OcrIntegration::prepare( + &VERTEX_MISTRAL, &connection, &Default::default(), "model",