refactor(ocr): encode integration and preparation contracts in types

This commit is contained in:
Yujong Lee 2026-09-09 16:50:56 -07:00
parent 6a9e4287f5
commit 7107f2ebbd
15 changed files with 443 additions and 257 deletions

View file

@ -1,6 +1,6 @@
use crate::auth::azure::AzureAuthInputs;
use crate::constants::{AZURE_AI_OCR_PATH, AZURE_DI_API_VERSION};
use crate::ocr::backends::{OcrBackend, PreparedOcrBackend};
use crate::ocr::backends::{HostConfig, OcrHost, OcrIntegration, PreparedOcrBackend};
use crate::ocr::error::{OcrError, OcrRequestError};
use crate::ocr::formats::document_intelligence::{
AzureDocumentIntelligenceOcrFormat,
@ -25,20 +25,20 @@ fn encode_model_id(model: &str) -> Result<String, OcrRequestError> {
Ok(utf8_percent_encode(model, PATH_SEGMENT).to_string())
}
pub struct AzureMistralOcrBackend;
#[derive(Clone, Debug)]
pub struct AzureMistral;
impl OcrBackend<MistralOcrFormat> for AzureMistralOcrBackend {
type Config = AzureAuthInputs;
fn provider_name(&self) -> &'static str {
"azure_ai"
}
impl OcrIntegration for AzureMistral {
type Host = AzureHost;
type Format = MistralOcrFormat;
type PreparedDocument = crate::ocr::document::InlineOcrDocument;
const FORMAT: Self::Format = MistralOcrFormat;
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
async fn prepare(
&self,
connection: &OcrConnection,
config: &Self::Config,
config: &HostConfig<Self>,
_model: &str,
_params: &MistralOcrParams,
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
@ -57,13 +57,21 @@ impl OcrBackend<MistralOcrFormat> for AzureMistralOcrBackend {
})
}
fn validate_request_body(
&self,
body: &crate::ocr::formats::mistral::types::MistralOcrRequest,
) -> Result<(), crate::ocr::error::OcrRequestError> {
crate::ocr::document::InlineOcrDocument::try_from(body.document.clone())?;
Ok(())
}
async fn prepare_document(
&self,
client: &crate::ocr::OcrClient,
document: OcrDocument,
connection: &OcrConnection,
_headers: &[(String, String)],
) -> Result<OcrDocument, OcrError> {
) -> Result<Self::PreparedDocument, OcrError> {
crate::ocr::document::inline_remote_document(
client.document_fetcher(),
document,
@ -73,19 +81,19 @@ impl OcrBackend<MistralOcrFormat> for AzureMistralOcrBackend {
}
}
pub struct AzureDocumentIntelligenceOcrBackend;
#[derive(Clone, Debug)]
pub struct AzureDocumentIntelligence;
impl OcrBackend<AzureDocumentIntelligenceOcrFormat> for AzureDocumentIntelligenceOcrBackend {
type Config = AzureAuthInputs;
fn provider_name(&self) -> &'static str {
"azure_ai"
}
impl OcrIntegration for AzureDocumentIntelligence {
type Host = AzureHost;
type Format = AzureDocumentIntelligenceOcrFormat;
type PreparedDocument = OcrDocument;
const FORMAT: Self::Format = AzureDocumentIntelligenceOcrFormat;
async fn prepare(
&self,
connection: &OcrConnection,
config: &Self::Config,
config: &HostConfig<Self>,
model: &str,
params: &DocumentIntelligenceParams,
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
@ -124,7 +132,7 @@ impl OcrBackend<AzureDocumentIntelligenceOcrFormat> for AzureDocumentIntelligenc
document: OcrDocument,
_connection: &OcrConnection,
_headers: &[(String, String)],
) -> Result<OcrDocument, OcrError> {
) -> Result<Self::PreparedDocument, OcrError> {
Ok(document)
}
@ -156,3 +164,11 @@ impl OcrBackend<AzureDocumentIntelligenceOcrFormat> for AzureDocumentIntelligenc
.await
}
}
#[derive(Clone, Debug)]
pub struct AzureHost;
impl OcrHost for AzureHost {
type Config = AzureAuthInputs;
const PROVIDER: crate::ocr::registry::OcrProvider = crate::ocr::registry::OcrProvider::AzureAi;
}

View file

@ -1,12 +1,10 @@
use crate::constants::MISTRAL_OCR_API_BASE;
use crate::ocr::backends::{OcrBackend, PreparedOcrBackend};
use crate::ocr::backends::{HostConfig, OcrHost, OcrIntegration, PreparedOcrBackend};
use crate::ocr::error::OcrError;
use crate::ocr::formats::mistral::{MistralOcrFormat, types::MistralOcrParams};
use crate::ocr::types::{OcrConnection, OcrDocument};
use crate::providers::mistral::auth;
pub struct MistralOcrBackend;
pub fn complete_url(api_base: Option<&str>) -> String {
let base = api_base
.map(str::trim)
@ -20,17 +18,19 @@ pub fn complete_url(api_base: Option<&str>) -> String {
}
}
impl OcrBackend<MistralOcrFormat> for MistralOcrBackend {
type Config = ();
#[derive(Clone, Debug)]
pub struct MistralDirect;
fn provider_name(&self) -> &'static str {
"mistral"
}
impl OcrIntegration for MistralDirect {
type Host = MistralHost;
type Format = MistralOcrFormat;
type PreparedDocument = OcrDocument;
const FORMAT: Self::Format = MistralOcrFormat;
async fn prepare(
&self,
connection: &OcrConnection,
_config: &Self::Config,
_config: &HostConfig<Self>,
_model: &str,
_params: &MistralOcrParams,
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
@ -52,7 +52,15 @@ impl OcrBackend<MistralOcrFormat> for MistralOcrBackend {
document: OcrDocument,
_connection: &OcrConnection,
_headers: &[(String, String)],
) -> Result<OcrDocument, OcrError> {
) -> Result<Self::PreparedDocument, OcrError> {
Ok(document)
}
}
#[derive(Clone, Debug)]
pub struct MistralHost;
impl OcrHost for MistralHost {
type Config = ();
const PROVIDER: crate::ocr::registry::OcrProvider = crate::ocr::registry::OcrProvider::Mistral;
}

View file

@ -17,17 +17,40 @@ pub struct PreparedOcrBackend {
pub headers: Vec<(String, String)>,
}
pub trait OcrBackend<F: OcrFormat>: Send + Sync + Sized + 'static {
type Config: Clone + Send + Sync + 'static;
pub trait OcrIntegration: Send + Sync + Sized + 'static {
type Host: OcrHost;
type Format: OcrFormat;
type PreparedDocument: Into<<Self::Format as OcrFormat>::PreparedDocument> + Send;
const FORMAT: Self::Format;
fn provider_name(&self) -> &'static str;
fn provider_name(&self) -> &'static str {
Self::Host::PROVIDER.as_str()
}
fn format(&self) -> Self::Format {
Self::FORMAT
}
fn decode_input_params(
&self,
params: serde_json::Map<String, serde_json::Value>,
prefix: &str,
) -> Result<InputParams<Self>, super::error::OcrRequestError> {
super::registry::validate_request_format(
&params,
self.supports_native_request_format(),
self.provider_name(),
)?;
Self::Format::validate_input_params(&params)?;
super::wire::decode_request_value(serde_json::Value::Object(params), prefix)
}
fn prepare(
&self,
connection: &OcrConnection,
config: &Self::Config,
config: &HostConfig<Self>,
model: &str,
params: &F::MappedParams,
params: &MappedParams<Self>,
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
) -> impl Future<Output = Result<PreparedOcrBackend, OcrError>> + Send;
@ -37,9 +60,16 @@ pub trait OcrBackend<F: OcrFormat>: Send + Sync + Sized + 'static {
document: OcrDocument,
connection: &OcrConnection,
headers: &[(String, String)],
) -> impl Future<Output = Result<F::PreparedDocument, OcrError>> + Send;
) -> impl Future<Output = Result<Self::PreparedDocument, OcrError>> + Send;
fn preserve_native_response(&self, _params: &F::MappedParams) -> bool {
fn validate_request_body(
&self,
_body: &<Self::Format as OcrFormat>::RequestBody,
) -> Result<(), super::error::OcrRequestError> {
Ok(())
}
fn preserve_native_response(&self, _params: &MappedParams<Self>) -> bool {
false
}
@ -54,8 +84,10 @@ pub trait OcrBackend<F: OcrFormat>: Send + Sync + Sized + 'static {
_url: &str,
_headers: &[(String, String)],
_connection: &OcrConnection,
params: &F::MappedParams,
) -> impl Future<Output = Result<DecodedOcrResponse<F::ResponseBody>, OcrError>> + Send {
params: &MappedParams<Self>,
) -> impl Future<
Output = Result<DecodedOcrResponse<<Self::Format as OcrFormat>::ResponseBody>, OcrError>,
> + Send {
super::client::read_json_response(response, self.preserve_native_response(params))
}
@ -63,3 +95,12 @@ pub trait OcrBackend<F: OcrFormat>: Send + Sync + Sized + 'static {
false
}
}
pub trait OcrHost: Send + Sync + 'static {
type Config: Clone + std::fmt::Debug + Send + Sync + 'static;
const PROVIDER: super::registry::OcrProvider;
}
pub type InputParams<I> = <<I as OcrIntegration>::Format as OcrFormat>::InputParams;
pub type MappedParams<I> = <<I as OcrIntegration>::Format as OcrFormat>::MappedParams;
pub type HostConfig<I> = <<I as OcrIntegration>::Host as OcrHost>::Config;

View file

@ -1,5 +1,5 @@
use crate::constants::{REDUCTO_ID_PREFIX, REDUCTO_OCR_API_BASE};
use crate::ocr::backends::{OcrBackend, PreparedOcrBackend};
use crate::ocr::backends::{HostConfig, OcrHost, OcrIntegration, PreparedOcrBackend};
use crate::ocr::document::InlineDocument;
use crate::ocr::error::{OcrError, OcrRequestError, OcrResponseError};
use crate::ocr::formats::reducto::{
@ -9,7 +9,13 @@ use crate::ocr::formats::reducto::{
use crate::ocr::types::{OcrConnection, OcrDocument};
use crate::providers::reducto::auth;
pub struct ReductoOcrBackend;
#[derive(Clone, Debug)]
pub struct ReductoHost;
impl OcrHost for ReductoHost {
type Config = ();
const PROVIDER: crate::ocr::registry::OcrProvider = crate::ocr::registry::OcrProvider::Reducto;
}
pub fn normalize_api_base(api_base: Option<&str>) -> &str {
api_base
@ -67,18 +73,20 @@ async fn prepare_reducto_document(
}
macro_rules! impl_reducto_backend {
($format:ty, $params:ty) => {
impl OcrBackend<$format> for ReductoOcrBackend {
type Config = ();
($integration:ident, $format:ident, $params:ty) => {
#[derive(Clone, Debug)]
pub struct $integration;
fn provider_name(&self) -> &'static str {
"reducto"
}
impl OcrIntegration for $integration {
type Host = ReductoHost;
type Format = $format;
type PreparedDocument = ReductoFileId;
const FORMAT: Self::Format = $format;
async fn prepare(
&self,
connection: &OcrConnection,
_config: &Self::Config,
_config: &HostConfig<Self>,
_model: &str,
_params: &$params,
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
@ -118,5 +126,5 @@ macro_rules! impl_reducto_backend {
};
}
impl_reducto_backend!(ReductoParseV3Format, ReductoV3Params);
impl_reducto_backend!(ReductoParseLegacyFormat, ReductoLegacyParams);
impl_reducto_backend!(ReductoV3, ReductoParseV3Format, ReductoV3Params);
impl_reducto_backend!(ReductoLegacy, ReductoParseLegacyFormat, ReductoLegacyParams);

View file

@ -1,13 +1,11 @@
use crate::constants::{VERTEX_DEEPSEEK_API_BASE, VERTEX_OCR_DEFAULT_LOCATION};
use crate::ocr::backends::{OcrBackend, PreparedOcrBackend};
use crate::ocr::backends::{HostConfig, OcrHost, OcrIntegration, PreparedOcrBackend};
use crate::ocr::error::OcrError;
use crate::ocr::formats::deepseek::{DeepSeekOcrFormat, types::DeepSeekOcrParams};
use crate::ocr::formats::mistral::{MistralOcrFormat, types::MistralOcrParams};
use crate::ocr::types::{OcrConnection, OcrDocument};
use crate::providers::vertex_ai::auth::{self, VertexAuthInputs};
pub struct VertexAiOcrBackend;
async fn authenticate(
connection: &OcrConnection,
config: &VertexAuthInputs,
@ -28,18 +26,20 @@ fn location(config: &VertexAuthInputs, env_lookup: &dyn Fn(&str) -> Option<Strin
.unwrap_or_else(|| VERTEX_OCR_DEFAULT_LOCATION.to_string())
}
impl OcrBackend<MistralOcrFormat> for VertexAiOcrBackend {
type Config = VertexAuthInputs;
#[derive(Clone, Debug)]
pub struct VertexMistral;
fn provider_name(&self) -> &'static str {
"vertex_ai"
}
impl OcrIntegration for VertexMistral {
type Host = VertexHost;
type Format = MistralOcrFormat;
type PreparedDocument = crate::ocr::document::InlineOcrDocument;
const FORMAT: Self::Format = MistralOcrFormat;
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
async fn prepare(
&self,
connection: &OcrConnection,
config: &Self::Config,
config: &HostConfig<Self>,
model: &str,
_params: &MistralOcrParams,
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
@ -61,13 +61,21 @@ impl OcrBackend<MistralOcrFormat> for VertexAiOcrBackend {
})
}
fn validate_request_body(
&self,
body: &crate::ocr::formats::mistral::types::MistralOcrRequest,
) -> Result<(), crate::ocr::error::OcrRequestError> {
crate::ocr::document::InlineOcrDocument::try_from(body.document.clone())?;
Ok(())
}
async fn prepare_document(
&self,
client: &crate::ocr::OcrClient,
document: OcrDocument,
connection: &OcrConnection,
_headers: &[(String, String)],
) -> Result<OcrDocument, OcrError> {
) -> Result<Self::PreparedDocument, OcrError> {
crate::ocr::document::inline_remote_document(
client.document_fetcher(),
document,
@ -77,17 +85,19 @@ impl OcrBackend<MistralOcrFormat> for VertexAiOcrBackend {
}
}
impl OcrBackend<DeepSeekOcrFormat> for VertexAiOcrBackend {
type Config = VertexAuthInputs;
#[derive(Clone, Debug)]
pub struct VertexDeepSeek;
fn provider_name(&self) -> &'static str {
"vertex_ai"
}
impl OcrIntegration for VertexDeepSeek {
type Host = VertexHost;
type Format = DeepSeekOcrFormat;
type PreparedDocument = OcrDocument;
const FORMAT: Self::Format = DeepSeekOcrFormat;
async fn prepare(
&self,
connection: &OcrConnection,
config: &Self::Config,
config: &HostConfig<Self>,
_model: &str,
_params: &DeepSeekOcrParams,
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
@ -114,7 +124,15 @@ impl OcrBackend<DeepSeekOcrFormat> for VertexAiOcrBackend {
document: OcrDocument,
_connection: &OcrConnection,
_headers: &[(String, String)],
) -> Result<OcrDocument, OcrError> {
) -> Result<Self::PreparedDocument, OcrError> {
Ok(document)
}
}
#[derive(Clone, Debug)]
pub struct VertexHost;
impl OcrHost for VertexHost {
type Config = VertexAuthInputs;
const PROVIDER: crate::ocr::registry::OcrProvider = crate::ocr::registry::OcrProvider::VertexAi;
}

View file

@ -41,14 +41,34 @@ impl<'a> InlineDocument<'a> {
}
}
#[derive(Debug)]
pub struct InlineOcrDocument(OcrDocument);
impl TryFrom<OcrDocument> for InlineOcrDocument {
type Error = OcrRequestError;
fn try_from(document: OcrDocument) -> Result<Self, Self::Error> {
let inline =
InlineDocument::parse(document.source())?.ok_or(OcrRequestError::InvalidDataUri)?;
inline.decode(crate::constants::OCR_INLINE_MAX_BYTES)?;
Ok(Self(document))
}
}
impl From<InlineOcrDocument> for OcrDocument {
fn from(document: InlineOcrDocument) -> Self {
document.0
}
}
pub(crate) async fn inline_remote_document(
fetcher: &MediaFetcher,
document: OcrDocument,
connection: &OcrConnection,
) -> Result<OcrDocument, OcrError> {
) -> Result<InlineOcrDocument, OcrError> {
let source = document.source();
if !source.starts_with("http://") && !source.starts_with("https://") {
return Ok(document);
return Ok(InlineOcrDocument::try_from(document)?);
}
let url = Url::parse(source).map_err(|_| OcrRequestError::RequestField {
path: "document URL".into(),
@ -64,11 +84,11 @@ pub(crate) async fn inline_remote_document(
)
.await
.map_err(map_media_error)?;
Ok(document.with_source(format!(
Ok(InlineOcrDocument::try_from(document.with_source(format!(
"data:{};base64,{}",
downloaded.content_type,
STANDARD.encode(downloaded.bytes)
)))
)))?)
}
fn map_media_error(error: MediaError) -> OcrError {
@ -119,6 +139,27 @@ mod tests {
}
}
#[rstest]
#[case("https://example.com/document.pdf")]
#[case("reducto://document")]
#[case("data:application/pdf;base64,INVALID!")]
#[case("")]
fn inline_document_rejects_unprepared_sources(#[case] source: &str) {
let document = OcrDocument::DocumentUrl {
document_url: source.into(),
};
assert!(super::InlineOcrDocument::try_from(document).is_err());
}
#[test]
fn inline_document_preserves_valid_image_source() {
let document = OcrDocument::ImageUrl {
image_url: "data:image/png;base64,YWJj".into(),
};
let prepared = super::InlineOcrDocument::try_from(document.clone()).unwrap();
assert_eq!(OcrDocument::from(prepared), document);
}
#[test]
fn preserves_mime_parameters_and_uses_standard_default() {
let document = InlineDocument::parse("data:application/pdf;version=1.7;base64,YQ==")
@ -216,7 +257,7 @@ mod tests {
let request = server.await.expect("server completes");
assert_eq!(
prepared.source(),
OcrDocument::from(prepared).source(),
"data:application/pdf;base64,YWJj"
);
assert!(!request.to_ascii_lowercase().contains("authorization"));

View file

@ -12,7 +12,7 @@ pub mod mistral;
pub mod reducto;
pub trait OcrFormat: Send + Sync + Sized + 'static {
type InputParams: Clone + Serialize + DeserializeOwned + Send + Sync;
type InputParams: std::fmt::Debug + Clone + Serialize + DeserializeOwned + Send + Sync;
type MappedParams: Clone + Serialize + Send + Sync + Into<Self::InputParams>;
type PreparedDocument: Send;
type RequestBody: Serialize + DeserializeOwned + Send + Sync;

View file

@ -1,10 +1,10 @@
use std::sync::Arc;
use super::OcrClient;
use super::backends::OcrBackend;
use super::backends::{MappedParams, OcrIntegration};
use super::formats::{OcrFormat, request_error};
use super::hooks::{OcrDuringCallRequest, OcrHooks, OcrLifecycleHooks};
use super::prepare::{PreparedOcrRequest, prepare_ocr_call};
use super::prepare::{MappedOcrRequest, prepare_ocr_call};
use super::registry::{
AZURE_DOCUMENT_INTELLIGENCE, AZURE_MISTRAL, MISTRAL, OcrIntegrationInput,
OcrIntegrationRequest, REDUCTO_LEGACY, REDUCTO_V3, VERTEX_DEEPSEEK, VERTEX_MISTRAL,
@ -56,14 +56,13 @@ pub(crate) async fn perform_ocr_request(
}
impl OcrExecution<'_> {
async fn run<F, B>(
async fn run<I>(
self,
integration: super::registry::OcrIntegration<F, B>,
input: OcrIntegrationInput<F::InputParams, B::Config>,
integration: I,
input: OcrIntegrationInput<I>,
) -> Result<OcrResponseData, Error>
where
F: OcrFormat,
B: OcrBackend<F>,
I: OcrIntegration,
{
let Self {
client,
@ -100,18 +99,17 @@ impl OcrExecution<'_> {
}
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
async fn execute_ocr_provider_call<F, B>(
async fn execute_ocr_provider_call<I>(
client: &OcrClient,
request: PreparedOcrRequest<F, B>,
request: MappedOcrRequest<I>,
hooks: &dyn OcrHooks,
provider_name: &str,
) -> Result<OcrResponseData, Error>
where
F: OcrFormat,
B: OcrBackend<F>,
I: OcrIntegration,
{
let backend = &request.integration.backend;
let format = &request.integration.format;
let backend = &request.integration;
let format = &request.integration.format();
let prepared_backend = backend
.prepare(
&request.connection,
@ -140,30 +138,94 @@ where
let document = backend
.prepare_document(client, document, &request.connection, &headers)
.await?;
let body = serde_json::to_value(format.transform_ocr_request(
&request.model,
document,
&request.params,
)?)
.map_err(|_| request_error("body"))?;
let body = format.transform_ocr_request(&request.model, document.into(), &request.params)?;
let body = if !backend.guard_document_before_preparation() && hooks.has_guardrails() {
let guarded = hooks
.during_call(OcrDuringCallRequest {
model: request.model.clone(),
custom_llm_provider: provider_name.into(),
url: url.clone(),
body,
body: serde_json::to_value(body).map_err(|_| request_error("body"))?,
})
.await?;
guarded.body
} else {
let body: OcrWireBody<<I::Format as OcrFormat>::RequestBody> =
OcrWireBody::decode(guarded.body)?;
backend.validate_request_body(&body.body)?;
body
} else {
OcrWireBody {
body,
extra: serde_json::Map::new(),
}
};
send_ocr_call(
client,
ReadyOcrCall {
integration: request.integration,
model: request.model,
params: request.params,
connection: request.connection,
url,
headers,
body,
},
)
.await
}
#[derive(serde::Serialize)]
struct OcrWireBody<B> {
#[serde(flatten)]
body: B,
#[serde(flatten)]
extra: serde_json::Map<String, serde_json::Value>,
}
impl<B: serde::Serialize + serde::de::DeserializeOwned> OcrWireBody<B> {
fn decode(value: serde_json::Value) -> Result<Self, Error> {
let body: B = super::wire::decode_request_value(value.clone(), "guardrail.body")?;
let serde_json::Value::Object(fields) = value else {
return Err(request_error("guardrail.body"));
};
let known = serde_json::to_value(&body).map_err(|_| request_error("guardrail.body"))?;
let extra = fields
.into_iter()
.filter(|(key, _)| known.get(key).is_none())
.collect();
Ok(Self { body, extra })
}
}
struct ReadyOcrCall<I: OcrIntegration> {
integration: I,
model: String,
params: MappedParams<I>,
connection: OcrConnection,
url: String,
headers: Vec<(String, String)>,
body: OcrWireBody<<I::Format as OcrFormat>::RequestBody>,
}
async fn send_ocr_call<I: OcrIntegration>(
client: &OcrClient,
request: ReadyOcrCall<I>,
) -> Result<OcrResponseData, Error> {
let ReadyOcrCall {
integration,
model,
params,
connection,
url,
headers,
body,
} = request;
let backend = &integration;
let format = integration.format();
let mut builder = client
.provider_http()
.post(&url)
.json(&body)
.timeout(request.connection.timeout);
.timeout(connection.timeout);
for (name, value) in &headers {
builder = builder.header(name, value);
}
@ -171,16 +233,9 @@ where
.await
.map_err(crate::error::TransportError::from)?;
let decoded = backend
.read_response(
client,
response,
&url,
&headers,
&request.connection,
&request.params,
)
.read_response(client, response, &url, &headers, &connection, &params)
.await?;
let response = format.transform_ocr_response(&request.model, decoded.data, &request.params)?;
let response = format.transform_ocr_response(&model, decoded.data, &params)?;
Ok(OcrResponseData {
provider_native_response: decoded.native,
..response

View file

@ -2,9 +2,9 @@ use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use super::backends::OcrBackend;
use super::backends::OcrIntegration;
use super::formats::OcrFormat;
use super::prepare::PreparedOcrRequest;
use super::prepare::MappedOcrRequest;
use super::types::{OcrDocument, OcrResponseData};
use crate::Error;
use crate::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming};
@ -64,35 +64,29 @@ pub trait OcrHooks: Send + Sync {
pub struct NoopOcrHooks;
impl OcrHooks for NoopOcrHooks {}
pub(crate) struct OcrLifecycleHooks<F, B>
pub(crate) struct OcrLifecycleHooks<I>
where
F: OcrFormat,
B: OcrBackend<F>,
I: OcrIntegration,
{
pub hooks: Arc<dyn OcrHooks>,
pub provider_name: String,
pub marker: std::marker::PhantomData<(F, B)>,
pub marker: std::marker::PhantomData<I>,
}
impl<F, B>
CallLifecycleHooks<
Result<PreparedOcrRequest<F, B>, Error>,
PreparedOcrRequest<F, B>,
OcrResponseData,
> for OcrLifecycleHooks<F, B>
impl<I> CallLifecycleHooks<Result<MappedOcrRequest<I>, Error>, MappedOcrRequest<I>, OcrResponseData>
for OcrLifecycleHooks<I>
where
F: OcrFormat,
B: OcrBackend<F>,
I: OcrIntegration,
{
type PreCallFuture<'a> = OcrHookFuture<'a, Result<PreparedOcrRequest<F, B>, Error>>;
type DuringCallFuture<'a> = OcrHookFuture<'a, PreparedOcrRequest<F, B>>;
type PreCallFuture<'a> = OcrHookFuture<'a, Result<MappedOcrRequest<I>, Error>>;
type DuringCallFuture<'a> = OcrHookFuture<'a, MappedOcrRequest<I>>;
type SuccessFuture<'a> = OcrLogFuture<'a>;
type FailureFuture<'a> = OcrLogFuture<'a>;
fn async_pre_call_hook<'a>(
&'a self,
_context: &'a CallLifecycleContext,
request: Result<PreparedOcrRequest<F, B>, Error>,
request: Result<MappedOcrRequest<I>, Error>,
) -> Self::PreCallFuture<'a> {
Box::pin(async move {
let request = request?;
@ -124,8 +118,8 @@ where
})?,
"guardrail.optional_params",
)
.and_then(|params| request.integration.format.map_ocr_params(params))?;
Ok(Ok(PreparedOcrRequest {
.and_then(|params| request.integration.format().map_ocr_params(params))?;
Ok(Ok(MappedOcrRequest {
document: changed.document,
params,
..request
@ -136,7 +130,7 @@ where
fn async_during_call_hook<'a>(
&'a self,
_context: &'a CallLifecycleContext,
request: Result<PreparedOcrRequest<F, B>, Error>,
request: Result<MappedOcrRequest<I>, Error>,
) -> Self::DuringCallFuture<'a> {
Box::pin(async move { request })
}

View file

@ -1,6 +1,6 @@
use super::backends::OcrBackend;
use super::backends::{HostConfig, InputParams, MappedParams, OcrIntegration};
use super::formats::OcrFormat;
use super::registry::OcrIntegration;
use super::types::{OcrConnection, OcrDocument};
use crate::Error;
@ -8,34 +8,32 @@ pub use super::registry::{
OcrIntegrationKind, OcrIntegrationRequest, OcrModel, OcrProvider, resolve_ocr_integration,
};
pub(crate) struct PreparedOcrRequest<F, B>
pub(crate) struct MappedOcrRequest<I>
where
F: OcrFormat,
B: OcrBackend<F>,
I: OcrIntegration,
{
pub integration: OcrIntegration<F, B>,
pub integration: I,
pub model: String,
pub document: OcrDocument,
pub params: F::MappedParams,
pub backend_config: B::Config,
pub params: MappedParams<I>,
pub backend_config: HostConfig<I>,
pub connection: OcrConnection,
}
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
pub(crate) fn prepare_ocr_call<F, B>(
integration: OcrIntegration<F, B>,
pub(crate) fn prepare_ocr_call<I>(
integration: I,
model: String,
document: OcrDocument,
params: F::InputParams,
backend_config: B::Config,
params: InputParams<I>,
backend_config: HostConfig<I>,
connection: OcrConnection,
) -> Result<PreparedOcrRequest<F, B>, Error>
) -> Result<MappedOcrRequest<I>, Error>
where
F: OcrFormat,
B: OcrBackend<F>,
I: OcrIntegration,
{
let params = integration.format.map_ocr_params(params)?;
Ok(PreparedOcrRequest {
let params = integration.format().map_ocr_params(params)?;
Ok(MappedOcrRequest {
integration,
model,
document,

View file

@ -6,57 +6,15 @@ use crate::auth::azure::AzureAuthInputs;
use crate::providers::vertex_ai::auth::VertexAuthInputs;
use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider};
use super::backends::OcrBackend;
use super::backends::azure_ai::{AzureDocumentIntelligenceOcrBackend, AzureMistralOcrBackend};
use super::backends::mistral::MistralOcrBackend;
use super::backends::reducto::ReductoOcrBackend;
use super::backends::vertex_ai::VertexAiOcrBackend;
use super::formats::OcrFormat;
use super::formats::deepseek::{DeepSeekOcrFormat, types::DeepSeekOcrParams};
use super::formats::document_intelligence::{
AzureDocumentIntelligenceOcrFormat, types::DocumentIntelligenceInputParams,
};
use super::formats::mistral::{MistralOcrFormat, types::MistralOcrParams};
use super::formats::reducto::{
ReductoParseLegacyFormat, ReductoParseV3Format,
types::{ReductoLegacyParams, ReductoV3Params},
};
pub use super::backends::azure_ai::{AzureDocumentIntelligence, AzureMistral};
pub use super::backends::mistral::MistralDirect;
pub use super::backends::reducto::{ReductoLegacy, ReductoV3};
pub use super::backends::vertex_ai::{VertexDeepSeek, VertexMistral};
use super::backends::{HostConfig, InputParams};
pub use super::backends::{OcrHost, OcrIntegration};
use super::types::OcrRequestFormat;
pub(crate) struct OcrIntegration<F, B>
where
F: OcrFormat,
B: OcrBackend<F>,
{
pub(crate) format: F,
pub(crate) backend: B,
}
impl<F, B> OcrIntegration<F, B>
where
F: OcrFormat,
B: OcrBackend<F>,
{
const fn new(format: F, backend: B) -> Self {
Self { format, backend }
}
pub(crate) fn decode_input_params(
&self,
params: Map<String, Value>,
prefix: &str,
) -> Result<F::InputParams, super::error::OcrRequestError> {
validate_request_format(
&params,
self.backend.supports_native_request_format(),
self.backend.provider_name(),
)?;
F::validate_input_params(&params)?;
super::wire::decode_request_value(Value::Object(params), prefix)
}
}
fn validate_request_format(
pub(crate) fn validate_request_format(
params: &Map<String, Value>,
supports_native: bool,
provider: &'static str,
@ -72,43 +30,55 @@ fn validate_request_format(
Ok(())
}
pub(crate) const MISTRAL: OcrIntegration<MistralOcrFormat, MistralOcrBackend> =
OcrIntegration::new(MistralOcrFormat, MistralOcrBackend);
pub(crate) const AZURE_MISTRAL: OcrIntegration<MistralOcrFormat, AzureMistralOcrBackend> =
OcrIntegration::new(MistralOcrFormat, AzureMistralOcrBackend);
pub(crate) const AZURE_DOCUMENT_INTELLIGENCE: OcrIntegration<
AzureDocumentIntelligenceOcrFormat,
AzureDocumentIntelligenceOcrBackend,
> = OcrIntegration::new(
AzureDocumentIntelligenceOcrFormat,
AzureDocumentIntelligenceOcrBackend,
);
pub(crate) const VERTEX_MISTRAL: OcrIntegration<MistralOcrFormat, VertexAiOcrBackend> =
OcrIntegration::new(MistralOcrFormat, VertexAiOcrBackend);
pub(crate) const VERTEX_DEEPSEEK: OcrIntegration<DeepSeekOcrFormat, VertexAiOcrBackend> =
OcrIntegration::new(DeepSeekOcrFormat, VertexAiOcrBackend);
pub(crate) const REDUCTO_V3: OcrIntegration<ReductoParseV3Format, ReductoOcrBackend> =
OcrIntegration::new(ReductoParseV3Format, ReductoOcrBackend);
pub(crate) const REDUCTO_LEGACY: OcrIntegration<ReductoParseLegacyFormat, ReductoOcrBackend> =
OcrIntegration::new(ReductoParseLegacyFormat, ReductoOcrBackend);
pub(crate) const MISTRAL: MistralDirect = MistralDirect;
pub(crate) const AZURE_MISTRAL: AzureMistral = AzureMistral;
pub(crate) const AZURE_DOCUMENT_INTELLIGENCE: AzureDocumentIntelligence = AzureDocumentIntelligence;
pub(crate) const VERTEX_MISTRAL: VertexMistral = VertexMistral;
pub(crate) const VERTEX_DEEPSEEK: VertexDeepSeek = VertexDeepSeek;
pub(crate) const REDUCTO_V3: ReductoV3 = ReductoV3;
pub(crate) const REDUCTO_LEGACY: ReductoLegacy = ReductoLegacy;
/// ```
/// use litellm_core::ocr::registry::{MistralDirect, OcrIntegrationInput};
/// use litellm_core::ocr::formats::mistral::types::MistralOcrParams;
/// let input = OcrIntegrationInput::<MistralDirect> {
/// params: MistralOcrParams::default(),
/// backend_config: (),
/// };
/// ```
///
/// ```compile_fail
/// use litellm_core::ocr::registry::{MistralDirect, OcrIntegrationInput};
/// use litellm_core::ocr::formats::deepseek::types::DeepSeekOcrParams;
/// let input = OcrIntegrationInput::<MistralDirect> {
/// params: DeepSeekOcrParams::default(),
/// backend_config: (),
/// };
/// ```
///
/// ```compile_fail
/// use litellm_core::ocr::registry::{VertexMistral, OcrIntegrationInput};
/// use litellm_core::ocr::formats::mistral::types::MistralOcrParams;
/// let input = OcrIntegrationInput::<VertexMistral> {
/// params: MistralOcrParams::default(),
/// backend_config: (),
/// };
/// ```
#[derive(Clone, Debug)]
pub struct OcrIntegrationInput<P, C> {
pub params: P,
pub backend_config: C,
pub struct OcrIntegrationInput<I: OcrIntegration> {
pub params: InputParams<I>,
pub backend_config: HostConfig<I>,
}
#[derive(Clone, Debug)]
pub enum OcrIntegrationRequest {
Mistral(OcrIntegrationInput<MistralOcrParams, ()>),
AzureMistral(OcrIntegrationInput<MistralOcrParams, AzureAuthInputs>),
AzureDocumentIntelligence(
OcrIntegrationInput<DocumentIntelligenceInputParams, AzureAuthInputs>,
),
VertexMistral(OcrIntegrationInput<MistralOcrParams, VertexAuthInputs>),
VertexDeepSeek(OcrIntegrationInput<DeepSeekOcrParams, VertexAuthInputs>),
ReductoV3(OcrIntegrationInput<ReductoV3Params, ()>),
ReductoLegacy(OcrIntegrationInput<ReductoLegacyParams, ()>),
Mistral(OcrIntegrationInput<MistralDirect>),
AzureMistral(OcrIntegrationInput<AzureMistral>),
AzureDocumentIntelligence(OcrIntegrationInput<AzureDocumentIntelligence>),
VertexMistral(OcrIntegrationInput<VertexMistral>),
VertexDeepSeek(OcrIntegrationInput<VertexDeepSeek>),
ReductoV3(OcrIntegrationInput<ReductoV3>),
ReductoLegacy(OcrIntegrationInput<ReductoLegacy>),
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]

View file

@ -1,4 +1,4 @@
use crate::ocr::backends::OcrBackend;
use crate::ocr::backends::OcrIntegration;
use crate::ocr::registry::{AZURE_MISTRAL, MISTRAL};
use crate::ocr::tests::body;
use crate::ocr::types::OcrConnection;
@ -23,7 +23,6 @@ async fn azure_ai_mistral_ocr_uses_generic_api_base() {
..Default::default()
};
let prepared = AZURE_MISTRAL
.backend
.prepare(
&connection,
&Default::default(),

View file

@ -1,4 +1,4 @@
use crate::ocr::backends::OcrBackend;
use crate::ocr::backends::OcrIntegration;
use crate::ocr::formats::OcrFormat;
use crate::ocr::prepare::OcrIntegrationKind;
use crate::ocr::registry::{AZURE_DOCUMENT_INTELLIGENCE, decode_integration_request};
@ -26,7 +26,6 @@ async fn prepared_url(
params: &<crate::ocr::formats::document_intelligence::AzureDocumentIntelligenceOcrFormat as OcrFormat>::MappedParams,
) -> Result<String, crate::ocr::error::OcrError> {
AZURE_DOCUMENT_INTELLIGENCE
.backend
.prepare(&connection(), &Default::default(), model, params, &|_| None)
.await
.map(|prepared| prepared.url)
@ -69,7 +68,7 @@ fn invalid_page_values_never_panic(#[case] pages: Value) {
let input = serde_json::from_value(json!({"pages":pages})).unwrap();
assert!(
AZURE_DOCUMENT_INTELLIGENCE
.format
.format()
.map_ocr_params(input)
.is_err()
);
@ -107,7 +106,7 @@ async fn document_intelligence_maps_features(#[case] features: Value, #[case] ex
fn document_intelligence_rejects_invalid_features(#[case] features: Value) {
assert!(
AZURE_DOCUMENT_INTELLIGENCE
.format
.format()
.map_ocr_params(serde_json::from_value(json!({"features":features})).unwrap())
.is_err()
);

View file

@ -3,9 +3,9 @@ use std::sync::{Arc, Mutex};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
use super::backends::OcrBackend;
use super::backends::{MappedParams, OcrIntegration};
use super::formats::OcrFormat;
use super::registry::OcrIntegration;
use super::types::OcrConnection;
use super::wire::{OcrWireRequest, decode_request, decode_response};
use super::{OcrClient, OcrRequest};
@ -25,55 +25,51 @@ pub(crate) async fn perform_ocr(
ocr_client().perform(request).await
}
pub(crate) fn params<F, B>(integration: &OcrIntegration<F, B>, value: Value) -> F::MappedParams
pub(crate) fn params<I>(integration: &I, value: Value) -> MappedParams<I>
where
F: OcrFormat,
B: OcrBackend<F>,
I: OcrIntegration,
{
integration
.format
.format()
.map_ocr_params(serde_json::from_value(value).unwrap())
.unwrap()
}
pub(crate) fn transform<F, B>(
integration: &OcrIntegration<F, B>,
pub(crate) fn transform<I>(
integration: &I,
model: &str,
value: Value,
options: Value,
) -> Result<Value, OcrError>
where
F: OcrFormat,
B: OcrBackend<F>,
I: OcrIntegration,
{
let params = integration
.format
.format()
.map_ocr_params(serde_json::from_value(options).unwrap())?;
let decoded = decode_response(
&serde_json::to_vec(&value).unwrap(),
integration.backend.preserve_native_response(&params),
integration.preserve_native_response(&params),
)?;
let mut response = integration
.format
.format()
.transform_ocr_response(model, decoded.data, &params)?;
response.provider_native_response = decoded.native;
Ok(response.into_json())
}
pub(crate) async fn body<F, B>(
integration: &OcrIntegration<F, B>,
pub(crate) async fn body<I>(
integration: &I,
model: &str,
document: Value,
options: Value,
) -> Result<Value, OcrError>
where
F: OcrFormat,
B: OcrBackend<F>,
I: OcrIntegration,
{
let params = integration
.format
.format()
.map_ocr_params(serde_json::from_value(options).unwrap())?;
let client = ocr_client();
let document = integration
.backend
.prepare_document(
&client,
serde_json::from_value(document).unwrap(),
@ -81,12 +77,14 @@ where
&[],
)
.await?;
Ok(serde_json::to_value(
integration
.format
.transform_ocr_request(model, document, &params)?,
Ok(
serde_json::to_value(integration.format().transform_ocr_request(
model,
document.into(),
&params,
)?)
.unwrap(),
)
.unwrap())
}
pub(crate) fn wire_request(model: &str, base: &str, options: Value) -> OcrRequest {
decode_request(OcrWireRequest {
@ -343,3 +341,45 @@ fn typed_response_rejects_malformed_pages_with_field_path() {
assert!(error.to_string().contains("pages[0].markdown"));
assert!(!error.to_string().contains("42"));
}
#[rstest::rstest]
#[case("mistral/model", json!({"model":"model","document":42}), "guardrail.body.document")]
#[case("azure_ai/model", json!({"model":"model","document":{"type":"document_url","document_url":"https://example.com/doc.pdf"}}), "data URI")]
#[case("vertex_ai/model", json!({"model":"model","document":{"type":"document_url","document_url":"data:application/pdf;base64,INVALID!"}}), "data URI")]
#[tokio::test]
async fn rejects_invalid_guardrail_body_before_sending(
#[case] model: &str,
#[case] body: Value,
#[case] expected: &str,
) {
use crate::ocr::hooks::{OcrDuringCallRequest, OcrHookFuture, OcrHooks};
struct Rewrite(Value);
impl OcrHooks for Rewrite {
fn has_guardrails(&self) -> bool {
true
}
fn during_call(
&self,
request: OcrDuringCallRequest,
) -> OcrHookFuture<'_, OcrDuringCallRequest> {
Box::pin(async move {
Ok(OcrDuringCallRequest {
body: self.0.clone(),
..request
})
})
}
}
let mut request = wire_request(
model,
"http://127.0.0.1:1",
json!({"vertex_project":"test-project"}),
);
request.hooks = Arc::new(Rewrite(body));
let error = perform_ocr(request).await.unwrap_err();
assert_eq!(error.kind(), crate::error::ErrorKind::InvalidRequest);
assert!(error.to_string().contains(expected), "{error}");
}

View file

@ -1,5 +1,4 @@
use crate::ocr::backends::OcrBackend;
use crate::ocr::formats::mistral::MistralOcrFormat;
use crate::ocr::backends::OcrIntegration;
use crate::ocr::registry::{MISTRAL, VERTEX_MISTRAL};
use crate::ocr::tests::body;
use crate::ocr::types::OcrConnection;
@ -35,8 +34,8 @@ async fn vertex_mistral_url_uses_project_location_and_model() {
.unwrap(),
)
.unwrap();
let prepared = OcrBackend::<MistralOcrFormat>::prepare(
&VERTEX_MISTRAL.backend,
let prepared = OcrIntegration::prepare(
&VERTEX_MISTRAL,
&connection,
&config,
"mistral-ocr-maas",
@ -57,8 +56,8 @@ async fn vertex_mistral_resolves_provider_environment_in_the_backend() {
api_key: Some("token".into()),
..Default::default()
};
let prepared = OcrBackend::<MistralOcrFormat>::prepare(
&VERTEX_MISTRAL.backend,
let prepared = OcrIntegration::prepare(
&VERTEX_MISTRAL,
&connection,
&Default::default(),
"model",