From 82e6caf87f5f101321af4751d296d383422dd46f Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 1 Sep 2026 12:25:10 -0700 Subject: [PATCH] feat(rust): scaffold OCR compiler contracts --- litellm-rust/Cargo.lock | 4 + litellm-rust/Cargo.toml | 3 + litellm-rust/crates/core/Cargo.toml | 3 + litellm-rust/crates/core/src/ocr/canonical.rs | 240 ++++++++++++++++++ litellm-rust/crates/core/src/ocr/compiler.rs | 201 +++++++++++++++ litellm-rust/crates/core/src/ocr/mod.rs | 5 + litellm-rust/crates/core/src/ocr/plan.rs | 50 ++++ litellm-rust/crates/core/src/ocr/policy.rs | 179 +++++++++++++ litellm-rust/crates/core/src/ocr/response.rs | 86 +++++++ litellm-rust/crates/core/src/ocr/types.rs | 28 ++ .../azure_ai/ocr/document_intelligence.rs | 26 ++ .../src/providers/azure_ai/ocr/foundry.rs | 17 ++ .../core/src/providers/azure_ai/ocr/mod.rs | 2 + .../src/providers/mistral/ocr/compiler.rs | 63 +++++ .../core/src/providers/mistral/ocr/mod.rs | 1 + litellm-rust/crates/core/src/providers/mod.rs | 1 + .../crates/core/src/providers/reducto/mod.rs | 1 + .../core/src/providers/reducto/ocr/legacy.rs | 24 ++ .../core/src/providers/reducto/ocr/mod.rs | 4 + .../src/providers/reducto/ocr/response.rs | 2 + .../core/src/providers/reducto/ocr/upload.rs | 10 + .../core/src/providers/reducto/ocr/v3.rs | 24 ++ .../src/providers/vertex_ai/ocr/deepseek.rs | 15 ++ .../src/providers/vertex_ai/ocr/mistral.rs | 17 ++ .../core/src/providers/vertex_ai/ocr/mod.rs | 2 + 25 files changed, 1008 insertions(+) create mode 100644 litellm-rust/crates/core/src/ocr/canonical.rs create mode 100644 litellm-rust/crates/core/src/ocr/compiler.rs create mode 100644 litellm-rust/crates/core/src/ocr/plan.rs create mode 100644 litellm-rust/crates/core/src/ocr/policy.rs create mode 100644 litellm-rust/crates/core/src/ocr/response.rs create mode 100644 litellm-rust/crates/core/src/providers/azure_ai/ocr/document_intelligence.rs create mode 100644 litellm-rust/crates/core/src/providers/azure_ai/ocr/foundry.rs create mode 100644 litellm-rust/crates/core/src/providers/mistral/ocr/compiler.rs create mode 100644 litellm-rust/crates/core/src/providers/reducto/mod.rs create mode 100644 litellm-rust/crates/core/src/providers/reducto/ocr/legacy.rs create mode 100644 litellm-rust/crates/core/src/providers/reducto/ocr/mod.rs create mode 100644 litellm-rust/crates/core/src/providers/reducto/ocr/response.rs create mode 100644 litellm-rust/crates/core/src/providers/reducto/ocr/upload.rs create mode 100644 litellm-rust/crates/core/src/providers/reducto/ocr/v3.rs create mode 100644 litellm-rust/crates/core/src/providers/vertex_ai/ocr/deepseek.rs create mode 100644 litellm-rust/crates/core/src/providers/vertex_ai/ocr/mistral.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 22f29f566c4..3ffe1e54476 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -1428,7 +1428,9 @@ dependencies = [ "aws-sigv4", "aws-smithy-runtime-api", "aws-types", + "bytes", "futures-util", + "mime", "rand 0.8.7", "reqwest", "serde", @@ -1436,6 +1438,7 @@ dependencies = [ "sha2 0.10.9", "thiserror 2.0.19", "tokio", + "url", ] [[package]] @@ -2720,6 +2723,7 @@ dependencies = [ "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index c447d915abe..631322f988c 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -33,6 +33,9 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"] tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] } futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] } base64 = "0.22" +bytes = "1" +mime = "0.3" +url = { version = "2", features = ["serde"] } [profile.release] opt-level = 3 diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index 2b7c76a9bec..46e22bc866a 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -7,11 +7,14 @@ repository.workspace = true [dependencies] futures-util.workspace = true +bytes.workspace = true +mime.workspace = true rand.workspace = true reqwest.workspace = true serde.workspace = true serde_json.workspace = true thiserror.workspace = true +url.workspace = true sha2.workspace = true aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"], optional = true } aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"], optional = true } diff --git a/litellm-rust/crates/core/src/ocr/canonical.rs b/litellm-rust/crates/core/src/ocr/canonical.rs new file mode 100644 index 00000000000..317573bb5fa --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/canonical.rs @@ -0,0 +1,240 @@ +use std::collections::BTreeMap; + +use bytes::Bytes; +use mime::Mime; +use serde_json::Value; +use url::Url; + +use super::compiler::CompileError; +use super::policy::OcrCanonicalField; +use super::types::{Field, OcrDialectId}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DocumentKind { + Image, + Pdf, +} + +#[derive(Clone, PartialEq)] +pub enum OcrDocument { + RemoteUrl { + kind: DocumentKind, + url: Url, + }, + Inline { + kind: DocumentKind, + media_type: Mime, + bytes: Bytes, + }, + ProviderReference { + provider: OcrDialectId, + id: String, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PageSelection(Vec); + +impl PageSelection { + pub fn new(pages: impl IntoIterator) -> Self { + Self(pages.into_iter().collect()) + } + + pub fn pages(&self) -> &[u32] { + &self.0 + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct AnnotationFormat(Value); + +impl AnnotationFormat { + pub fn new(schema: Value) -> Result { + if !schema.is_object() { + return Err(CompileError::InvalidParameter { + field: "annotation_format", + reason: "must be a JSON object", + }); + } + Ok(Self(schema)) + } + + pub fn as_value(&self) -> &Value { + &self.0 + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TableFormat { + Html, + Markdown, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ConfidenceScoresGranularity { + Page, + Word, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct OcrOutputOptions { + pub include_image_base64: Field, + pub image_limit: Field, + pub image_min_size: Field, + pub bbox_annotation_format: Field, + pub document_annotation_format: Field, + pub document_annotation_prompt: Field, + pub extract_header: Field, + pub extract_footer: Field, + pub table_format: Field, + pub confidence_scores_granularity: Field, + pub include_blocks: Field, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OcrRequestId(String); + +impl OcrRequestId { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + if value.trim().is_empty() { + return Err(CompileError::InvalidParameter { + field: "id", + reason: "must not be blank", + }); + } + Ok(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ExplicitProviderExtras { + dialect: OcrDialectId, + fields: BTreeMap, +} + +impl ExplicitProviderExtras { + pub fn try_new( + dialect: OcrDialectId, + fields: BTreeMap, + ) -> Result { + if let Some(field) = fields.keys().find(|field| { + OcrCanonicalField::from_wire_name(field).is_some() || is_litellm_control(field) + }) { + return Err(if OcrCanonicalField::from_wire_name(field).is_some() { + CompileError::ExtraCollidesWithCanonical(field.clone()) + } else { + CompileError::ReservedExtra(field.clone()) + }); + } + Ok(Self { dialect, fields }) + } + + pub fn dialect(&self) -> OcrDialectId { + self.dialect + } + + pub fn fields(&self) -> &BTreeMap { + &self.fields + } +} + +fn is_litellm_control(field: &str) -> bool { + field.starts_with("litellm_") + || matches!( + field, + "api_base" + | "api_key" + | "custom_llm_provider" + | "fallbacks" + | "metadata" + | "mock_response" + | "num_retries" + | "request_timeout" + | "retry_policy" + ) +} + +/// ```compile_fail +/// fn assert_serialize() {} +/// assert_serialize::(); +/// ``` +#[derive(Clone, PartialEq)] +pub struct CanonicalOcrRequest { + pub model: String, + pub document: OcrDocument, + pub pages: Field, + pub output: OcrOutputOptions, + pub request_id: Field, + pub provider_extras: ExplicitProviderExtras, +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn extras_reject_canonical_field_collisions() { + let error = ExplicitProviderExtras::try_new( + OcrDialectId::Mistral, + BTreeMap::from([("pages".to_string(), json!([0]))]), + ) + .expect_err("canonical fields cannot enter extras"); + + assert_eq!( + error, + CompileError::ExtraCollidesWithCanonical("pages".to_string()) + ); + } + + #[test] + fn extras_reject_litellm_controls() { + let error = ExplicitProviderExtras::try_new( + OcrDialectId::Mistral, + BTreeMap::from([("request_timeout".to_string(), json!(30))]), + ) + .expect_err("LiteLLM controls cannot enter extras"); + + assert_eq!( + error, + CompileError::ReservedExtra("request_timeout".to_string()) + ); + + let prefixed_error = ExplicitProviderExtras::try_new( + OcrDialectId::Mistral, + BTreeMap::from([("litellm_future_control".to_string(), json!(true))]), + ) + .expect_err("reserved LiteLLM prefixes cannot enter extras"); + + assert_eq!( + prefixed_error, + CompileError::ReservedExtra("litellm_future_control".to_string()) + ); + } + + #[test] + fn extras_remain_bound_to_one_dialect() { + let extras = ExplicitProviderExtras::try_new( + OcrDialectId::ReductoV3, + BTreeMap::from([("chunking".to_string(), json!({"size": 1}))]), + ) + .expect("provider field is accepted"); + + assert_eq!(extras.dialect(), OcrDialectId::ReductoV3); + assert_eq!(extras.fields()["chunking"], json!({"size": 1})); + } + + #[test] + fn canonical_fields_preserve_absent_null_and_value() { + assert_ne!(Field::::Absent, Field::Null); + assert_ne!( + Field::Null, + Field::Value(PageSelection::new([0_u32, 2_u32])) + ); + } +} diff --git a/litellm-rust/crates/core/src/ocr/compiler.rs b/litellm-rust/crates/core/src/ocr/compiler.rs new file mode 100644 index 00000000000..748b49877fb --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/compiler.rs @@ -0,0 +1,201 @@ +use std::collections::BTreeMap; + +use serde::Serialize; +use serde_json::Value; +use thiserror::Error; +use url::Url; + +use super::canonical::{CanonicalOcrRequest, DocumentKind}; +use super::plan::{CompletionPlan, DocumentPlan}; +use super::policy::OcrParameterPolicy; +use super::response::NormalizedOcr; +use super::types::OcrDialectId; + +#[derive(Clone, Debug, Error, PartialEq, Eq)] +pub enum CompileError { + #[error("invalid OCR parameter {field}: {reason}")] + InvalidParameter { + field: &'static str, + reason: &'static str, + }, + #[error("provider extra collides with canonical OCR field: {0}")] + ExtraCollidesWithCanonical(String), + #[error("LiteLLM control cannot enter OCR provider extras: {0}")] + ReservedExtra(String), + #[error("OCR dialect is not compiled yet: {0:?}")] + UnsupportedDialect(OcrDialectId), +} + +#[derive(Clone, Debug, Error, PartialEq, Eq)] +pub enum NormalizeError { + #[error("invalid terminal OCR response: {0}")] + InvalidPayload(&'static str), +} + +#[derive(Clone, Default, PartialEq, Eq)] +pub struct OcrCredentials { + api_key: Option, + oauth_token: Option, +} + +impl OcrCredentials { + pub fn new(api_key: Option, oauth_token: Option) -> Self { + Self { + api_key, + oauth_token, + } + } + + pub fn api_key(&self) -> Option<&str> { + self.api_key.as_deref() + } + + pub fn oauth_token(&self) -> Option<&str> { + self.oauth_token.as_deref() + } +} + +/// ```compile_fail +/// fn assert_serialize() {} +/// assert_serialize::(); +/// assert_serialize::(); +/// ``` +#[derive(Clone, PartialEq)] +pub struct ResolvedOcrTarget { + dialect: OcrDialectId, + api_base: Url, + credentials: OcrCredentials, +} + +impl ResolvedOcrTarget { + pub fn new(dialect: OcrDialectId, api_base: Url, credentials: OcrCredentials) -> Self { + Self { + dialect, + api_base, + credentials, + } + } + + pub fn dialect(&self) -> OcrDialectId { + self.dialect + } + + pub fn api_base(&self) -> &Url { + &self.api_base + } + + pub fn credentials(&self) -> &OcrCredentials { + &self.credentials + } +} + +#[derive(Clone, PartialEq)] +pub enum ProviderDocument { + RemoteUrl { + kind: DocumentKind, + url: Url, + }, + InlineDataUri { + kind: DocumentKind, + data_uri: String, + }, + Reference { + id: String, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "UPPERCASE")] +pub enum HttpMethod { + Get, + Post, +} + +#[derive(Clone, PartialEq, Serialize)] +#[serde(transparent)] +pub struct OcrWireBody(pub(crate) Value); + +impl OcrWireBody { + pub fn as_value(&self) -> &Value { + &self.0 + } +} + +#[derive(Clone, PartialEq, Serialize)] +pub struct CompiledHttpRequest { + pub method: HttpMethod, + pub url: Url, + pub headers: BTreeMap, + pub body: OcrWireBody, +} + +#[derive(Clone, PartialEq)] +pub struct ProviderPayload(Value); + +impl ProviderPayload { + pub fn new(value: Value) -> Self { + Self(value) + } + + pub fn as_value(&self) -> &Value { + &self.0 + } + + pub fn into_value(self) -> Value { + self.0 + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum OcrDocumentPolicy { + Ready, + FetchRemoteUrlAndInline, + UploadUnlessProviderReference, +} + +pub trait OcrDialectCompiler: Send + Sync { + fn parameter_policy(&self) -> &'static OcrParameterPolicy; + + fn prepare_document( + &self, + document: &super::canonical::OcrDocument, + target: &ResolvedOcrTarget, + ) -> Result; + + fn compile_submit( + &self, + request: &CanonicalOcrRequest, + document: ProviderDocument, + target: &ResolvedOcrTarget, + ) -> Result; + + fn completion_plan(&self) -> CompletionPlan; + + fn normalize( + &self, + terminal_response: ProviderPayload, + ) -> Result; +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn compiled_http_request_is_the_serializable_wire_boundary() { + let request = CompiledHttpRequest { + method: HttpMethod::Post, + url: Url::parse("https://example.com/ocr").expect("valid URL"), + headers: BTreeMap::from([("content-type".to_string(), "application/json".to_string())]), + body: OcrWireBody(json!({"model": "ocr-model"})), + }; + + let serialized = serde_json::to_value(request).expect("wire request serializes"); + + assert_eq!(serialized["method"], "POST"); + assert_eq!(serialized["url"], "https://example.com/ocr"); + assert_eq!(serialized["body"]["model"], "ocr-model"); + } +} diff --git a/litellm-rust/crates/core/src/ocr/mod.rs b/litellm-rust/crates/core/src/ocr/mod.rs index ec2fbb969a6..f013d9cbd30 100644 --- a/litellm-rust/crates/core/src/ocr/mod.rs +++ b/litellm-rust/crates/core/src/ocr/mod.rs @@ -1,2 +1,7 @@ +pub mod canonical; +pub mod compiler; +pub mod plan; +pub mod policy; +pub mod response; pub mod transformation; pub mod types; diff --git a/litellm-rust/crates/core/src/ocr/plan.rs b/litellm-rust/crates/core/src/ocr/plan.rs new file mode 100644 index 00000000000..40fb1d4b293 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/plan.rs @@ -0,0 +1,50 @@ +use std::time::Duration; + +use url::Url; + +use super::canonical::{DocumentKind, OcrDocument}; +use super::compiler::ProviderDocument; +use super::types::OcrDialectId; + +#[derive(Clone, PartialEq)] +pub enum DocumentPlan { + Ready(ProviderDocument), + FetchAndInline(FetchPlan), + Upload(UploadPlan), +} + +#[derive(Clone, PartialEq)] +pub struct FetchPlan { + pub kind: DocumentKind, + pub url: Url, + pub max_bytes: usize, +} + +#[derive(Clone, PartialEq)] +pub struct UploadPlan { + pub dialect: OcrDialectId, + pub document: OcrDocument, +} + +#[derive(Clone, PartialEq)] +pub enum CompletionPlan { + Immediate, + Poll(PollPlan), +} + +#[derive(Clone, PartialEq, Eq)] +pub struct PollPlan { + pub operation_location_header: &'static str, + pub interval: Duration, + pub timeout: Duration, +} + +/// ```compile_fail +/// fn assert_serialize() {} +/// assert_serialize::(); +/// assert_serialize::(); +/// assert_serialize::(); +/// assert_serialize::(); +/// assert_serialize::(); +/// ``` +const _: () = (); diff --git a/litellm-rust/crates/core/src/ocr/policy.rs b/litellm-rust/crates/core/src/ocr/policy.rs new file mode 100644 index 00000000000..96e54294ca7 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/policy.rs @@ -0,0 +1,179 @@ +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ParameterDisposition { + Forward, + Rename(&'static str), + Transform, + ConsumeAsConfiguration, + Reject, +} + +macro_rules! ocr_parameter_schema { + ($(($variant:ident, $field:ident, $wire_name:literal)),+ $(,)?) => { + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub enum OcrCanonicalField { + $($variant),+ + } + + impl OcrCanonicalField { + pub const ALL: [Self; ocr_parameter_schema!(@count $($variant),+)] = [ + $(Self::$variant),+ + ]; + + pub const fn wire_name(self) -> &'static str { + match self { + $(Self::$variant => $wire_name),+ + } + } + + pub fn from_wire_name(value: &str) -> Option { + match value { + $($wire_name => Some(Self::$variant)),+, + _ => None, + } + } + } + + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub struct OcrParameterPolicy { + $(pub $field: ParameterDisposition),+ + } + + impl OcrParameterPolicy { + pub const fn disposition(self, field: OcrCanonicalField) -> ParameterDisposition { + match field { + $(OcrCanonicalField::$variant => self.$field),+ + } + } + } + }; + (@count $($item:ident),+) => { + <[()]>::len(&[$(ocr_parameter_schema!(@replace $item ())),+]) + }; + (@replace $_item:ident $sub:expr) => { $sub }; +} + +ocr_parameter_schema!( + (Pages, pages, "pages"), + ( + IncludeImageBase64, + include_image_base64, + "include_image_base64" + ), + (ImageLimit, image_limit, "image_limit"), + (ImageMinSize, image_min_size, "image_min_size"), + ( + BboxAnnotationFormat, + bbox_annotation_format, + "bbox_annotation_format" + ), + ( + DocumentAnnotationFormat, + document_annotation_format, + "document_annotation_format" + ), + ( + DocumentAnnotationPrompt, + document_annotation_prompt, + "document_annotation_prompt" + ), + (ExtractHeader, extract_header, "extract_header"), + (ExtractFooter, extract_footer, "extract_footer"), + (TableFormat, table_format, "table_format"), + ( + ConfidenceScoresGranularity, + confidence_scores_granularity, + "confidence_scores_granularity" + ), + (IncludeBlocks, include_blocks, "include_blocks"), + (RequestId, request_id, "id"), +); + +pub const MISTRAL_OCR_PARAMETER_POLICY: OcrParameterPolicy = OcrParameterPolicy { + pages: ParameterDisposition::Forward, + include_image_base64: ParameterDisposition::Forward, + image_limit: ParameterDisposition::Forward, + image_min_size: ParameterDisposition::Forward, + bbox_annotation_format: ParameterDisposition::Forward, + document_annotation_format: ParameterDisposition::Forward, + document_annotation_prompt: ParameterDisposition::Forward, + extract_header: ParameterDisposition::Forward, + extract_footer: ParameterDisposition::Forward, + table_format: ParameterDisposition::Forward, + confidence_scores_granularity: ParameterDisposition::Forward, + include_blocks: ParameterDisposition::Forward, + request_id: ParameterDisposition::Forward, +}; + +pub const AZURE_DOCUMENT_INTELLIGENCE_PARAMETER_POLICY: OcrParameterPolicy = OcrParameterPolicy { + pages: ParameterDisposition::Transform, + include_image_base64: ParameterDisposition::Reject, + image_limit: ParameterDisposition::Reject, + image_min_size: ParameterDisposition::Reject, + bbox_annotation_format: ParameterDisposition::Reject, + document_annotation_format: ParameterDisposition::Reject, + document_annotation_prompt: ParameterDisposition::Reject, + extract_header: ParameterDisposition::Reject, + extract_footer: ParameterDisposition::Reject, + table_format: ParameterDisposition::Reject, + confidence_scores_granularity: ParameterDisposition::Reject, + include_blocks: ParameterDisposition::Reject, + request_id: ParameterDisposition::Reject, +}; + +pub const REJECT_CANONICAL_OCR_PARAMETER_POLICY: OcrParameterPolicy = OcrParameterPolicy { + pages: ParameterDisposition::Reject, + include_image_base64: ParameterDisposition::Reject, + image_limit: ParameterDisposition::Reject, + image_min_size: ParameterDisposition::Reject, + bbox_annotation_format: ParameterDisposition::Reject, + document_annotation_format: ParameterDisposition::Reject, + document_annotation_prompt: ParameterDisposition::Reject, + extract_header: ParameterDisposition::Reject, + extract_footer: ParameterDisposition::Reject, + table_format: ParameterDisposition::Reject, + confidence_scores_granularity: ParameterDisposition::Reject, + include_blocks: ParameterDisposition::Reject, + request_id: ParameterDisposition::Reject, +}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_canonical_field_has_a_mistral_disposition() { + let classified = OcrCanonicalField::ALL.map(|field| { + ( + field.wire_name(), + MISTRAL_OCR_PARAMETER_POLICY.disposition(field), + ) + }); + + assert_eq!(classified.len(), 13); + assert!( + classified + .iter() + .all(|(_, disposition)| *disposition == ParameterDisposition::Forward) + ); + } + + #[test] + fn every_canonical_field_has_an_explicit_non_mistral_disposition() { + let azure = OcrCanonicalField::ALL + .map(|field| AZURE_DOCUMENT_INTELLIGENCE_PARAMETER_POLICY.disposition(field)); + let provider_bound = OcrCanonicalField::ALL + .map(|field| REJECT_CANONICAL_OCR_PARAMETER_POLICY.disposition(field)); + + assert_eq!(azure[0], ParameterDisposition::Transform); + assert!( + azure[1..] + .iter() + .all(|disposition| *disposition == ParameterDisposition::Reject) + ); + assert!( + provider_bound + .iter() + .all(|disposition| *disposition == ParameterDisposition::Reject) + ); + } +} diff --git a/litellm-rust/crates/core/src/ocr/response.rs b/litellm-rust/crates/core/src/ocr/response.rs new file mode 100644 index 00000000000..4b29502f9df --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/response.rs @@ -0,0 +1,86 @@ +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OcrPage { + pub index: u32, + pub markdown: String, +} + +#[derive(Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct OcrUsage { + pub pages_processed: Option, + pub credits: Option, + pub document_size_bytes: Option, +} + +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct NormalizedOcr { + pub pages: Vec, + pub model: String, + pub document_annotation: Option, + pub content: Option, + pub tables: Option>, + pub key_value_pairs: Option>, + pub usage: OcrUsage, +} + +#[derive(Clone, PartialEq)] +pub struct NativeOcrPayload(Value); + +impl NativeOcrPayload { + pub fn new(value: Value) -> Self { + Self(value) + } + + pub fn as_value(&self) -> &Value { + &self.0 + } + + pub fn into_value(self) -> Value { + self.0 + } +} + +/// ```compile_fail +/// fn assert_serialize() {} +/// assert_serialize::(); +/// assert_serialize::(); +/// ``` +#[derive(Clone, PartialEq)] +pub struct OcrOutcome { + pub normalized: NormalizedOcr, + pub native: NativeOcrPayload, +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn normalized_serialization_cannot_include_native_payload() { + let outcome = OcrOutcome { + normalized: NormalizedOcr { + pages: vec![OcrPage { + index: 0, + markdown: "portable".to_string(), + }], + model: "ocr-model".to_string(), + document_annotation: None, + content: None, + tables: None, + key_value_pairs: None, + usage: OcrUsage::default(), + }, + native: NativeOcrPayload::new(json!({"provider_secret_field": "native"})), + }; + + let public = + serde_json::to_value(&outcome.normalized).expect("normalized output serializes"); + + assert_eq!(public["pages"][0]["markdown"], "portable"); + assert!(public.get("provider_secret_field").is_none()); + assert_eq!(outcome.native.as_value()["provider_secret_field"], "native"); + } +} diff --git a/litellm-rust/crates/core/src/ocr/types.rs b/litellm-rust/crates/core/src/ocr/types.rs index 1a72b8f1d66..e5ed78987e8 100644 --- a/litellm-rust/crates/core/src/ocr/types.rs +++ b/litellm-rust/crates/core/src/ocr/types.rs @@ -1,6 +1,34 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Field { + Absent, + Null, + Value(T), +} + +impl Field { + pub fn as_ref(&self) -> Field<&T> { + match self { + Self::Absent => Field::Absent, + Self::Null => Field::Null, + Self::Value(value) => Field::Value(value), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum OcrDialectId { + Mistral, + AzureFoundryMistral, + AzureDocumentIntelligence, + VertexMistral, + VertexDeepSeek, + ReductoV3, + ReductoLegacy, +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct OcrRequestData { pub data: Value, diff --git a/litellm-rust/crates/core/src/providers/azure_ai/ocr/document_intelligence.rs b/litellm-rust/crates/core/src/providers/azure_ai/ocr/document_intelligence.rs new file mode 100644 index 00000000000..fac32f5510e --- /dev/null +++ b/litellm-rust/crates/core/src/providers/azure_ai/ocr/document_intelligence.rs @@ -0,0 +1,26 @@ +use std::time::Duration; + +use crate::ocr::compiler::OcrDocumentPolicy; +use crate::ocr::plan::{CompletionPlan, PollPlan}; +use crate::ocr::policy::{AZURE_DOCUMENT_INTELLIGENCE_PARAMETER_POLICY, OcrParameterPolicy}; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct AzureDocumentIntelligenceOcrCompiler; + +impl AzureDocumentIntelligenceOcrCompiler { + pub fn parameter_policy(&self) -> &'static OcrParameterPolicy { + &AZURE_DOCUMENT_INTELLIGENCE_PARAMETER_POLICY + } + + pub fn document_policy(&self) -> OcrDocumentPolicy { + OcrDocumentPolicy::Ready + } + + pub fn completion_plan(&self, interval: Duration, timeout: Duration) -> CompletionPlan { + CompletionPlan::Poll(PollPlan { + operation_location_header: "operation-location", + interval, + timeout, + }) + } +} diff --git a/litellm-rust/crates/core/src/providers/azure_ai/ocr/foundry.rs b/litellm-rust/crates/core/src/providers/azure_ai/ocr/foundry.rs new file mode 100644 index 00000000000..240acb4d21c --- /dev/null +++ b/litellm-rust/crates/core/src/providers/azure_ai/ocr/foundry.rs @@ -0,0 +1,17 @@ +use crate::ocr::compiler::OcrDocumentPolicy; +use crate::providers::mistral::ocr::compiler::MistralOcrCodec; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct AzureFoundryMistralOcrCompiler { + codec: MistralOcrCodec, +} + +impl AzureFoundryMistralOcrCompiler { + pub fn codec(&self) -> &MistralOcrCodec { + &self.codec + } + + pub fn document_policy(&self) -> OcrDocumentPolicy { + OcrDocumentPolicy::FetchRemoteUrlAndInline + } +} diff --git a/litellm-rust/crates/core/src/providers/azure_ai/ocr/mod.rs b/litellm-rust/crates/core/src/providers/azure_ai/ocr/mod.rs index f239b6921fa..ddaa2b35053 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/ocr/mod.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/ocr/mod.rs @@ -1 +1,3 @@ +pub mod document_intelligence; +pub mod foundry; pub mod transformation; diff --git a/litellm-rust/crates/core/src/providers/mistral/ocr/compiler.rs b/litellm-rust/crates/core/src/providers/mistral/ocr/compiler.rs new file mode 100644 index 00000000000..1f4aadef15c --- /dev/null +++ b/litellm-rust/crates/core/src/providers/mistral/ocr/compiler.rs @@ -0,0 +1,63 @@ +use crate::ocr::compiler::OcrDocumentPolicy; +use crate::ocr::policy::{MISTRAL_OCR_PARAMETER_POLICY, OcrParameterPolicy}; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct MistralOcrCodec; + +impl MistralOcrCodec { + pub fn parameter_policy(&self) -> &'static OcrParameterPolicy { + &MISTRAL_OCR_PARAMETER_POLICY + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct MistralOcrCompiler { + codec: MistralOcrCodec, +} + +impl MistralOcrCompiler { + pub fn codec(&self) -> &MistralOcrCodec { + &self.codec + } + + pub fn document_policy(&self) -> OcrDocumentPolicy { + OcrDocumentPolicy::Ready + } +} + +#[cfg(test)] +mod tests { + use crate::ocr::compiler::OcrDocumentPolicy; + use crate::ocr::policy::OcrCanonicalField; + use crate::providers::azure_ai::ocr::foundry::AzureFoundryMistralOcrCompiler; + use crate::providers::vertex_ai::ocr::mistral::VertexMistralOcrCompiler; + + use super::*; + + #[test] + fn mistral_compatible_dialects_share_the_codec_policy() { + let mistral = MistralOcrCompiler::default(); + let foundry = AzureFoundryMistralOcrCompiler::default(); + let vertex = VertexMistralOcrCompiler::default(); + + for field in OcrCanonicalField::ALL { + assert_eq!( + mistral.codec().parameter_policy().disposition(field), + foundry.codec().parameter_policy().disposition(field) + ); + assert_eq!( + mistral.codec().parameter_policy().disposition(field), + vertex.codec().parameter_policy().disposition(field) + ); + } + assert_eq!(mistral.document_policy(), OcrDocumentPolicy::Ready); + assert_eq!( + foundry.document_policy(), + OcrDocumentPolicy::FetchRemoteUrlAndInline + ); + assert_eq!( + vertex.document_policy(), + OcrDocumentPolicy::FetchRemoteUrlAndInline + ); + } +} diff --git a/litellm-rust/crates/core/src/providers/mistral/ocr/mod.rs b/litellm-rust/crates/core/src/providers/mistral/ocr/mod.rs index f239b6921fa..7d70fb17286 100644 --- a/litellm-rust/crates/core/src/providers/mistral/ocr/mod.rs +++ b/litellm-rust/crates/core/src/providers/mistral/ocr/mod.rs @@ -1 +1,2 @@ +pub mod compiler; pub mod transformation; diff --git a/litellm-rust/crates/core/src/providers/mod.rs b/litellm-rust/crates/core/src/providers/mod.rs index 805600d6dbe..c0c2c69831b 100644 --- a/litellm-rust/crates/core/src/providers/mod.rs +++ b/litellm-rust/crates/core/src/providers/mod.rs @@ -4,4 +4,5 @@ pub mod azure_ai; pub mod bedrock; pub mod mistral; pub mod openai; +pub mod reducto; pub mod vertex_ai; diff --git a/litellm-rust/crates/core/src/providers/reducto/mod.rs b/litellm-rust/crates/core/src/providers/reducto/mod.rs new file mode 100644 index 00000000000..3621ff6a2fd --- /dev/null +++ b/litellm-rust/crates/core/src/providers/reducto/mod.rs @@ -0,0 +1 @@ +pub mod ocr; diff --git a/litellm-rust/crates/core/src/providers/reducto/ocr/legacy.rs b/litellm-rust/crates/core/src/providers/reducto/ocr/legacy.rs new file mode 100644 index 00000000000..0b16db192b4 --- /dev/null +++ b/litellm-rust/crates/core/src/providers/reducto/ocr/legacy.rs @@ -0,0 +1,24 @@ +use crate::ocr::policy::{OcrParameterPolicy, REJECT_CANONICAL_OCR_PARAMETER_POLICY}; + +use super::response::ReductoOcrResponseNormalizer; +use super::upload::ReductoDocumentAdapter; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ReductoLegacyOcrCompiler { + document_adapter: ReductoDocumentAdapter, + response_normalizer: ReductoOcrResponseNormalizer, +} + +impl ReductoLegacyOcrCompiler { + pub fn parameter_policy(&self) -> &'static OcrParameterPolicy { + &REJECT_CANONICAL_OCR_PARAMETER_POLICY + } + + pub fn document_adapter(&self) -> &ReductoDocumentAdapter { + &self.document_adapter + } + + pub fn response_normalizer(&self) -> &ReductoOcrResponseNormalizer { + &self.response_normalizer + } +} diff --git a/litellm-rust/crates/core/src/providers/reducto/ocr/mod.rs b/litellm-rust/crates/core/src/providers/reducto/ocr/mod.rs new file mode 100644 index 00000000000..b5ce96f4ada --- /dev/null +++ b/litellm-rust/crates/core/src/providers/reducto/ocr/mod.rs @@ -0,0 +1,4 @@ +pub mod legacy; +pub mod response; +pub mod upload; +pub mod v3; diff --git a/litellm-rust/crates/core/src/providers/reducto/ocr/response.rs b/litellm-rust/crates/core/src/providers/reducto/ocr/response.rs new file mode 100644 index 00000000000..af5e5295248 --- /dev/null +++ b/litellm-rust/crates/core/src/providers/reducto/ocr/response.rs @@ -0,0 +1,2 @@ +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ReductoOcrResponseNormalizer; diff --git a/litellm-rust/crates/core/src/providers/reducto/ocr/upload.rs b/litellm-rust/crates/core/src/providers/reducto/ocr/upload.rs new file mode 100644 index 00000000000..c02a2a8df6c --- /dev/null +++ b/litellm-rust/crates/core/src/providers/reducto/ocr/upload.rs @@ -0,0 +1,10 @@ +use crate::ocr::compiler::OcrDocumentPolicy; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ReductoDocumentAdapter; + +impl ReductoDocumentAdapter { + pub fn document_policy(&self) -> OcrDocumentPolicy { + OcrDocumentPolicy::UploadUnlessProviderReference + } +} diff --git a/litellm-rust/crates/core/src/providers/reducto/ocr/v3.rs b/litellm-rust/crates/core/src/providers/reducto/ocr/v3.rs new file mode 100644 index 00000000000..88f9c529234 --- /dev/null +++ b/litellm-rust/crates/core/src/providers/reducto/ocr/v3.rs @@ -0,0 +1,24 @@ +use crate::ocr::policy::{OcrParameterPolicy, REJECT_CANONICAL_OCR_PARAMETER_POLICY}; + +use super::response::ReductoOcrResponseNormalizer; +use super::upload::ReductoDocumentAdapter; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ReductoV3OcrCompiler { + document_adapter: ReductoDocumentAdapter, + response_normalizer: ReductoOcrResponseNormalizer, +} + +impl ReductoV3OcrCompiler { + pub fn parameter_policy(&self) -> &'static OcrParameterPolicy { + &REJECT_CANONICAL_OCR_PARAMETER_POLICY + } + + pub fn document_adapter(&self) -> &ReductoDocumentAdapter { + &self.document_adapter + } + + pub fn response_normalizer(&self) -> &ReductoOcrResponseNormalizer { + &self.response_normalizer + } +} diff --git a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/deepseek.rs b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/deepseek.rs new file mode 100644 index 00000000000..9b78f4fb5a0 --- /dev/null +++ b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/deepseek.rs @@ -0,0 +1,15 @@ +use crate::ocr::compiler::OcrDocumentPolicy; +use crate::ocr::policy::{OcrParameterPolicy, REJECT_CANONICAL_OCR_PARAMETER_POLICY}; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct VertexDeepSeekOcrCompiler; + +impl VertexDeepSeekOcrCompiler { + pub fn parameter_policy(&self) -> &'static OcrParameterPolicy { + &REJECT_CANONICAL_OCR_PARAMETER_POLICY + } + + pub fn document_policy(&self) -> OcrDocumentPolicy { + OcrDocumentPolicy::Ready + } +} diff --git a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/mistral.rs b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/mistral.rs new file mode 100644 index 00000000000..2b392030eba --- /dev/null +++ b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/mistral.rs @@ -0,0 +1,17 @@ +use crate::ocr::compiler::OcrDocumentPolicy; +use crate::providers::mistral::ocr::compiler::MistralOcrCodec; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct VertexMistralOcrCompiler { + codec: MistralOcrCodec, +} + +impl VertexMistralOcrCompiler { + pub fn codec(&self) -> &MistralOcrCodec { + &self.codec + } + + pub fn document_policy(&self) -> OcrDocumentPolicy { + OcrDocumentPolicy::FetchRemoteUrlAndInline + } +} diff --git a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/mod.rs b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/mod.rs index f239b6921fa..8fe358c8d4c 100644 --- a/litellm-rust/crates/core/src/providers/vertex_ai/ocr/mod.rs +++ b/litellm-rust/crates/core/src/providers/vertex_ai/ocr/mod.rs @@ -1 +1,3 @@ +pub mod deepseek; +pub mod mistral; pub mod transformation;