scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-06 02:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

refactor(ocr): remove the Python OCR execution path and require the Rust route (#43081)

Browse source 
* refactor(ocr): remove the Python OCR execution path and require the Rust route

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fmt

* refactor(ocr): tidy the native OCR passthrough binding

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* style(ocr): ruff format the azure passthrough transformation

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(ocr): resolve passthrough OCR costing in one Rust call

Replace passthrough_url/passthrough_transform with passthrough_response,
which matches the relayed endpoint against each Azure config's path
segments instead of building a fake request to call get_complete_url.
The binding drops the unused headers, status and api_base arguments.

Catch the ValueError/RuntimeError the binding raises so a relayed body
that is not OCR-shaped falls back to the passthrough object instead of
failing logging, and cover the relay against the real binding.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(ocr): drop the unused LlmProviders import from health check helpers

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* ci: drop the ocr_testing job now that tests/ocr_tests is gone

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test(ocr): restore the live OCR matrix and the ocr_testing job

The public litellm.ocr / aocr / Router interface is unchanged by the Rust
migration, so the live provider matrix still applies. Drops the stale VCR skip
list for the deleted test_rust_bridge.py.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test(ocr): import Final in the health check helper tests

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Yujong Lee <yujong@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
devin-ai-integration[bot] 2026-09-24 18:18:50 -07:00 • committed by GitHub
parent f1ef7fc0c2
commit e2302be068
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
252 changed files with 495 additions and 66994 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

100
litellm-rust/crates/core/src/ocr/provider_config.rs
View file

@ -5,15 +5,15 @@ use litellm_llms::{
transformation::TextractDetectTextConfig,
},
azure_ai::ocr::{
cohere_parse_transformation::AzureAICohereParseConfig,
cohere_parse_transformation::{AZURE_COHERE_PARSE_PATH, AzureAICohereParseConfig},
document_intelligence::transformation::AzureDocumentIntelligenceOcrConfig,
transformation::AzureAiOcrConfig,
transformation::{AZURE_AI_OCR_PATH, AzureAiOcrConfig},
},
base_llm::ocr::{
error::Error,
handler::{self, CallHooks, OcrClient},
transformation::{
BaseOcrConfig, LiteLLMOcrResponse, OcrCredentialInputs, OcrDocument,
BaseOcrConfig, LiteLLMOcrResponse, OcrCredentialInputs, OcrDocument, OcrResponseFormat,
PreparedOcrRequest, ResolvedOcrCredentials,
},
},
@ -157,6 +157,36 @@ pub fn get_health_check_document(
.get_health_check_document())
}
/// Normalize a relayed Azure AI response into the LiteLLM OCR shape when
/// `endpoint` is the OCR route of the model's resolved config.
pub fn passthrough_response(
model: &str,
endpoint: &str,
body: &[u8],
) -> Result<Option<LiteLLMOcrResponse>, Error> {
let (model, config) = resolve_provider_config(model, Some("azure_ai"))?;
let segments: Vec<&str> = endpoint
.split('/')
.filter(|segment| !segment.is_empty())
.collect();
let is_ocr_endpoint = match config {
OcrConfigKind::AzureAi => segments == AZURE_AI_OCR_PATH,
OcrConfigKind::AzureCohere => segments == AZURE_COHERE_PARSE_PATH,
OcrConfigKind::AzureDocumentIntelligence => {
segments == AzureDocumentIntelligenceOcrConfig::analyze_path(&model)?
}
other => {
let provider: &'static str = other.provider().into();
return Err(Error::InvalidProvider(provider.to_owned()));
}
};
if !is_ocr_endpoint {
return Ok(None);
}
with_config!(config, config => config.transform_ocr_response(&model, body, OcrResponseFormat::Litellm))
.map(Some)
}
#[derive(Clone, Copy, Debug, EnumString, IntoStaticStr, PartialEq, Eq)]
#[strum(serialize_all = "snake_case")]
pub(crate) enum OcrProvider {
@ -520,4 +550,68 @@ mod tests {
assert!(matches!(&error, Error::InvalidProvider(provider) if provider == "not_a_provider"));
assert_eq!(error.http_status_code(), Some(400));
}
#[rstest]
#[case("azure_ai/mistral-document-ai-2512", "providers/mistral/azure/ocr")]
#[case("azure_ai/mistral-document-ai-2512", "/providers/mistral/azure/ocr/")]
#[case("azure_ai/Cohere-parse-v5", "providers/cohere/v2/parse")]
#[case(
"azure_ai/doc-intelligence/prebuilt-layout",
"documentintelligence/documentModels/prebuilt-layout:analyze"
)]
fn passthrough_response_recognizes_the_resolved_config_ocr_route(
#[case] model: &str,
#[case] endpoint: &str,
) {
assert!(passthrough_response(model, endpoint, b"not json").is_err());
}
#[rstest]
#[case("azure_ai/mistral-document-ai-2512", "models/info")]
#[case("azure_ai/mistral-document-ai-2512", "providers/cohere/v2/parse")]
#[case("azure_ai/Cohere-parse-v5", "providers/mistral/azure/ocr")]
#[case(
"azure_ai/doc-intelligence/prebuilt-layout",
"documentintelligence/documentModels/prebuilt-read:analyze"
)]
fn passthrough_response_skips_other_routes(#[case] model: &str, #[case] endpoint: &str) {
assert!(
passthrough_response(model, endpoint, b"not json")
.unwrap()
.is_none()
);
}
#[test]
fn passthrough_response_normalizes_the_mistral_body() {
let body = br#"{
"pages": [{"index": 0, "markdown": "page one"}, {"index": 1, "markdown": "page two"}],
"model": "mistral-document-ai-2512",
"usage_info": {"pages_processed": 2}
}"#;
let json = passthrough_response(
"azure_ai/mistral-document-ai-2512",
"providers/mistral/azure/ocr",
body,
)
.unwrap()
.unwrap()
.into_json();
assert_eq!(json["usage_info"]["pages_processed"], 2);
assert_eq!(json["pages"][0]["markdown"], "page one");
}
#[test]
fn passthrough_response_counts_cohere_billed_pages() {
let body = br#"{"id": "parse-1", "pages": [], "meta": {"billed_units": {"pages": 3}}}"#;
let json = passthrough_response(
"azure_ai/Cohere-parse-v5",
"providers/cohere/v2/parse",
body,
)
.unwrap()
.unwrap()
.into_json();
assert_eq!(json["usage_info"]["pages_processed"], 3);
}
}

4
litellm-rust/crates/llms/src/azure_ai/ocr/cohere_parse_transformation.rs
View file

@ -16,6 +16,8 @@ use crate::{
},
};
pub const AZURE_COHERE_PARSE_PATH: [&str; 4] = ["providers", "cohere", "v2", "parse"];
#[derive(Default)]
pub struct AzureAICohereParseConfig;
@ -131,7 +133,7 @@ impl AzureAICohereParseConfig {
}
url.set_path(path.strip_suffix("/models").unwrap_or(&path));
ApiUrl::parse(url.as_str())
.and_then(|url| url.complete_path(&["providers", "cohere", "v2", "parse"]))
.and_then(|url| url.complete_path(&AZURE_COHERE_PARSE_PATH))
.map(|url| url.into_string())
.map_err(|_| invalid_api_base())
}

12
litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs
View file

@ -561,6 +561,14 @@ async fn poll_operation(
}
impl AzureDocumentIntelligenceOcrConfig {
pub fn analyze_path(model: &str) -> Result<[String; 3], Error> {
Ok([
"documentintelligence".into(),
"documentModels".into(),
format!("{}:analyze", model_id(model)?),
])
}
fn build_ocr_url(
&self,
endpoint: &str,
@ -568,9 +576,9 @@ impl AzureDocumentIntelligenceOcrConfig {
params: &DocumentIntelligenceParams,
api_version: &str,
) -> Result<String, Error> {
let model = format!("{}:analyze", model_id(model)?);
let path = Self::analyze_path(model)?;
ApiUrl::parse(endpoint)
.and_then(|url| url.complete_path(&["documentintelligence", "documentModels", &model]))
.and_then(|url| url.complete_path(&path.each_ref().map(String::as_str)))
.map(|url| {
url.append_query_pairs(
[("api-version", api_version)]

5
litellm-rust/crates/llms/src/azure_ai/ocr/transformation.rs
View file

@ -17,7 +17,7 @@ use crate::{
mistral::ocr::transformation::{MistralOcrConfig, MistralOcrRequest},
};
const AZURE_AI_OCR_PATH: &str = "/providers/mistral/azure/ocr";
pub const AZURE_AI_OCR_PATH: [&str; 4] = ["providers", "mistral", "azure", "ocr"];
const AZURE_AI_API_KEY_ENV: &str = "AZURE_AI_API_KEY";
const AZURE_AI_API_BASE_ENV: &str = "AZURE_AI_API_BASE";
@ -179,9 +179,8 @@ impl AzureAiOcrConfig {
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> Result<String, Error> {
let base = Self::resolve_api_base(api_base, env_lookup)?;
let path: Vec<&str> = AZURE_AI_OCR_PATH.trim_matches('/').split('/').collect();
ApiUrl::parse(&base)
.and_then(|url| url.complete_path(&path))
.and_then(|url| url.complete_path(&AZURE_AI_OCR_PATH))
.map(|url| url.into_string())
.map_err(|_| Error::RequestField {
path: "api_base".into(),

4
litellm-rust/crates/python-bridge/src/lib.rs
View file

@ -34,7 +34,7 @@ mod _native {
#[pymodule_export]
use crate::routes::messages::{amessages, messages};
#[pymodule_export]
use crate::routes::ocr::{aocr, ocr};
use crate::routes::ocr::{aocr, ocr, ocr_health_check_document, ocr_passthrough_response};
#[pymodule_export]
use crate::routes::responses::{ResponsesWebSocketConnection, aresponses, responses};
#[pymodule_export]
@ -85,6 +85,8 @@ mod tests {
"ProcessReservedForForking",
"ocr",
"aocr",
"ocr_health_check_document",
"ocr_passthrough_response",
"embedding",
"aembedding",
"transcription",

27
litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs
View file

@ -8,8 +8,9 @@ use std::sync::LazyLock;
use host::OcrRouteHost;
use litellm_auth_gcp::VertexAuth;
use litellm_callbacks_legacy_python::{LegacySurface, PublicCall, run_legacy_call};
use litellm_core::ocr::route::ocr_machine;
use litellm_core::ocr::{provider_config, route::ocr_machine};
use litellm_core_utils::settings::ProcessEnvironment;
use litellm_host_python::to_py;
use litellm_llms::base_llm::ocr::{handler::OcrClient, settings::OcrSettings};
use pyo3::{
prelude::*,
@ -106,6 +107,30 @@ pub(crate) fn aocr(
run_ocr(py, request, args, kwargs, true)
}
#[pyfunction]
pub(crate) fn ocr_health_check_document(
py: Python<'_>,
model: &str,
custom_llm_provider: Option<&str>,
) -> PyResult<Py<PyAny>> {
let document = provider_config::get_health_check_document(model, custom_llm_provider)
.map_err(errors::to_pyerr)?;
to_py(py, &document)
}
#[pyfunction]
pub(crate) fn ocr_passthrough_response(
py: Python<'_>,
model: &str,
endpoint: &str,
body: &[u8],
) -> PyResult<Option<Py<PyAny>>> {
provider_config::passthrough_response(model, endpoint, body)
.map_err(errors::to_pyerr)?
.map(|response| to_py(py, &response.into_json()))
.transpose()
}
#[cfg(test)]
mod tests {
use pyo3::prelude::*;

4
litellm/_lazy_imports_registry.py
View file

@ -394,12 +394,10 @@ UTILS_MODULE_NAMES: Final = (
"redact_message_input_output_from_logging",
"CustomStreamWrapper",
"BaseGoogleGenAIGenerateContentConfig",
"BaseOCRConfig",
"BaseSearchConfig",
"BaseTextToSpeechConfig",
"BedrockModelInfo",
"CohereModelInfo",
"MistralOCRConfig",
"Rules",
"AsyncHTTPHandler",
"HTTPHandler",
@ -1367,7 +1365,6 @@ _UTILS_MODULE_IMPORT_MAP: Final = {
"litellm.llms.base_llm.google_genai.transformation",
"BaseGoogleGenAIGenerateContentConfig",
),
"BaseOCRConfig": ("litellm.llms.base_llm.ocr.transformation", "BaseOCRConfig"),
"BaseSearchConfig": (
"litellm.llms.base_llm.search.transformation",
"BaseSearchConfig",
@ -1378,7 +1375,6 @@ _UTILS_MODULE_IMPORT_MAP: Final = {
),
"BedrockModelInfo": ("litellm.llms.bedrock.common_utils", "BedrockModelInfo"),
"CohereModelInfo": ("litellm.llms.cohere.common_utils", "CohereModelInfo"),
"MistralOCRConfig": ("litellm.llms.mistral.ocr.transformation", "MistralOCRConfig"),
"Rules": ("litellm.litellm_core_utils.rules", "Rules"),
"AsyncHTTPHandler": ("litellm.llms.custom_httpx.http_handler", "AsyncHTTPHandler"),
"HTTPHandler": ("litellm.llms.custom_httpx.http_handler", "HTTPHandler"),

17
litellm/litellm_core_utils/health_check_helpers.py
View file

@ -6,8 +6,10 @@ import base64
from collections.abc import Awaitable, Callable
from typing import TYPE_CHECKING, Final, Literal
from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, DocumentType
from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS, LlmProviders
from litellm.llms.base_llm.ocr.transformation import DocumentType
from litellm.rust_bridge import runtime
from litellm.rust_bridge.ocr.entrypoints import NATIVE_OCR_HEALTH_CHECK_DOCUMENT
from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging
@ -29,11 +31,12 @@ def get_image_file_for_health_check() -> bytes:
def _ocr_health_check_document(model: str, custom_llm_provider: str) -> DocumentType:
from litellm.utils import ProviderConfigManager
provider: Final = next((known for known in LlmProviders if known.value == custom_llm_provider), None)
config: Final = ProviderConfigManager.get_provider_ocr_config(model=model, provider=provider) if provider else None
return (config or BaseOCRConfig()).get_health_check_document()
native: Final = NATIVE_OCR_HEALTH_CHECK_DOCUMENT.load()
if native is None:
raise runtime.NoPythonImplementationError(
"ocr health check documents are resolved by the Rust extension, which is not available"
)
return native(model, custom_llm_provider)
class HealthCheckHelpers:

15
litellm/llms/azure_ai/ocr/__init__.py
View file

@ -1,15 +0,0 @@
"""Azure AI OCR module."""
from .cohere_parse_transformation import AzureAICohereParseConfig
from .common_utils import get_azure_ai_ocr_config
from .document_intelligence.transformation import (
AzureDocumentIntelligenceOCRConfig,
)
from .transformation import AzureAIOCRConfig
__all__ = [
"AzureAICohereParseConfig",
"AzureAIOCRConfig",
"AzureDocumentIntelligenceOCRConfig",
"get_azure_ai_ocr_config",
]

91
litellm/llms/azure_ai/ocr/cohere_parse_transformation.py
View file

@ -1,91 +0,0 @@
"""Cohere Parse served from Azure AI Foundry (`/providers/cohere/v2/parse`)."""
from collections.abc import Mapping
from typing import Final
import httpx
from litellm.litellm_core_utils.prompt_templates.image_handling import (
async_convert_url_to_base64,
convert_url_to_base64,
)
from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers
from litellm.llms.cohere.ocr.transformation import COHERE_PARSE_PATH, CohereParseConfig
from litellm.secret_managers.main import get_secret_str
AZURE_AI_API_KEY_ENV_VAR: Final = "AZURE_AI_API_KEY"
AZURE_AI_API_BASE_ENV_VAR: Final = "AZURE_AI_API_BASE"
AZURE_AI_COHERE_PROVIDER_PATH: Final = "/providers/cohere"
AZURE_AI_MODELS_PATH_SUFFIX: Final = "/models"
class AzureAICohereParseConfig(CohereParseConfig):
"""Same request and response shape as Cohere Parse, behind Azure AI auth and URL layout.
Foundry cannot fetch external URLs, so remote images are inlined as base64 data URIs.
"""
def get_api_key_env_var(self) -> str | None:
return AZURE_AI_API_KEY_ENV_VAR
def _llm_provider(self) -> str:
return "azure_ai"
def validate_environment(
self,
headers: Mapping[str, str],
model: str,
api_key: str | None = None,
api_base: str | None = None,
litellm_params: Mapping[str, object] | None = None,
**kwargs: object, # kwargs-ok: BaseOCRConfig.validate_environment signature
) -> dict[str, str]: # mutable-ok: BaseOCRConfig signature
resolved_base: Final = api_base or get_secret_str(AZURE_AI_API_BASE_ENV_VAR)
if resolved_base is None:
raise ValueError(
f"Missing Azure AI API Base - Set {AZURE_AI_API_BASE_ENV_VAR} environment variable "
"or pass api_base parameter"
)
resolved_key: Final = api_key or get_secret_str(AZURE_AI_API_KEY_ENV_VAR)
return { # mutable-ok: BaseOCRConfig signature
**get_azure_ai_auth_headers(api_key=resolved_key, litellm_params=litellm_params),
"Content-Type": "application/json",
**headers,
}
def get_complete_url(
self,
api_base: str | None,
model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object] | None = None,
**kwargs: object, # kwargs-ok: BaseOCRConfig.get_complete_url signature
) -> str:
resolved_base: Final = api_base or get_secret_str(AZURE_AI_API_BASE_ENV_VAR)
if resolved_base is None:
raise ValueError(
f"Missing Azure AI API Base - Set {AZURE_AI_API_BASE_ENV_VAR} environment variable "
"or pass api_base parameter"
)
url: Final = httpx.URL(resolved_base)
if not url.is_absolute_url:
raise ValueError(
"Azure AI API Base must be an absolute URL including scheme (e.g. "
f"'https://<resource>.services.ai.azure.com'). Got api_base={resolved_base!r}."
)
path: Final = url.path.rstrip("/")
if path.endswith(COHERE_PARSE_PATH):
return str(url.copy_with(path=path))
if path.endswith(f"{AZURE_AI_COHERE_PROVIDER_PATH}/v2"):
return str(url.copy_with(path=f"{path}/parse"))
return str(
url.copy_with(
path=f"{path.removesuffix(AZURE_AI_MODELS_PATH_SUFFIX)}{AZURE_AI_COHERE_PROVIDER_PATH}{COHERE_PARSE_PATH}"
)
)
def _resolve_image_url_sync(self, image_url: str) -> str:
return convert_url_to_base64(image_url)
async def _resolve_image_url_async(self, image_url: str) -> str:
return await async_convert_url_to_base64(image_url)

71
litellm/llms/azure_ai/ocr/common_utils.py
View file

@ -1,71 +0,0 @@
"""
Common utilities for Azure AI OCR providers.
This module provides routing logic to determine which OCR configuration to use
based on the model name.
"""
from typing import TYPE_CHECKING, Final, Optional
from litellm._logging import verbose_logger
if TYPE_CHECKING:
from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig
def is_azure_document_intelligence_model(model: str) -> bool:
"""Whether an azure_ai OCR model routes to Azure Document Intelligence.
Azure AI exposes two OCR services on the same provider; the sub-route in the
model name (`azure_ai/doc-intelligence/<model>`) selects Document Intelligence
over Mistral OCR. This is the single source of truth for that routing decision.
"""
lowered: Final = model.lower()
return "doc-intelligence" in lowered or "documentintelligence" in lowered
def is_azure_cohere_parse_model(model: str) -> bool:
lowered: Final = model.lower()
return "cohere" in lowered and "parse" in lowered
def get_azure_ai_ocr_config(model: str) -> Optional["BaseOCRConfig"]:
"""
Determine which Azure AI OCR configuration to use based on the model name.
Azure AI supports multiple OCR services:
- Azure Document Intelligence: azure_ai/doc-intelligence/<model>
- Mistral OCR (via Azure AI): azure_ai/<model>
Args:
model: The model name (e.g., "azure_ai/doc-intelligence/prebuilt-read",
"azure_ai/pixtral-12b-2409")
Returns:
OCR configuration instance for the specified model
Examples:
>>> get_azure_ai_ocr_config("azure_ai/doc-intelligence/prebuilt-read")
<AzureDocumentIntelligenceOCRConfig object>
>>> get_azure_ai_ocr_config("azure_ai/pixtral-12b-2409")
<AzureAIOCRConfig object>
"""
from litellm.llms.azure_ai.ocr.cohere_parse_transformation import AzureAICohereParseConfig
from litellm.llms.azure_ai.ocr.document_intelligence.transformation import (
AzureDocumentIntelligenceOCRConfig,
)
from litellm.llms.azure_ai.ocr.transformation import AzureAIOCRConfig
# Check for Azure Document Intelligence models
if is_azure_document_intelligence_model(model):
verbose_logger.debug("Routing %s to Azure Document Intelligence OCR config", model)
return AzureDocumentIntelligenceOCRConfig()
if is_azure_cohere_parse_model(model):
verbose_logger.debug("Routing %s to Azure AI Cohere Parse config", model)
return AzureAICohereParseConfig()
# Default to Mistral-based OCR for other azure_ai models
verbose_logger.debug("Routing %s to Azure AI (Mistral) OCR config", model)
return AzureAIOCRConfig()

5
litellm/llms/azure_ai/ocr/document_intelligence/__init__.py
View file

@ -1,5 +0,0 @@
"""Azure Document Intelligence OCR module."""
from .transformation import AzureDocumentIntelligenceOCRConfig
__all__ = ["AzureDocumentIntelligenceOCRConfig"]

806
litellm/llms/azure_ai/ocr/document_intelligence/transformation.py
View file

@ -1,806 +0,0 @@
"""
Azure Document Intelligence OCR transformation implementation.
Azure Document Intelligence (formerly Form Recognizer) provides advanced document analysis capabilities.
This implementation transforms between Mistral OCR format and Azure Document Intelligence API v4.0.
Note: Azure Document Intelligence API is async - POST returns 202 Accepted with Operation-Location header.
The operation location must be polled until the analysis completes.
"""
import asyncio
import re
import time
from collections.abc import Mapping
from typing import TYPE_CHECKING, Final
from urllib.parse import quote
import httpx
from pydantic import BaseModel
from litellm._logging import verbose_logger
from litellm.constants import (
AZURE_DOCUMENT_INTELLIGENCE_API_VERSION,
AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI,
AZURE_OPERATION_POLLING_TIMEOUT,
)
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin, encode_url_path_segment
from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers
from litellm.llms.base_llm.ocr.transformation import (
OCR_REQUEST_FORMAT_PARAM,
BaseOCRConfig,
DocumentType,
OCRPage,
OCRPageDimensions,
OCRRequestData,
OCRRequestFormat,
OCRResponse,
OCRUsageInfo,
parse_ocr_request_format,
)
from litellm.secret_managers.main import get_secret_str
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV_VAR: Final = "AZURE_DOCUMENT_INTELLIGENCE_API_KEY"
class AzureDocumentIntelligenceLine(BaseModel):
content: str | None = None
class AzureDocumentIntelligencePage(BaseModel):
pageNumber: int | None = None
width: float | None = None
height: float | None = None
unit: str | None = None
lines: tuple[AzureDocumentIntelligenceLine, ...] = ()
class AzureDocumentIntelligenceAnalyzeResult(BaseModel):
content: str | None = None
pages: tuple[AzureDocumentIntelligencePage, ...] = ()
tables: list[dict[str, object]] | None = None
keyValuePairs: list[dict[str, object]] | None = None
class AzureDocumentIntelligenceOperation(BaseModel):
status: str | None = None
analyzeResult: AzureDocumentIntelligenceAnalyzeResult | None = None
class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
"""
Azure Document Intelligence OCR transformation configuration.
Supports Azure Document Intelligence v4.0 (2024-11-30) API.
Model route: azure_ai/doc-intelligence/<model>
Supported models:
- prebuilt-layout: Extracts text with markdown, tables, and structure (closest to Mistral OCR)
- prebuilt-read: Basic text extraction optimized for reading
- prebuilt-document: General document analysis
Reference: https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/
"""
def __init__(self) -> None:
super().__init__()
def get_api_key_env_var(self) -> str | None:
return AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV_VAR
def resolve_connection_params(
self,
*,
api_key: str | None,
api_base: str | None,
dynamic_api_key: str | None,
dynamic_api_base: str | None,
) -> tuple[str | None, str | None]:
explicit_api_key: Final = None if api_key is None else dynamic_api_key or api_key
explicit_api_base: Final = None if api_base is None else dynamic_api_base or api_base
return explicit_api_key, explicit_api_base
def get_supported_ocr_params(self, model: str) -> list:
"""
Get supported OCR parameters for Azure Document Intelligence.
Azure DI exposes a `pages` query parameter on the analyze endpoint
(1-based, e.g. "1-3,5,7-9"). To keep the public request shape
aligned with Mistral OCR, callers pass `pages` using Mistral
semantics — a list of 0-based integers — or a pre-formatted
Azure-style string. Azure DI also exposes a `features` query
parameter enabling add-on capabilities (e.g. "keyValuePairs",
"languages"), passed as a list of feature names or a
comma-separated string. Other Mistral-specific params (e.g.
`include_image_base64`) are not supported by Azure DI and are
ignored during transformation.
`req_format` selects the response shape: "litellm" (default) returns
the normalized OCR schema, "native" returns Azure DI's own analyze
operation payload as-is.
"""
return ["pages", "features", OCR_REQUEST_FORMAT_PARAM]
def map_ocr_params(
self,
non_default_params: Mapping[str, object],
optional_params: dict,
model: str,
) -> dict:
"""
Map OCR params to Azure DI format.
Translates Mistral-style `pages` (list[int], 0-based) into Azure's
`pages` query string (1-based, e.g. "1,2,3" or "1-3,5"). A raw
string that already matches Azure's format is passed through
unchanged. `features` (list[str] or comma-separated string) is
normalized into Azure's comma-joined `features` query string.
"""
pages: Final = non_default_params.get("pages")
features: Final = non_default_params.get("features")
request_format: Final = non_default_params.get(OCR_REQUEST_FORMAT_PARAM)
normalized_pages: Final = self._normalize_pages_param(pages) if pages is not None else ""
normalized_features: Final = self._normalize_features_param(features) if features is not None else ""
return {
**optional_params,
**({"pages": normalized_pages} if normalized_pages else {}),
**({"features": normalized_features} if normalized_features else {}),
**(
{OCR_REQUEST_FORMAT_PARAM: self._parse_request_format(request_format, model)}
if request_format is not None
else {}
),
}
@staticmethod
def _parse_request_format(request_format: object, model: str) -> OCRRequestFormat:
try:
return parse_ocr_request_format(request_format)
except ValueError as e:
raise UnsupportedParamsError(message=f"{e}", model=model, llm_provider="azure_ai") from e
@staticmethod
def _normalize_pages_param(pages: object) -> str:
"""
Convert a caller-provided `pages` value to Azure DI's query-string
form. Azure expects 1-based page numbers, grammar: `^(\\d+(-\\d+)?)(,\\s*(\\d+(-\\d+)?))*$`.
Accepted inputs:
- list[int]: Mistral-style 0-based indices. Converted to 1-based
and joined (e.g. [0,1,2] -> "1,2,3").
- list[str]: tokens like "1" or "3-5". Validated, joined as-is
(treated as Azure-native, i.e. 1-based).
- str: already in Azure format. Validated and whitespace-stripped.
"""
pages_pattern: Final = re.compile(r"^\s*\d+(-\d+)?(\s*,\s*\d+(-\d+)?)*\s*$")
if isinstance(pages, str):
if not pages_pattern.match(pages):
raise ValueError(
f"Invalid `pages` string for Azure Document Intelligence: "
f"{pages!r}. Expected format like '1-3,5,7-9'."
)
return pages.replace(" ", "")
if isinstance(pages, list):
if len(pages) == 0:
return ""
if any(isinstance(p, bool) for p in pages):
raise ValueError("`pages` must be integers, not booleans")
if all(isinstance(p, int) for p in pages):
if any(p < 0 for p in pages):
raise ValueError("`pages` integers must be >= 0 (Mistral 0-based indices)")
# Mistral 0-based -> Azure 1-based.
return ",".join(str(p + 1) for p in sorted(set(pages)))
if all(isinstance(p, str) for p in pages):
joined: Final = ",".join(p.strip() for p in pages)
if not pages_pattern.match(joined):
raise ValueError(
f"Invalid `pages` list for Azure Document Intelligence: "
f"{pages!r}. Expected tokens like '1' or '3-5'."
)
return joined
raise ValueError("`pages` must be a list[int] (0-based, Mistral-style) or a string like '1-3,5,7-9'.")
@staticmethod
def _normalize_features_param(features: object) -> str:
"""
Convert a caller-provided `features` value to Azure DI's query-string
form (comma-joined feature names, e.g. "keyValuePairs,languages").
Accepted inputs:
- list[str]: feature names like ["keyValuePairs", "languages"].
- str: a single feature name or comma-separated names.
"""
invalid_features_error: Final = ValueError(
f"Invalid `features` for Azure Document Intelligence: {features!r}. "
f"Expected a list of feature names or a comma-separated string like "
f"'keyValuePairs' or 'keyValuePairs,languages'."
)
if isinstance(features, str):
raw_tokens = features.split(",")
elif isinstance(features, list):
if len(features) == 0:
return ""
raw_tokens = [feature for feature in features if isinstance(feature, str)]
if len(raw_tokens) != len(features):
raise invalid_features_error
else:
raise invalid_features_error
tokens: Final = tuple(token.strip() for token in raw_tokens)
feature_pattern: Final = re.compile(r"^[A-Za-z][A-Za-z0-9]*$")
if not all(feature_pattern.match(token) for token in tokens):
raise invalid_features_error
return ",".join(tokens)
def validate_environment(
self,
headers: dict,
model: str,
api_key: str | None = None,
api_base: str | None = None,
litellm_params: dict | None = None,
**kwargs,
) -> dict:
"""
Validate environment and return headers for Azure Document Intelligence.
Authentication uses the Ocp-Apim-Subscription-Key header, or an Entra ID / OAuth bearer
token when no subscription key is set.
"""
# Get API key from environment if not provided
if api_key is None:
api_key = get_secret_str(AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV_VAR)
# Validate API base/endpoint is provided
if api_base is None:
api_base = get_secret_str("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT")
if api_base is None:
raise ValueError(
"Missing Azure Document Intelligence Endpoint - Set AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT environment variable or pass api_base parameter"
)
headers = {
**get_azure_ai_auth_headers(
api_key=api_key,
litellm_params=litellm_params,
api_key_header="Ocp-Apim-Subscription-Key",
api_key_env_var=AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV_VAR,
),
"Content-Type": "application/json",
**headers,
}
return headers
def get_complete_url(
self,
api_base: str | None,
model: str,
optional_params: dict,
litellm_params: dict | None = None,
**kwargs,
) -> str:
"""
Get complete URL for Azure Document Intelligence endpoint.
Format: {endpoint}/documentintelligence/documentModels/{modelId}:analyze?api-version=2024-11-30
Note: API version 2024-11-30 uses /documentintelligence/ path (not /formrecognizer/)
Args:
api_base: Azure Document Intelligence endpoint (e.g., https://your-resource.cognitiveservices.azure.com)
model: Model ID (e.g., "prebuilt-layout", "prebuilt-read")
optional_params: Optional parameters
Returns: Complete URL for Azure DI analyze endpoint
"""
if api_base is None:
api_base = get_secret_str("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT")
if api_base is None:
raise ValueError(
"Missing Azure Document Intelligence Endpoint - Set AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT environment variable or pass api_base parameter"
)
# Ensure no trailing slash
api_base = api_base.rstrip("/")
# Extract model ID from full model path if needed
# Model can be "prebuilt-layout" or "azure_ai/doc-intelligence/prebuilt-layout"
model_id = model
if "/" in model:
# Extract the last part after the last slash
model_id = model.split("/")[-1]
encoded_model_id: Final = encode_url_path_segment(model_id, field_name="model_id")
# Azure Document Intelligence analyze endpoint
# Note: API version 2024-11-30+ uses /documentintelligence/ (not /formrecognizer/)
url: Final = (
f"{api_base}/documentintelligence/documentModels/{encoded_model_id}:analyze"
f"?api-version={AZURE_DOCUMENT_INTELLIGENCE_API_VERSION}"
)
# Azure DI accepts `pages` (1-based, e.g. "1-3,5") and `features`
# (comma-joined names, e.g. "keyValuePairs") as query params.
# `optional_params` has already been normalized in `map_ocr_params`.
pages: Final = optional_params.get("pages") if optional_params else None
features: Final = optional_params.get("features") if optional_params else None
pages_query: Final = f"&pages={quote(str(pages), safe=',-')}" if pages else ""
features_query: Final = f"&features={quote(str(features), safe=',')}" if features else ""
return f"{url}{pages_query}{features_query}"
def _extract_base64_from_data_uri(self, data_uri: str) -> str:
"""
Extract base64 content from a data URI.
Args:
data_uri: Data URI like "data:application/pdf;base64,..."
Returns:
Base64 string without the data URI prefix
"""
# Match pattern: data:[<mediatype>][;base64],<data>
match: Final = re.match(r"data:([^;]+)(?:;base64)?,(.+)", data_uri)
if match:
return match.group(2)
return data_uri
def transform_ocr_request(
self,
model: str,
document: DocumentType,
optional_params: dict,
headers: dict,
**kwargs,
) -> OCRRequestData:
"""
Transform OCR request to Azure Document Intelligence format.
Mistral OCR format:
{
"document": {
"type": "document_url",
"document_url": "https://example.com/doc.pdf"
}
}
Azure DI format:
{
"urlSource": "https://example.com/doc.pdf"
}
OR
{
"base64Source": "base64_encoded_content"
}
Args:
model: Model name
document: Document dict from user (Mistral format)
optional_params: Already mapped optional parameters
headers: Request headers
Returns:
OCRRequestData with JSON data
"""
verbose_logger.debug("Azure Document Intelligence transform_ocr_request - model: %s", model)
if not isinstance(document, dict):
raise ValueError(f"Expected document dict, got {type(document)}")
# Extract document URL from Mistral format
doc_type: Final = document.get("type")
document_url = None
if doc_type == "document_url":
document_url = document.get("document_url", "")
elif doc_type == "image_url":
document_url = document.get("image_url", "")
else:
raise ValueError(f"Invalid document type: {doc_type}. Must be 'document_url' or 'image_url'")
if not document_url:
raise ValueError("Document URL is required")
# Build Azure DI request
data: Final[dict[str, str]] = {}
# Check if it's a data URI (base64)
if document_url.startswith("data:"):
# Extract base64 content
base64_content: Final = self._extract_base64_from_data_uri(document_url)
data["base64Source"] = base64_content
verbose_logger.debug("Using base64Source for Azure Document Intelligence")
else:
# Regular URL
data["urlSource"] = document_url
verbose_logger.debug("Using urlSource for Azure Document Intelligence")
# Azure DI: `pages` is a query param (wired in get_complete_url),
# not a body field. Other Mistral-specific params (e.g.
# include_image_base64, image_limit) are unsupported and ignored.
return OCRRequestData(data=data, files=None)
def _transform_azure_page(self, azure_page: AzureDocumentIntelligencePage) -> OCRPage:
page_number: Final = azure_page.pageNumber if azure_page.pageNumber is not None else 1
markdown: Final = "\n".join(line.content or "" for line in azure_page.lines)
dimensions: Final = self._convert_dimensions(
width=azure_page.width if azure_page.width is not None else 8.5,
height=azure_page.height if azure_page.height is not None else 11,
unit=azure_page.unit if azure_page.unit is not None else "inch",
)
return OCRPage(index=page_number - 1, markdown=markdown, dimensions=dimensions)
def _convert_dimensions(self, width: float, height: float, unit: str) -> OCRPageDimensions:
"""
Convert Azure DI dimensions to pixels.
Azure DI provides dimensions in inches. We convert to pixels using configured DPI.
Args:
width: Width in specified unit
height: Height in specified unit
unit: Unit of measurement (e.g., "inch")
Returns:
OCRPageDimensions with pixel values
"""
# Convert to pixels using configured DPI
dpi: Final = AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI
if unit == "inch":
width_px = int(width * dpi)
height_px = int(height * dpi)
else:
# If unit is not inches, assume it's already in pixels
width_px = int(width)
height_px = int(height)
return OCRPageDimensions(width=width_px, height=height_px, dpi=dpi)
@staticmethod
def _check_timeout(start_time: float, timeout_secs: int) -> None:
"""
Check if operation has timed out.
Args:
start_time: Start time of the operation
timeout_secs: Timeout duration in seconds
Raises:
TimeoutError: If operation has exceeded timeout
"""
if time.time() - start_time > timeout_secs:
raise TimeoutError(f"Azure Document Intelligence operation polling timed out after {timeout_secs} seconds")
@staticmethod
def _get_retry_after(response: httpx.Response) -> int:
"""
Get retry-after duration from response headers.
Args:
response: HTTP response
Returns:
Retry-after duration in seconds (default: 2)
"""
retry_after: Final = int(response.headers.get("retry-after", "2"))
verbose_logger.debug("Retry polling after: %s seconds", retry_after)
return retry_after
@staticmethod
def _check_operation_status(response: httpx.Response) -> str:
"""
Check Azure DI operation status from response.
Args:
response: HTTP response from operation endpoint
Returns:
Operation status string
Raises:
ValueError: If operation failed or status is unknown
"""
try:
result: Final = response.json()
status: Final = result.get("status")
verbose_logger.debug("Azure DI operation status: %s", status)
if status == "succeeded":
return "succeeded"
elif status == "failed":
error_msg: Final = result.get("error", {}).get("message", "Unknown error")
raise ValueError(f"Azure Document Intelligence analysis failed: {error_msg}")
elif status in ["running", "notStarted"]:
return "running"
else:
raise ValueError(f"Unknown operation status: {status}")
except Exception as e:
if "succeeded" in str(e) or "failed" in str(e):
raise
# If we can't parse JSON, something went wrong
raise ValueError(f"Failed to parse Azure DI operation response: {e}")
def _poll_operation_sync(
self,
operation_url: str,
headers: dict[str, str],
timeout_secs: int,
) -> httpx.Response:
"""
Poll Azure Document Intelligence operation until completion (sync).
Azure DI POST returns 202 with Operation-Location header.
We need to poll that URL until status is "succeeded" or "failed".
Args:
operation_url: The Operation-Location URL to poll
headers: Request headers (including auth)
timeout_secs: Total timeout in seconds
Returns:
Final response with completed analysis
"""
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
client: Final = _get_httpx_client()
start_time: Final = time.time()
verbose_logger.debug("Polling Azure DI operation: %s", operation_url)
while True:
self._check_timeout(start_time=start_time, timeout_secs=timeout_secs)
# Poll the operation status
response = client.get(url=operation_url, headers=headers)
# Check operation status
status = self._check_operation_status(response=response)
if status == "succeeded":
return response
elif status == "running":
# Wait before polling again
retry_after = self._get_retry_after(response=response)
time.sleep(retry_after)
async def _poll_operation_async(
self,
operation_url: str,
headers: dict[str, str],
timeout_secs: int,
) -> httpx.Response:
"""
Poll Azure Document Intelligence operation until completion (async).
Args:
operation_url: The Operation-Location URL to poll
headers: Request headers (including auth)
timeout_secs: Total timeout in seconds
Returns:
Final response with completed analysis
"""
import litellm
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.AZURE_AI)
start_time: Final = time.time()
verbose_logger.debug("Polling Azure DI operation (async): %s", operation_url)
while True:
self._check_timeout(start_time=start_time, timeout_secs=timeout_secs)
# Poll the operation status
response = await client.get(url=operation_url, headers=headers)
# Check operation status
status = self._check_operation_status(response=response)
if status == "succeeded":
return response
elif status == "running":
# Wait before polling again
retry_after = self._get_retry_after(response=response)
await asyncio.sleep(retry_after)
def _get_polling_target(self, raw_response: httpx.Response) -> tuple[str, dict[str, str]]:
operation_url: Final = raw_response.headers.get("Operation-Location")
if not operation_url:
raise ValueError("Azure Document Intelligence returned 202 but no Operation-Location header found")
# Reject cross-origin polling URLs — the auth headers
# below would otherwise leak to whatever URL the upstream
# (or an attacker-controlled upstream) returns. VERIA-51.
try:
assert_same_origin(operation_url, str(raw_response.request.url))
except SSRFError as ssrf_err:
raise ValueError(f"Azure Document Intelligence: rejected polling URL ({ssrf_err})")
poll_headers: Final = {
header: raw_response.request.headers[header]
for header in ("Ocp-Apim-Subscription-Key", "Authorization")
if header in raw_response.request.headers
}
return operation_url, poll_headers
@staticmethod
def _get_request_format(optional_params: object) -> OCRRequestFormat:
if not isinstance(optional_params, dict):
return "litellm"
request_format: Final = optional_params.get(OCR_REQUEST_FORMAT_PARAM)
if request_format is None:
return "litellm"
return parse_ocr_request_format(request_format)
def _transform_completed_response(
self,
model: str,
raw_response: httpx.Response,
request_format: OCRRequestFormat,
) -> OCRResponse:
"""
Transform a completed Azure Document Intelligence analyze operation
into the Mistral OCR response shape, preserving Azure-native
`analyzeResult` fields (`content`, `tables`, `keyValuePairs`) as
top-level response fields.
When `request_format` is "native", the untouched Azure operation
payload is attached to the response's hidden params so the proxy can
return it verbatim while cost tracking still reads `usage_info`.
"""
raw_operation: Final[Mapping[str, object]] = raw_response.json()
operation: Final = AzureDocumentIntelligenceOperation.model_validate(raw_operation)
verbose_logger.debug("Azure Document Intelligence response status: %s", operation.status)
if operation.status != "succeeded":
raise ValueError(f"Azure Document Intelligence analysis failed with status: {operation.status}")
analyze_result: Final = (
operation.analyzeResult if operation.analyzeResult is not None else AzureDocumentIntelligenceAnalyzeResult()
)
mistral_pages: Final = [self._transform_azure_page(azure_page) for azure_page in analyze_result.pages]
usage_info: Final = OCRUsageInfo(pages_processed=len(mistral_pages), doc_size_bytes=None)
response: Final = OCRResponse(
pages=mistral_pages,
model=model,
usage_info=usage_info,
object="ocr",
content=analyze_result.content,
tables=analyze_result.tables,
keyValuePairs=analyze_result.keyValuePairs,
)
if request_format == "native":
response.set_provider_native_response(raw_operation)
return response
def transform_ocr_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: "LiteLLMLoggingObj",
**kwargs,
) -> OCRResponse:
"""
Transform Azure Document Intelligence response to Mistral OCR format.
Handles async operation polling: If response is 202 Accepted, polls Operation-Location
until analysis completes.
Azure DI response (after polling):
{
"status": "succeeded",
"analyzeResult": {
"content": "Full document text...",
"pages": [
{
"pageNumber": 1,
"width": 8.5,
"height": 11,
"unit": "inch",
"lines": [{"content": "text", "boundingBox": [...]}]
}
],
"tables": [...],
"keyValuePairs": [...]
}
}
Mistral OCR format (with Azure-native fields preserved):
{
"pages": [
{
"index": 0,
"markdown": "extracted text",
"dimensions": {"width": 816, "height": 1056, "dpi": 96}
}
],
"model": "azure_ai/doc-intelligence/prebuilt-layout",
"usage_info": {"pages_processed": 1},
"object": "ocr",
"content": "Full document text...",
"tables": [...],
"keyValuePairs": [...]
}
Args:
model: Model name
raw_response: Raw HTTP response from Azure DI (may be 202 Accepted)
logging_obj: Logging object
Returns:
OCRResponse in Mistral format
"""
request_format: Final = self._get_request_format(kwargs.get("optional_params"))
if raw_response.status_code != 202:
return self._transform_completed_response(
model=model, raw_response=raw_response, request_format=request_format
)
verbose_logger.debug("Azure DI returned 202 Accepted, polling operation...")
operation_url, poll_headers = self._get_polling_target(raw_response)
completed_response: Final = self._poll_operation_sync(
operation_url=operation_url,
headers=poll_headers,
timeout_secs=AZURE_OPERATION_POLLING_TIMEOUT,
)
return self._transform_completed_response(
model=model, raw_response=completed_response, request_format=request_format
)
async def async_transform_ocr_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: "LiteLLMLoggingObj",
**kwargs,
) -> OCRResponse:
"""
Async transform Azure Document Intelligence response to Mistral OCR format.
Handles async operation polling: If response is 202 Accepted, polls Operation-Location
until analysis completes using async polling.
Args:
model: Model name
raw_response: Raw HTTP response from Azure DI (may be 202 Accepted)
logging_obj: Logging object
Returns:
OCRResponse in Mistral format
"""
request_format: Final = self._get_request_format(kwargs.get("optional_params"))
if raw_response.status_code != 202:
return self._transform_completed_response(
model=model, raw_response=raw_response, request_format=request_format
)
verbose_logger.debug("Azure DI returned 202 Accepted, polling operation (async)...")
operation_url, poll_headers = self._get_polling_target(raw_response)
completed_response: Final = await self._poll_operation_async(
operation_url=operation_url,
headers=poll_headers,
timeout_secs=AZURE_OPERATION_POLLING_TIMEOUT,
)
return self._transform_completed_response(
model=model, raw_response=completed_response, request_format=request_format
)

263
litellm/llms/azure_ai/ocr/transformation.py
View file

@ -1,263 +0,0 @@
"""
Azure AI OCR transformation implementation.
"""
from typing import Final
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.image_handling import (
async_convert_url_to_base64,
convert_url_to_base64,
)
from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers
from litellm.llms.base_llm.ocr.transformation import DocumentType, OCRRequestData
from litellm.llms.mistral.ocr.transformation import MistralOCRConfig
from litellm.secret_managers.main import get_secret_str
AZURE_AI_OCR_API_KEY_ENV_VAR: Final = "AZURE_AI_API_KEY"
class AzureAIOCRConfig(MistralOCRConfig):
"""
Azure AI OCR transformation configuration.
Azure AI uses Mistral's OCR API but with a different endpoint format.
Inherits transformation logic from MistralOCRConfig since they use the same format.
Reference: Azure AI Foundry OCR documentation
Important: Azure AI only supports base64 data URIs (data:image/..., data:application/pdf;base64,...).
Regular URLs are not supported.
"""
def __init__(self) -> None:
super().__init__()
def get_api_key_env_var(self) -> str | None:
return AZURE_AI_OCR_API_KEY_ENV_VAR
def validate_environment(
self,
headers: dict,
model: str,
api_key: str | None = None,
api_base: str | None = None,
litellm_params: dict | None = None,
**kwargs,
) -> dict:
"""
Validate environment and return headers for Azure AI OCR.
Authenticates with AZURE_AI_API_KEY, or with an Entra ID / OAuth token when no key is set.
"""
# Get API key from environment if not provided
if api_key is None:
api_key = get_secret_str(AZURE_AI_OCR_API_KEY_ENV_VAR)
# Validate API base is provided
if api_base is None:
api_base = get_secret_str("AZURE_AI_API_BASE")
if api_base is None:
raise ValueError(
"Missing Azure AI API Base - Set AZURE_AI_API_BASE environment variable or pass api_base parameter"
)
headers = {
**get_azure_ai_auth_headers(api_key=api_key, litellm_params=litellm_params),
"Content-Type": "application/json",
**headers,
}
return headers
def get_complete_url(
self,
api_base: str | None,
model: str,
optional_params: dict,
litellm_params: dict | None = None,
**kwargs,
) -> str:
"""
Get complete URL for Azure AI OCR endpoint.
Azure AI endpoint format: https://<api_base>/providers/mistral/azure/ocr
Args:
api_base: Azure AI API base URL
model: Model name (not used in URL construction)
optional_params: Optional parameters
Returns: Complete URL for Azure AI OCR endpoint
"""
if api_base is None:
raise ValueError(
"Missing Azure AI API Base - Set AZURE_AI_API_BASE environment variable or pass api_base parameter"
)
# Ensure no trailing slash
api_base = api_base.rstrip("/")
# Azure AI OCR endpoint format
return f"{api_base}/providers/mistral/azure/ocr"
def _convert_url_to_data_uri_sync(self, url: str) -> str:
"""
Synchronously convert a URL to a base64 data URI.
Azure AI OCR doesn't have internet access, so we need to fetch URLs
and convert them to base64 data URIs.
Args:
url: The URL to convert
Returns:
Base64 data URI string
"""
verbose_logger.debug("Azure AI OCR: Converting URL to base64 data URI (sync): %s", url)
# Fetch and convert to base64 data URI
# convert_url_to_base64 already returns a full data URI like "data:image/jpeg;base64,..."
data_uri: Final = convert_url_to_base64(url=url)
verbose_logger.debug("Azure AI OCR: Converted URL to data URI (length: %s)", len(data_uri))
return data_uri
async def _convert_url_to_data_uri_async(self, url: str) -> str:
"""
Asynchronously convert a URL to a base64 data URI.
Azure AI OCR doesn't have internet access, so we need to fetch URLs
and convert them to base64 data URIs.
Args:
url: The URL to convert
Returns:
Base64 data URI string
"""
verbose_logger.debug("Azure AI OCR: Converting URL to base64 data URI (async): %s", url)
# Fetch and convert to base64 data URI asynchronously
# async_convert_url_to_base64 already returns a full data URI like "data:image/jpeg;base64,..."
data_uri: Final = await async_convert_url_to_base64(url=url)
verbose_logger.debug("Azure AI OCR: Converted URL to data URI (length: %s)", len(data_uri))
return data_uri
def transform_ocr_request(
self,
model: str,
document: DocumentType,
optional_params: dict,
headers: dict,
**kwargs,
) -> OCRRequestData:
"""
Transform OCR request for Azure AI, converting URLs to base64 data URIs (sync).
Azure AI OCR doesn't have internet access, so we automatically fetch
any URLs and convert them to base64 data URIs synchronously.
Args:
model: Model name
document: Document dict from user
optional_params: Already mapped optional parameters
headers: Request headers
**kwargs: Additional arguments
Returns:
OCRRequestData with JSON data
"""
verbose_logger.debug("Azure AI OCR transform_ocr_request (sync) - model: %s", model)
if not isinstance(document, dict):
raise ValueError(f"Expected document dict, got {type(document)}")
# Check if we need to convert URL to base64
doc_type: Final = document.get("type")
transformed_document: Final = document.copy()
if doc_type == "document_url":
document_url: Final = document.get("document_url", "")
# If it's not already a data URI, convert it
if document_url and not document_url.startswith("data:"):
verbose_logger.debug("Azure AI OCR: Converting document URL to base64 data URI (sync)")
data_uri = self._convert_url_to_data_uri_sync(url=document_url)
transformed_document["document_url"] = data_uri
elif doc_type == "image_url":
image_url: Final = document.get("image_url", "")
# If it's not already a data URI, convert it
if image_url and not image_url.startswith("data:"):
verbose_logger.debug("Azure AI OCR: Converting image URL to base64 data URI (sync)")
data_uri = self._convert_url_to_data_uri_sync(url=image_url)
transformed_document["image_url"] = data_uri
# Call parent's transform to build the request
return super().transform_ocr_request(
model=model,
document=transformed_document,
optional_params=optional_params,
headers=headers,
**kwargs,
)
async def async_transform_ocr_request(
self,
model: str,
document: DocumentType,
optional_params: dict,
headers: dict,
**kwargs,
) -> OCRRequestData:
"""
Transform OCR request for Azure AI, converting URLs to base64 data URIs (async).
Azure AI OCR doesn't have internet access, so we automatically fetch
any URLs and convert them to base64 data URIs asynchronously.
Args:
model: Model name
document: Document dict from user
optional_params: Already mapped optional parameters
headers: Request headers
**kwargs: Additional arguments
Returns:
OCRRequestData with JSON data
"""
verbose_logger.debug("Azure AI OCR async_transform_ocr_request - model: %s", model)
if not isinstance(document, dict):
raise ValueError(f"Expected document dict, got {type(document)}")
# Check if we need to convert URL to base64
doc_type: Final = document.get("type")
transformed_document: Final = document.copy()
if doc_type == "document_url":
document_url: Final = document.get("document_url", "")
# If it's not already a data URI, convert it
if document_url and not document_url.startswith("data:"):
verbose_logger.debug("Azure AI OCR: Converting document URL to base64 data URI (async)")
data_uri = await self._convert_url_to_data_uri_async(url=document_url)
transformed_document["document_url"] = data_uri
elif doc_type == "image_url":
image_url: Final = document.get("image_url", "")
# If it's not already a data URI, convert it
if image_url and not image_url.startswith("data:"):
verbose_logger.debug("Azure AI OCR: Converting image URL to base64 data URI (async)")
data_uri = await self._convert_url_to_data_uri_async(url=image_url)
transformed_document["image_url"] = data_uri
# Call parent's transform to build the request
return super().transform_ocr_request(
model=model,
document=transformed_document,
optional_params=optional_params,
headers=headers,
**kwargs,
)

38
litellm/llms/azure_ai/passthrough/transformation.py
View file

@ -1,6 +1,6 @@
from __future__ import annotations
from collections.abc import Callable, Mapping, Sequence
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Final
@ -13,7 +13,7 @@ from litellm.llms.azure_ai.common_utils import (
api_key_header_for_base,
get_azure_ai_auth_headers,
)
from litellm.llms.azure_ai.ocr.common_utils import get_azure_ai_ocr_config
from litellm.llms.base_llm.ocr.transformation import OCRResponse
from litellm.llms.base_llm.passthrough.transformation import (
BasePassthroughConfig,
RelayShape,
@ -22,6 +22,7 @@ from litellm.llms.base_llm.passthrough.transformation import (
relayed_body,
strip_leading_model_segment,
)
from litellm.rust_bridge.ocr.entrypoints import NATIVE_OCR_PASSTHROUGH_RESPONSE, NativeOcrPassthroughResponse
from litellm.types.llms.openai import AllMessageValues
from litellm.types.rerank import RerankResponse
from litellm.types.utils import CallTypes, ImageResponse, StandardPassThroughResponseObject
@ -30,7 +31,6 @@ if TYPE_CHECKING:
from httpx import URL, Response
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, OCRResponse
from litellm.llms.base_llm.passthrough.transformation import LoggedRelayResponse
@ -92,9 +92,9 @@ FOUNDRY_RELAY_SHAPES: Final = (
class AzureAIPassthroughConfig(AzureFoundryModelInfo, BasePassthroughConfig):
def __init__(self, ocr_config_for: Callable[[str], BaseOCRConfig | None] = get_azure_ai_ocr_config) -> None:
def __init__(self, passthrough_ocr: NativeOcrPassthroughResponse | None = None) -> None:
super().__init__()
self.ocr_config_for: Final = ocr_config_for
self._passthrough_ocr: Final = passthrough_ocr
def is_streaming_request(self, endpoint: str, request_data: Mapping[str, object]) -> bool:
return bool(request_data.get("stream"))
@ -168,28 +168,22 @@ class AzureAIPassthroughConfig(AzureFoundryModelInfo, BasePassthroughConfig):
def logged_ocr_response(
self, model: str, httpx_response: Response, logging_obj: Logging, endpoint: str
) -> OCRResponse | None:
ocr_config: Final = self.ocr_config_for(model)
if ocr_config is None or httpx_response.status_code != 200:
return None
relayed_url: Final = httpx_response.request.url
relayed_origin: Final = str(relayed_url.copy_with(path="/", query=None, fragment=None)).rstrip("/")
ocr_url: Final = httpx.URL(
ocr_config.get_complete_url(
api_base=relayed_origin,
model=model,
optional_params={}, # mutable-ok: BaseOCRConfig wants a dict
)
passthrough_ocr: Final = (
self._passthrough_ocr if self._passthrough_ocr is not None else NATIVE_OCR_PASSTHROUGH_RESPONSE.load()
)
if passthrough_ocr is None or httpx_response.status_code != 200:
return None
known_prefixes: Final = (model, model_group_from(logging_obj.litellm_params))
native_endpoint: Final = strip_leading_model_segment(endpoint, known_prefixes)
if f"/{native_endpoint.strip('/')}" != ocr_url.path:
return None
try:
ocr_response: Final = ocr_config.transform_ocr_response(
model=model, raw_response=httpx_response, logging_obj=logging_obj
result: Final = passthrough_ocr(model, native_endpoint, httpx_response.content)
ocr_response: Final = OCRResponse.model_validate(result) if result is not None else None
except (ValueError, RuntimeError) as error:
verbose_logger.warning(
"azure_ai passthrough: OCR body from %s is not costable: %s", httpx_response.request.url, error
)
except (ValueError, AttributeError) as error:
verbose_logger.warning("azure_ai passthrough: OCR body from %s is not costable: %s", ocr_url, error)
return None
if ocr_response is None:
return None
logging_obj.call_type = CallTypes.aocr.value # rebind-ok: routes cost calculation to the per-page OCR path
return ocr_response

4
litellm/llms/base_llm/ocr/__init__.py
View file

@ -1,23 +1,19 @@
"""Base OCR transformation module."""
from .transformation import (
BaseOCRConfig,
DocumentType,
OCRPage,
OCRPageDimensions,
OCRPageImage,
OCRRequestData,
OCRResponse,
OCRUsageInfo,
)
__all__ = [
"BaseOCRConfig",
"DocumentType",
"OCRPage",
"OCRPageDimensions",
"OCRPageImage",
"OCRRequestData",
"OCRResponse",
"OCRUsageInfo",
]

220
litellm/llms/base_llm/ocr/transformation.py
View file

@ -1,27 +1,16 @@
"""
Base OCR transformation configuration.
Base OCR types shared by the Rust OCR route and Python consumers.
"""
import builtins
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, Literal
from typing import Any, Final, Literal
import httpx
from pydantic import PrivateAttr
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.base import LiteLLMPydanticObjectBase
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
# DocumentType for OCR - providers always receive a dict with
# type="document_url" or type="image_url" (str values only).
# File-type inputs are preprocessed to this format in litellm/ocr/main.py.
DocumentType = dict[str, str]
DocumentType = Mapping[str, object]
OCRRequestFormat = Literal["litellm", "native"]
@ -33,8 +22,6 @@ OCR_REQUEST_FORMAT_HEADER: Final = "x-req-format"
PROVIDER_NATIVE_RESPONSE_KEY: Final = "provider_native_response"
HEALTH_CHECK_PDF_DATA_URI: Final = "data:application/pdf;base64,JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9UeXBlIC9QYWdlCi9QYXJlbnQgMSAwIFIKL01lZGlhQm94IFswIDAgNjEyIDc5Ml0KL0NvbnRlbnRzIDQgMCBSCi9SZXNvdXJjZXMgPDwvRm9udCA8PC9GMSAyIDAgUj4+Pj4+PgplbmRvYmoKNCAwIG9iago8PC9MZW5ndGggNDQ+PgpzdHJlYW0KQlQKL0YxIDI0IFRmCjEwMCA3MDAgVGQKKHRlc3QpIFRqCkVUCmVuZHN0cmVhbQplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9Gb250Ci9TdWJ0eXBlIC9UeXBlMQovQmFzZUZvbnQgL0hlbHZldGljYT4+CmVuZG9iagoxIDAgb2JqCjw8L1R5cGUgL1BhZ2VzCi9LaWRzIFszIDAgUl0KL0NvdW50IDE+PgplbmRvYmoKNSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAxIDAgUj4+CmVuZG9iagp0cmFpbGVyCjw8L1NpemUgNgovUm9vdCA1IDAgUj4+CnN0YXJ0eHJlZgozMjQKJSVFT0Y="
def parse_ocr_request_format(value: object) -> OCRRequestFormat:
if value == "litellm":
@ -102,7 +89,6 @@ class OCRResponse(LiteLLMPydanticObjectBase):
model_config = {"extra": "allow"}
# Define private attributes using PrivateAttr
_hidden_params: dict = PrivateAttr(default_factory=dict)
def set_provider_native_response(self, native_response: Mapping[str, builtins.object]) -> None:
@ -113,203 +99,3 @@ class OCRResponse(LiteLLMPydanticObjectBase):
"""The provider's own response payload, when `req_format=native` was requested."""
native_response: Final = self._hidden_params.get(PROVIDER_NATIVE_RESPONSE_KEY)
return native_response if isinstance(native_response, dict) else None
class OCRRequestData(LiteLLMPydanticObjectBase):
"""OCR request data structure."""
data: dict | bytes | None = None
files: dict[str, Any] | None = None
class BaseOCRConfig:
"""
Base configuration for OCR transformations.
Handles provider-agnostic OCR operations.
"""
def __init__(self) -> None:
pass
def get_supported_ocr_params(self, model: str) -> list:
"""
Get supported OCR parameters for this provider.
Override this method in provider-specific implementations.
"""
return []
def get_api_key_env_var(self) -> str | None:
"""
Return the provider-specific API key environment variable name, if any.
"""
return None
def resolve_connection_params(
self,
*,
api_key: str | None,
api_base: str | None,
dynamic_api_key: str | None,
dynamic_api_base: str | None,
) -> tuple[str | None, str | None]:
return dynamic_api_key or api_key, dynamic_api_base or api_base
def get_health_check_document(self) -> DocumentType:
return { # mutable-ok: litellm.aocr rejects any document that is not a dict
"type": "document_url",
"document_url": HEALTH_CHECK_PDF_DATA_URI,
}
def map_ocr_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
) -> dict:
"""Map OCR parameters to provider-specific parameters."""
return optional_params
def validate_environment(
self,
headers: dict,
model: str,
api_key: str | None = None,
api_base: str | None = None,
litellm_params: dict | None = None,
**kwargs,
) -> dict:
"""
Validate environment and return headers.
Override in provider-specific implementations.
"""
return headers
def get_complete_url(
self,
api_base: str | None,
model: str,
optional_params: dict,
litellm_params: dict | None = None,
**kwargs,
) -> str:
"""
Get complete URL for OCR endpoint.
Override in provider-specific implementations.
"""
raise NotImplementedError("get_complete_url must be implemented by provider")
def transform_ocr_request(
self,
model: str,
document: DocumentType,
optional_params: dict,
headers: dict,
**kwargs,
) -> OCRRequestData:
"""
Transform OCR request to provider-specific format.
Override in provider-specific implementations.
Note: By the time this method is called, any file-type documents have already
been converted to document_url/image_url format with base64 data URIs by
the preprocessing in litellm/ocr/main.py.
Args:
model: Model name
document: Document to process - always a dict with type="document_url" or type="image_url"
optional_params: Optional parameters for the request
headers: Request headers
Returns:
OCRRequestData with data and files fields
"""
raise NotImplementedError("transform_ocr_request must be implemented by provider")
async def async_transform_ocr_request(
self,
model: str,
document: DocumentType,
optional_params: dict,
headers: dict,
**kwargs,
) -> OCRRequestData:
"""
Async transform OCR request to provider-specific format.
Optional method - providers can override if they need async transformations
(e.g., Azure AI for URL-to-base64 conversion).
Default implementation falls back to sync transform_ocr_request.
Args:
model: Model name
document: Document to process (Mistral format dict, or file path, bytes, etc.)
optional_params: Optional parameters for the request
headers: Request headers
Returns:
OCRRequestData with data and files fields
"""
# Default implementation: call sync version
return self.transform_ocr_request(
model=model,
document=document,
optional_params=optional_params,
headers=headers,
**kwargs,
)
def transform_ocr_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
**kwargs,
) -> OCRResponse:
"""
Transform provider-specific OCR response to standard format.
Override in provider-specific implementations.
"""
raise NotImplementedError("transform_ocr_response must be implemented by provider")
async def async_transform_ocr_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
**kwargs,
) -> OCRResponse:
"""
Async transform provider-specific OCR response to standard format.
Optional method - providers can override if they need async transformations
(e.g., Azure Document Intelligence for async operation polling).
Default implementation falls back to sync transform_ocr_response.
Args:
model: Model name
raw_response: Raw HTTP response
logging_obj: Logging object
Returns:
OCRResponse in standard format
"""
# Default implementation: call sync version
return self.transform_ocr_response(
model=model,
raw_response=raw_response,
logging_obj=logging_obj,
**kwargs,
)
def get_error_class(
self,
error_message: str,
status_code: int,
headers: dict,
) -> Exception:
"""Get appropriate error class for the provider."""
return BaseLLMException(
status_code=status_code,
message=error_message,
headers=headers,
)

3
litellm/llms/cohere/ocr/__init__.py
View file

@ -1,3 +0,0 @@
from litellm.llms.cohere.ocr.transformation import CohereParseConfig
__all__ = ("CohereParseConfig",)

298
litellm/llms/cohere/ocr/transformation.py
View file

@ -1,298 +0,0 @@
"""Cohere Parse (`POST /v2/parse`) exposed through LiteLLM's OCR interface."""
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal
import httpx
from pydantic import BaseModel, ConfigDict, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
from litellm.exceptions import BadRequestError, UnsupportedParamsError
from litellm.llms.base_llm.ocr.transformation import (
OCR_REQUEST_FORMAT_PARAM,
BaseOCRConfig,
DocumentType,
OCRPage,
OCRPageImage,
OCRRequestData,
OCRRequestFormat,
OCRResponse,
OCRUsageInfo,
parse_ocr_request_format,
)
from litellm.llms.cohere.common_utils import CohereError
from litellm.secret_managers.main import get_secret_str
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
COHERE_API_KEY_ENV_VAR: Final = "COHERE_API_KEY"
COHERE_PARSE_API_BASE: Final = "https://api.cohere.com"
COHERE_PARSE_PATH: Final = "/v2/parse"
COHERE_PARSE_OUTPUT_FORMAT_PARAM: Final = "output_format"
COHERE_PARSE_OUTPUT_FORMATS: Final = ("markdown", "blocks")
COHERE_PARSE_DEFAULT_OUTPUT_FORMAT: Final = "markdown"
COHERE_PARSE_SUPPORTED_PARAMS: Final = (COHERE_PARSE_OUTPUT_FORMAT_PARAM, OCR_REQUEST_FORMAT_PARAM)
COHERE_PARSE_HEALTH_CHECK_IMAGE_DATA_URI: Final = (
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4//8/AAX+Av4N70a4AAAAAElFTkSuQmCC"
)
COHERE_PARSE_IMAGE_ONLY_MESSAGE: Final = (
"Cohere Parse only accepts `image_url` documents (an image URL or a base64 image data URI); "
"`document_url` and PDF inputs are not supported."
)
_NATIVE_RESPONSE_ADAPTER: Final = TypeAdapter(dict[str, object])
_BOUNDING_BOX_ADAPTER: Final = TypeAdapter(Mapping[str, object])
class _CohereParseDocument(TypedDict):
type: ReadOnly[Literal["image_url"]]
image_url: ReadOnly[str]
class _CohereParseRequestBody(TypedDict):
model: ReadOnly[str]
document: ReadOnly[_CohereParseDocument]
output_format: ReadOnly[str]
class _MarkdownPage(TypedDict):
index: ReadOnly[int]
markdown: ReadOnly[str]
images: ReadOnly[Sequence[OCRPageImage] | None]
class _BlocksPage(_MarkdownPage):
blocks: ReadOnly[Sequence[Mapping[str, object]]]
class _CohereParseMarkdown(BaseModel):
model_config = ConfigDict(frozen=True, extra="allow")
content: str = ""
images: Sequence[Mapping[str, object]] | None = None
class _CohereParsePage(BaseModel):
model_config = ConfigDict(frozen=True, extra="allow")
index: int | None = None
markdown: _CohereParseMarkdown | None = None
blocks: Sequence[Mapping[str, object]] | None = None
class _CohereParseBilledUnits(BaseModel):
model_config = ConfigDict(frozen=True, extra="allow")
pages: int | None = None
class _CohereParseMeta(BaseModel):
model_config = ConfigDict(frozen=True, extra="allow")
billed_units: _CohereParseBilledUnits | None = None
class _CohereParseResponse(BaseModel):
model_config = ConfigDict(frozen=True, extra="allow")
pages: Sequence[_CohereParsePage] = ()
meta: _CohereParseMeta | None = None
def _requested_format(optional_params: Mapping[str, object] | None) -> OCRRequestFormat:
if optional_params is None:
return "litellm"
return "native" if optional_params.get(OCR_REQUEST_FORMAT_PARAM) == "native" else "litellm"
def _page_image(image: Mapping[str, object]) -> OCRPageImage:
bounding_box: Final = image.get("bounding_box")
if not isinstance(bounding_box, Mapping):
return OCRPageImage.model_validate(image)
bbox: Final = _BOUNDING_BOX_ADAPTER.validate_python(bounding_box)
return OCRPageImage.model_validate(MappingProxyType({**image, "bbox": bbox}))
def _normalize_page(page: _CohereParsePage, position: int) -> OCRPage:
markdown: Final = page.markdown
images: Final = tuple(_page_image(image) for image in markdown.images) if markdown and markdown.images else None
normalized: Final[_MarkdownPage] = {
"index": page.index if page.index is not None else position,
"markdown": markdown.content if markdown else "",
"images": images,
}
if page.blocks is None:
return OCRPage.model_validate(normalized)
with_blocks: Final[_BlocksPage] = {**normalized, "blocks": page.blocks}
return OCRPage.model_validate(with_blocks)
def _billed_pages(parsed: _CohereParseResponse) -> int | None:
if parsed.meta is None or parsed.meta.billed_units is None:
return None
return parsed.meta.billed_units.pages
class CohereParseConfig(BaseOCRConfig):
"""Cohere Parse, an image-only document understanding endpoint returning markdown or blocks."""
def get_supported_ocr_params(self, model: str) -> list[str]: # mutable-ok: BaseOCRConfig signature
return list(COHERE_PARSE_SUPPORTED_PARAMS) # mutable-ok: BaseOCRConfig signature
def get_api_key_env_var(self) -> str | None:
return COHERE_API_KEY_ENV_VAR
def get_health_check_document(self) -> DocumentType:
return { # mutable-ok: litellm.aocr rejects any document that is not a dict
"type": "image_url",
"image_url": COHERE_PARSE_HEALTH_CHECK_IMAGE_DATA_URI,
}
def _llm_provider(self) -> str:
return "cohere"
def map_ocr_params(
self,
non_default_params: Mapping[str, object],
optional_params: Mapping[str, object],
model: str,
) -> dict[str, object]: # mutable-ok: BaseOCRConfig signature
output_format: Final = non_default_params.get(COHERE_PARSE_OUTPUT_FORMAT_PARAM)
if output_format is not None and output_format not in COHERE_PARSE_OUTPUT_FORMATS:
raise UnsupportedParamsError(
message=(
f"Invalid `{COHERE_PARSE_OUTPUT_FORMAT_PARAM}`: {output_format!r}. "
f"Expected one of {', '.join(COHERE_PARSE_OUTPUT_FORMATS)}."
),
model=model,
llm_provider=self._llm_provider(),
)
requested_format: Final = non_default_params.get(OCR_REQUEST_FORMAT_PARAM)
request_format: Final = parse_ocr_request_format(requested_format) if requested_format is not None else None
overrides: Final = tuple(
(key, value)
for key, value in (
(COHERE_PARSE_OUTPUT_FORMAT_PARAM, output_format),
(OCR_REQUEST_FORMAT_PARAM, request_format),
)
if value is not None
)
return {**optional_params, **dict(overrides)} # mutable-ok: BaseOCRConfig signature
def validate_environment(
self,
headers: Mapping[str, str],
model: str,
api_key: str | None = None,
api_base: str | None = None,
litellm_params: Mapping[str, object] | None = None,
**kwargs: object, # kwargs-ok: BaseOCRConfig.validate_environment signature
) -> dict[str, str]: # mutable-ok: BaseOCRConfig signature
resolved_key: Final = api_key or get_secret_str(COHERE_API_KEY_ENV_VAR)
if resolved_key is None:
raise ValueError(
f"Missing {COHERE_API_KEY_ENV_VAR} - set it in the environment or pass api_key to "
"litellm.ocr()/litellm.aocr()"
)
return { # mutable-ok: BaseOCRConfig signature
"Authorization": f"Bearer {resolved_key}",
"Content-Type": "application/json",
**headers,
}
def get_complete_url(
self,
api_base: str | None,
model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object] | None = None,
**kwargs: object, # kwargs-ok: BaseOCRConfig.get_complete_url signature
) -> str:
url: Final = httpx.URL(api_base or COHERE_PARSE_API_BASE)
path: Final = url.path.rstrip("/")
if path.endswith(COHERE_PARSE_PATH):
return str(url.copy_with(path=path))
if path.endswith("/v2"):
return str(url.copy_with(path=f"{path}/parse"))
return str(url.copy_with(path=f"{path}{COHERE_PARSE_PATH}"))
def _image_url(self, document: DocumentType, model: str) -> str:
image_url: Final = document.get("image_url", "")
if document.get("type") != "image_url" or not image_url or image_url.startswith("data:application/pdf"):
raise BadRequestError(
message=COHERE_PARSE_IMAGE_ONLY_MESSAGE,
model=model,
llm_provider=self._llm_provider(),
)
return image_url
def _resolve_image_url_sync(self, image_url: str) -> str:
return image_url
async def _resolve_image_url_async(self, image_url: str) -> str:
return image_url
def _build_request(self, model: str, image_url: str, optional_params: Mapping[str, object]) -> OCRRequestData:
body: Final[_CohereParseRequestBody] = {
"model": model,
"document": {"type": "image_url", "image_url": image_url},
"output_format": str(
optional_params.get(COHERE_PARSE_OUTPUT_FORMAT_PARAM, COHERE_PARSE_DEFAULT_OUTPUT_FORMAT)
),
}
return OCRRequestData(data=dict(body), files=None) # mutable-ok: OCRRequestData.data is a dict
def transform_ocr_request(
self,
model: str,
document: DocumentType,
optional_params: Mapping[str, object],
headers: Mapping[str, str],
**kwargs: object, # kwargs-ok: BaseOCRConfig.transform_ocr_request signature
) -> OCRRequestData:
image_url: Final = self._resolve_image_url_sync(self._image_url(document, model))
return self._build_request(model=model, image_url=image_url, optional_params=optional_params)
async def async_transform_ocr_request(
self,
model: str,
document: DocumentType,
optional_params: Mapping[str, object],
headers: Mapping[str, str],
**kwargs: object, # kwargs-ok: BaseOCRConfig.async_transform_ocr_request signature
) -> OCRRequestData:
image_url: Final = await self._resolve_image_url_async(self._image_url(document, model))
return self._build_request(model=model, image_url=image_url, optional_params=optional_params)
def transform_ocr_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: "LiteLLMLoggingObj",
optional_params: Mapping[str, object] | None = None,
**kwargs: object, # kwargs-ok: BaseOCRConfig.transform_ocr_response signature
) -> OCRResponse:
native: Final = _NATIVE_RESPONSE_ADAPTER.validate_python(raw_response.json())
parsed: Final = _CohereParseResponse.model_validate(native)
pages: Final = [ # mutable-ok: OCRResponse.pages is a list
_normalize_page(page, position) for position, page in enumerate(parsed.pages)
]
billed_pages: Final = _billed_pages(parsed)
response: Final = OCRResponse(
pages=pages,
model=model,
usage_info=OCRUsageInfo(pages_processed=billed_pages if billed_pages is not None else len(pages)),
)
if _requested_format(optional_params) == "native":
response.set_provider_native_response(native)
return response
def get_error_class(
self,
error_message: str,
status_code: int,
headers: Mapping[str, str],
) -> Exception:
return CohereError(status_code=status_code, message=error_message)

322
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -81,7 +81,6 @@ from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
from litellm.llms.base_llm.ocr.transformation import OCR_REQUEST_FORMAT_PARAM, BaseOCRConfig, OCRResponse
from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig
@ -1636,320 +1635,6 @@ class BaseLLMHTTPHandler:
api_key=api_key,
)
def _prepare_ocr_request(
self,
model: str,
document: dict[str, str],
optional_params: dict,
logging_obj: LiteLLMLoggingObj,
api_key: str | None,
api_base: str | None,
headers: dict[str, object] | None,
provider_config: BaseOCRConfig,
litellm_params: dict,
) -> tuple[dict[str, object], str, dict[str, object], None]:
"""
Shared logic for preparing OCR requests.
Returns: (headers, complete_url, data, files)
"""
from litellm.llms.base_llm.ocr.transformation import OCRRequestData
headers = provider_config.validate_environment(
api_key=api_key,
api_base=api_base,
headers=headers or {},
model=model,
litellm_params=litellm_params,
)
complete_url: Final = provider_config.get_complete_url(
api_base=api_base,
model=model,
optional_params=optional_params,
litellm_params=litellm_params,
)
# Transform the request to get data and files
transformed_result: Final = provider_config.transform_ocr_request(
model=model,
document=document,
optional_params={key: value for key, value in optional_params.items() if key != OCR_REQUEST_FORMAT_PARAM},
headers=headers,
api_key=api_key,
api_base=api_base,
)
# All providers return OCRRequestData
if not isinstance(transformed_result, OCRRequestData):
raise ValueError(f"Provider {provider_config.__class__.__name__} must return OCRRequestData")
# Data is always a dict for Mistral OCR format
if not isinstance(transformed_result.data, dict):
raise ValueError(f"Expected dict data for OCR request, got {type(transformed_result.data)}")
data: Final = transformed_result.data
## LOGGING
logging_obj.pre_call(
input="OCR document processing",
api_key=api_key,
additional_args={
"complete_input_dict": data,
"api_base": complete_url,
"headers": headers,
},
)
return headers, complete_url, data, None
async def _async_prepare_ocr_request(
self,
model: str,
document: dict[str, str],
optional_params: dict,
logging_obj: LiteLLMLoggingObj,
api_key: str | None,
api_base: str | None,
headers: dict[str, object] | None,
provider_config: BaseOCRConfig,
litellm_params: dict,
) -> tuple[dict[str, object], str, dict[str, object], None]:
"""
Async version of _prepare_ocr_request for providers that need async transforms.
Returns: (headers, complete_url, data, files)
"""
from litellm.llms.base_llm.ocr.transformation import OCRRequestData
headers = provider_config.validate_environment(
api_key=api_key,
api_base=api_base,
headers=headers or {},
model=model,
litellm_params=litellm_params,
)
complete_url: Final = provider_config.get_complete_url(
api_base=api_base,
model=model,
optional_params=optional_params,
litellm_params=litellm_params,
)
# Use async transform (providers can override this method if they need async operations)
transformed_result: Final = await provider_config.async_transform_ocr_request(
model=model,
document=document,
optional_params={key: value for key, value in optional_params.items() if key != OCR_REQUEST_FORMAT_PARAM},
headers=headers,
api_key=api_key,
api_base=api_base,
)
# All providers return OCRRequestData
if not isinstance(transformed_result, OCRRequestData):
raise ValueError(f"Provider {provider_config.__class__.__name__} must return OCRRequestData")
# Data is always a dict for Mistral OCR format
if not isinstance(transformed_result.data, dict):
raise ValueError(f"Expected dict data for OCR request, got {type(transformed_result.data)}")
data: Final = transformed_result.data
## LOGGING
logging_obj.pre_call(
input="OCR document processing",
api_key=api_key,
additional_args={
"complete_input_dict": data,
"api_base": complete_url,
"headers": headers,
},
)
return headers, complete_url, data, None
def _transform_ocr_response(
self,
provider_config: BaseOCRConfig,
model: str,
response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
optional_params: Mapping[str, object],
) -> OCRResponse:
"""Shared logic for transforming OCR responses."""
normalized: Final = provider_config.transform_ocr_response(
model=model,
raw_response=response,
logging_obj=logging_obj,
optional_params=optional_params,
)
return self._finalize_ocr_response(normalized, response, optional_params)
@staticmethod
def _finalize_ocr_response(
normalized: OCRResponse,
response: httpx.Response,
optional_params: Mapping[str, object],
) -> OCRResponse:
if (
optional_params.get(OCR_REQUEST_FORMAT_PARAM) == "native"
and normalized.get_provider_native_response() is None
):
normalized.set_provider_native_response(response.json())
return normalized
def ocr(
self,
model: str,
document: dict[str, str],
optional_params: dict,
timeout: float | httpx.Timeout,
logging_obj: LiteLLMLoggingObj,
api_key: str | None,
api_base: str | None,
custom_llm_provider: str,
client: HTTPHandler | AsyncHTTPHandler | None = None,
aocr: bool = False,
headers: dict[str, object] | None = None,
provider_config: BaseOCRConfig | None = None,
litellm_params: dict | None = None,
) -> OCRResponse | Coroutine[object, object, OCRResponse]:
"""
Sync OCR handler.
"""
if provider_config is None:
raise ValueError(f"No provider config found for model: {model} and provider: {custom_llm_provider}")
if litellm_params is None:
litellm_params = {}
if aocr is True:
return self.async_ocr(
model=model,
document=document,
optional_params=optional_params,
timeout=timeout,
logging_obj=logging_obj,
api_key=api_key,
api_base=api_base,
custom_llm_provider=custom_llm_provider,
client=client,
headers=headers,
provider_config=provider_config,
litellm_params=litellm_params,
)
# Prepare the request
headers, complete_url, data, files = self._prepare_ocr_request(
model=model,
document=document,
optional_params=optional_params,
logging_obj=logging_obj,
api_key=api_key,
api_base=api_base,
headers=headers,
provider_config=provider_config,
litellm_params=litellm_params,
)
if client is None or not isinstance(client, HTTPHandler):
client = _get_httpx_client()
try:
# Make the POST request with JSON data (Mistral format)
response: Final = client.post(
url=complete_url,
headers=headers,
json=data,
timeout=timeout,
)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
logging_obj.post_call(
api_key=api_key,
original_response=response.text,
additional_args={"complete_input_dict": data},
)
return self._transform_ocr_response(
provider_config=provider_config,
model=model,
response=response,
logging_obj=logging_obj,
optional_params=optional_params,
)
async def async_ocr(
self,
model: str,
document: dict[str, str],
optional_params: dict,
timeout: float | httpx.Timeout,
logging_obj: LiteLLMLoggingObj,
api_key: str | None,
api_base: str | None,
custom_llm_provider: str,
client: HTTPHandler | AsyncHTTPHandler | None = None,
headers: dict[str, object] | None = None,
provider_config: BaseOCRConfig | None = None,
litellm_params: dict | None = None,
) -> OCRResponse:
"""
Async OCR handler.
"""
if provider_config is None:
raise ValueError(f"No provider config found for model: {model} and provider: {custom_llm_provider}")
if litellm_params is None:
litellm_params = {}
# Prepare the request using async prepare method
headers, complete_url, data, files = await self._async_prepare_ocr_request(
model=model,
document=document,
optional_params=optional_params,
logging_obj=logging_obj,
api_key=api_key,
api_base=api_base,
headers=headers,
provider_config=provider_config,
litellm_params=litellm_params,
)
if client is None or not isinstance(client, AsyncHTTPHandler):
async_httpx_client = get_async_httpx_client(
llm_provider=litellm.LlmProviders(custom_llm_provider),
)
else:
async_httpx_client = client
try:
# Make the async POST request with JSON data (Mistral format)
response: Final = await async_httpx_client.post(
url=complete_url,
headers=headers,
json=data,
timeout=timeout,
)
except Exception as e:
raise self._handle_error(e=e, provider_config=provider_config)
logging_obj.post_call(
api_key=api_key,
original_response=response.text,
additional_args={"complete_input_dict": data},
)
# Use async response transform for async operations
normalized: Final = await provider_config.async_transform_ocr_response(
model=model,
raw_response=response,
logging_obj=logging_obj,
optional_params=optional_params,
)
return self._finalize_ocr_response(normalized, response, optional_params)
def search(
self,
query: str | list[str],
@ -6211,7 +5896,6 @@ class BaseLLMHTTPHandler:
BaseGoogleGenAIGenerateContentConfig,
BaseAnthropicMessagesConfig,
BaseBatchesConfig,
BaseOCRConfig,
BaseVideoConfig,
BaseSearchConfig,
BaseTextToSpeechConfig,
@ -6254,12 +5938,6 @@ class BaseLLMHTTPHandler:
status_code=status_code,
headers=error_headers,
)
if (
isinstance(provider_config, BaseOCRConfig)
and isinstance(provider_error, BaseLLMException)
and isinstance(error_response, httpx.Response)
):
provider_error.response = error_response
if not isinstance(received_status_code, int):
provider_error.status_code_is_synthesized = True
raise provider_error

244
litellm/llms/mistral/ocr/transformation.py
View file

@ -1,244 +0,0 @@
"""
Mistral OCR transformation implementation.
"""
from typing import TYPE_CHECKING, Final
import httpx
from litellm._logging import verbose_logger
from litellm.llms.base_llm.ocr.transformation import (
BaseOCRConfig,
DocumentType,
OCRRequestData,
OCRResponse,
)
from litellm.secret_managers.main import get_secret_str
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
MISTRAL_OCR_API_KEY_ENV_VAR: Final = "MISTRAL_API_KEY"
class MistralOCRConfig(BaseOCRConfig):
"""
Mistral OCR transformation configuration.
Reference: https://docs.mistral.ai/api/#tag/ocr
"""
def __init__(self) -> None:
super().__init__()
def get_supported_ocr_params(self, model: str) -> list:
"""
Get supported OCR parameters for Mistral OCR.
Mistral OCR supports:
- pages: List of page numbers to process
- include_image_base64: Whether to include base64 encoded images
- image_limit: Maximum number of images to return
- image_min_size: Minimum size of images to include
- bbox_annotation_format: Format for bounding box annotations
- document_annotation_format: Format for document annotations
- document_annotation_prompt: Prompt for document annotation extraction
- extract_header: Whether to extract document header
- extract_footer: Whether to extract document footer
- table_format: Table output format ("markdown" or "html")
- confidence_scores_granularity: Confidence score level ("word" or "page")
- include_blocks: Whether to return paragraph-level bounding boxes and typed content blocks (OCR 4)
- id: Request identifier
"""
return [
"pages",
"include_image_base64",
"image_limit",
"image_min_size",
"bbox_annotation_format",
"document_annotation_format",
"document_annotation_prompt",
"extract_header",
"extract_footer",
"table_format",
"confidence_scores_granularity",
"include_blocks",
"id",
]
def get_api_key_env_var(self) -> str | None:
return MISTRAL_OCR_API_KEY_ENV_VAR
def map_ocr_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
) -> dict:
"""
Map OCR parameters to Mistral-specific format.
Mistral accepts these parameters directly, so no transformation needed.
Just filter out unsupported params.
"""
supported_params: Final = self.get_supported_ocr_params(model=model)
# Only include params that are in the supported list
mapped_params: Final = {}
for param, value in non_default_params.items():
if param in supported_params:
mapped_params[param] = value
return mapped_params
def validate_environment(
self,
headers: dict,
model: str,
api_key: str | None = None,
api_base: str | None = None,
litellm_params: dict | None = None,
**kwargs,
) -> dict:
"""
Validate environment and return headers for Mistral OCR.
"""
# Get API key from environment if not provided
if api_key is None:
api_key = get_secret_str(MISTRAL_OCR_API_KEY_ENV_VAR)
if api_key is None:
raise ValueError(
"Missing Mistral API Key - A call is being made to Mistral but no key is set either in the environment variables or via params"
)
headers = {
"Authorization": f"Bearer {api_key}",
**headers,
}
# Don't set Content-Type for multipart/form-data - httpx will handle it
return headers
def get_complete_url(
self,
api_base: str | None,
model: str,
optional_params: dict,
litellm_params: dict | None = None,
**kwargs,
) -> str:
"""
Get complete URL for Mistral OCR endpoint.
Returns: https://api.mistral.ai/v1/ocr
"""
if api_base is None:
api_base = "https://api.mistral.ai/v1"
# Ensure no trailing slash
api_base = api_base.rstrip("/")
# Remove /v1 if it's already in the base to avoid duplication
if api_base.endswith("/v1"):
return f"{api_base}/ocr"
return f"{api_base}/v1/ocr"
def transform_ocr_request(
self,
model: str,
document: DocumentType,
optional_params: dict,
headers: dict,
**kwargs,
) -> OCRRequestData:
"""
Transform OCR request to Mistral-specific format.
Mistral OCR API accepts:
{
"model": "mistral-ocr-latest",
"document": {
"type": "document_url",
"document_url": "<https-url or data-uri>"
},
"pages": [0], # optional
"include_image_base64": false, # optional
...
}
Args:
model: Model name (e.g., "mistral-ocr-latest")
document: Document dict from user (Mistral format) - already validated in main.py
optional_params: Already mapped optional parameters
headers: Request headers
Returns:
OCRRequestData with JSON data
"""
verbose_logger.debug("Mistral OCR transform_ocr_request - model: %s", model)
# Document parameter is the Mistral-format dict from the user
# Just pass it through as-is to the Mistral API
if not isinstance(document, dict):
raise ValueError(f"Expected document dict, got {type(document)}")
# Build request data - use document dict directly
data: Final = {
"model": model,
"document": document, # Pass through the Mistral-format document dict
}
# Add all optional parameters from the already-mapped optional_params
data.update(optional_params)
# No multipart files - using JSON
return OCRRequestData(data=data, files=None)
def transform_ocr_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: "LiteLLMLoggingObj",
**kwargs,
) -> OCRResponse:
"""
Return Mistral OCR response in native format.
Mistral OCR is the standard format for LiteLLM OCR responses.
No transformation needed - return native response.
Mistral OCR returns:
{
"pages": [
{
"index": 0,
"markdown": "extracted text content",
"images": [...],
"dimensions": {...}
},
...
],
"model": "mistral-ocr-2505-completion",
"document_annotation": null,
"usage_info": {...}
}
"""
try:
response_json: Final = raw_response.json()
verbose_logger.debug("Mistral OCR response keys: %s", response_json.keys())
# Return native Mistral format - no transformation
return OCRResponse(
pages=response_json.get("pages", []),
model=response_json.get("model", model),
document_annotation=response_json.get("document_annotation"),
usage_info=response_json.get("usage_info"),
object="ocr",
)
except Exception as e:
verbose_logger.error("Error parsing Mistral OCR response: %s", e)
raise e

149
litellm/llms/reducto/common.py
View file

@ -1,149 +0,0 @@
import base64
import binascii
from collections import defaultdict
from typing import TYPE_CHECKING, Any, Final, NoReturn
import httpx
from litellm.constants import request_timeout
REDUCTO_API_BASE: Final = "https://platform.reducto.ai"
REDUCTO_ID_PREFIX: Final = "reducto://"
if TYPE_CHECKING:
from litellm.llms.base_llm.ocr.transformation import OCRPage
def _normalize_api_base(api_base: str | None) -> str:
return (api_base or REDUCTO_API_BASE).rstrip("/")
def _raise_bad_request(message: str, model: str) -> NoReturn:
import litellm
raise litellm.BadRequestError(
message=message,
model=model,
llm_provider="reducto",
)
def extract_file_id_or_bytes(
source_url: str,
model: str,
) -> tuple[str | None, bytes | None, str | None]:
if source_url.startswith(REDUCTO_ID_PREFIX):
return source_url, None, None
if source_url.startswith("http://") or source_url.startswith("https://"):
_raise_bad_request(
"Reducto requires type='file' (auto-uploaded) or a reducto:// id. Plain http(s) URLs are not supported; upload the file first.",
model=model,
)
if not source_url.startswith("data:"):
_raise_bad_request(
"Reducto requires a reducto:// id or a base64 data URI after OCR preprocessing.",
model=model,
)
try:
header, encoded = source_url.split(",", 1)
except ValueError:
_raise_bad_request("Invalid Reducto data URI provided.", model=model)
if ";base64" not in header:
_raise_bad_request("Reducto only supports base64-encoded data URIs.", model=model)
mime: Final = header.removeprefix("data:").split(";")[0] or "application/octet-stream"
try:
raw_bytes: Final = base64.b64decode(encoded, validate=True)
except (binascii.Error, ValueError):
_raise_bad_request("Invalid Reducto base64 payload provided.", model=model)
return None, raw_bytes, mime
def _extract_file_id_from_upload_response(response: httpx.Response) -> str:
try:
payload: Final = response.json()
except ValueError as exc:
raise ValueError(f"Reducto /upload returned a non-JSON 200 response: {response.text}") from exc
file_id: Final = (payload or {}).get("file_id") if isinstance(payload, dict) else None
if not isinstance(file_id, str) or not file_id:
raise ValueError(f"Reducto /upload returned 200 without a file_id; got payload={payload}")
return file_id
def upload_bytes_sync(
raw_bytes: bytes,
mime: str | None,
api_key: str,
api_base: str | None,
) -> str:
import litellm
response: Final = litellm.module_level_client.post(
url="{}{}".format(_normalize_api_base(api_base), "/upload"),
headers={"Authorization": f"Bearer {api_key}"},
files={"file": ("document", raw_bytes, mime or "application/octet-stream")},
timeout=request_timeout,
)
response.raise_for_status()
return _extract_file_id_from_upload_response(response)
async def upload_bytes_async(
raw_bytes: bytes,
mime: str | None,
api_key: str,
api_base: str | None,
) -> str:
import litellm
response: Final = await litellm.module_level_aclient.post(
url="{}{}".format(_normalize_api_base(api_base), "/upload"),
headers={"Authorization": f"Bearer {api_key}"},
files={"file": ("document", raw_bytes, mime or "application/octet-stream")},
timeout=request_timeout,
)
response.raise_for_status()
return _extract_file_id_from_upload_response(response)
def build_pages_from_reducto(result: dict[str, Any]) -> list["OCRPage"]:
from litellm.llms.base_llm.ocr.transformation import OCRPage
chunks: Final = result.get("chunks", []) or []
blocks_by_page: Final[dict[int, list[dict[str, Any]]]] = defaultdict(list)
for chunk in chunks:
for block in chunk.get("blocks", []) or []:
page_no = (block.get("bbox") or {}).get("page")
if page_no is None:
continue
try:
normalized_page = int(page_no)
except (TypeError, ValueError):
continue
blocks_by_page[normalized_page].append(block)
if not blocks_by_page:
fallback_markdown: Final = "\n\n".join(chunk.get("content", "") for chunk in chunks if chunk.get("content"))
if fallback_markdown == "":
return []
return [OCRPage(index=0, markdown=fallback_markdown)]
pages: Final[list[OCRPage]] = []
for page_no, blocks in sorted(blocks_by_page.items()):
markdown = "\n\n".join(block.get("content", "") for block in blocks if block.get("content"))
page_index = max(page_no - 1, 0)
page = OCRPage(
index=page_index,
markdown=markdown,
)
# OCRPage accepts extra keys at runtime; assign blocks after construction
# so static typing does not reject provider-specific metadata.
setattr(page, "blocks", blocks)
pages.append(page)
return pages

1
litellm/llms/reducto/ocr/__init__.py
View file

@ -1 +0,0 @@

236
litellm/llms/reducto/ocr/transformation.py
View file

@ -1,236 +0,0 @@
from typing import TYPE_CHECKING, Any, Final
import httpx
from litellm.llms.base_llm.ocr.transformation import (
BaseOCRConfig,
DocumentType,
OCRRequestData,
OCRResponse,
OCRUsageInfo,
)
from litellm.llms.reducto.common import (
REDUCTO_API_BASE,
build_pages_from_reducto,
extract_file_id_or_bytes,
upload_bytes_async,
upload_bytes_sync,
)
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
class _BaseReductoOCRConfig(BaseOCRConfig):
def map_ocr_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
) -> dict:
mapped_params: Final = dict(optional_params)
supported_params: Final = self.get_supported_ocr_params(model=model)
for param, value in non_default_params.items():
if param in supported_params:
mapped_params[param] = value
return mapped_params
def validate_environment(
self,
headers: dict,
model: str,
api_key: str | None = None,
api_base: str | None = None,
litellm_params: dict | None = None,
**kwargs,
) -> dict:
from litellm.secret_managers.main import get_secret_str
resolved_key: Final = api_key or get_secret_str("REDUCTO_API_KEY")
if resolved_key is None:
raise ValueError(
"Missing REDUCTO_API_KEY - set it in the environment or pass api_key to litellm.ocr()/litellm.aocr()"
)
return {
"Authorization": f"Bearer {resolved_key}",
"Content-Type": "application/json",
**headers,
}
def get_complete_url(
self,
api_base: str | None,
model: str,
optional_params: dict,
litellm_params: dict | None = None,
**kwargs,
) -> str:
return "{}/parse".format((api_base or REDUCTO_API_BASE).rstrip("/"))
def _get_source_url(self, document: DocumentType, model: str) -> str:
source_url: Final = document.get("document_url") or document.get("image_url")
if source_url is None:
raise ValueError(
f"Reducto expected OCR preprocessing to produce document_url or image_url for model={model}"
)
return source_url
@staticmethod
def _resolve_credentials(api_key: str | None, api_base: str | None) -> tuple[str, str]:
from litellm.secret_managers.main import get_secret_str
resolved_key: Final = api_key or get_secret_str("REDUCTO_API_KEY")
if resolved_key is None:
raise ValueError(
"Missing REDUCTO_API_KEY - set it in the environment or pass api_key to litellm.ocr()/litellm.aocr()"
)
resolved_base: Final = (api_base or REDUCTO_API_BASE).rstrip("/")
return resolved_key, resolved_base
def _ensure_file_id_sync(
self,
model: str,
document: DocumentType,
api_key: str | None,
api_base: str | None,
) -> str:
source_url: Final = self._get_source_url(document=document, model=model)
file_id, raw_bytes, mime = extract_file_id_or_bytes(source_url, model=model)
if file_id is not None:
return file_id
resolved_key, resolved_base = self._resolve_credentials(api_key, api_base)
return upload_bytes_sync(
raw_bytes=raw_bytes or b"",
mime=mime,
api_key=resolved_key,
api_base=resolved_base,
)
async def _ensure_file_id_async(
self,
model: str,
document: DocumentType,
api_key: str | None,
api_base: str | None,
) -> str:
source_url: Final = self._get_source_url(document=document, model=model)
file_id, raw_bytes, mime = extract_file_id_or_bytes(source_url, model=model)
if file_id is not None:
return file_id
resolved_key, resolved_base = self._resolve_credentials(api_key, api_base)
return await upload_bytes_async(
raw_bytes=raw_bytes or b"",
mime=mime,
api_key=resolved_key,
api_base=resolved_base,
)
def transform_ocr_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: "LiteLLMLoggingObj",
**kwargs,
) -> OCRResponse:
response_json: Final = raw_response.json()
result: Final = response_json.get("result", response_json) or {}
usage: Final = response_json.get("usage", {}) or {}
response: Final = OCRResponse(
pages=build_pages_from_reducto(result),
model=model,
usage_info=OCRUsageInfo(
pages_processed=usage.get("num_pages"),
credits=usage.get("credits"),
),
object="ocr",
)
response._hidden_params["reducto_raw"] = response_json
return response
class ReductoParseV3Config(_BaseReductoOCRConfig):
def get_supported_ocr_params(self, model: str) -> list:
return ["formatting", "retrieval", "settings"]
def transform_ocr_request(
self,
model: str,
document: DocumentType,
optional_params: dict,
headers: dict,
**kwargs,
) -> OCRRequestData:
file_id: Final = self._ensure_file_id_sync(
model=model,
document=document,
api_key=kwargs.get("api_key"),
api_base=kwargs.get("api_base"),
)
return OCRRequestData(data={"input": file_id, **optional_params}, files=None)
async def async_transform_ocr_request(
self,
model: str,
document: DocumentType,
optional_params: dict,
headers: dict,
**kwargs,
) -> OCRRequestData:
file_id: Final = await self._ensure_file_id_async(
model=model,
document=document,
api_key=kwargs.get("api_key"),
api_base=kwargs.get("api_base"),
)
return OCRRequestData(data={"input": file_id, **optional_params}, files=None)
class ReductoParseLegacyConfig(_BaseReductoOCRConfig):
def get_supported_ocr_params(self, model: str) -> list:
return ["enhance"]
def _build_legacy_body(self, file_id: str, optional_params: dict) -> dict[str, Any]:
body: Final[dict[str, Any]] = {"document_url": file_id}
enhance: Final = optional_params.get("enhance")
if enhance is not None:
body["options"] = {"enhance": enhance}
return body
def transform_ocr_request(
self,
model: str,
document: DocumentType,
optional_params: dict,
headers: dict,
**kwargs,
) -> OCRRequestData:
file_id: Final = self._ensure_file_id_sync(
model=model,
document=document,
api_key=kwargs.get("api_key"),
api_base=kwargs.get("api_base"),
)
return OCRRequestData(
data=self._build_legacy_body(file_id=file_id, optional_params=optional_params),
files=None,
)
async def async_transform_ocr_request(
self,
model: str,
document: DocumentType,
optional_params: dict,
headers: dict,
**kwargs,
) -> OCRRequestData:
file_id: Final = await self._ensure_file_id_async(
model=model,
document=document,
api_key=kwargs.get("api_key"),
api_base=kwargs.get("api_base"),
)
return OCRRequestData(
data=self._build_legacy_body(file_id=file_id, optional_params=optional_params),
files=None,
)

5
litellm/llms/vertex_ai/ocr/__init__.py
View file

@ -1,5 +0,0 @@
"""Vertex AI OCR module."""
from .transformation import VertexAIOCRConfig
__all__ = ["VertexAIOCRConfig"]

41
litellm/llms/vertex_ai/ocr/common_utils.py
View file

@ -1,41 +0,0 @@
"""
Common utilities for Vertex AI OCR providers.
This module provides routing logic to determine which OCR configuration to use
based on the model name.
"""
from typing import TYPE_CHECKING, Optional
if TYPE_CHECKING:
from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig
def get_vertex_ai_ocr_config(model: str) -> Optional["BaseOCRConfig"]:
"""
Determine which Vertex AI OCR configuration to use based on the model name.
Vertex AI supports multiple OCR services:
- Vertex AI OCR: vertex_ai/<model>
Args:
model: The model name (e.g., "vertex_ai/ocr/<model>")
Returns:
OCR configuration instance for the specified model
Examples:
>>> get_vertex_ai_ocr_config("vertex_ai/deepseek-ai/deepseek-ocr-maas")
<VertexAIDeepSeekOCRConfig object>
>>> get_vertex_ai_ocr_config("vertex_ai/ocr/mistral-ocr-maas")
<VertexAIOCRConfig object>
"""
from litellm.llms.vertex_ai.ocr.deepseek_transformation import (
VertexAIDeepSeekOCRConfig,
)
from litellm.llms.vertex_ai.ocr.transformation import VertexAIOCRConfig
if "deepseek" in model:
return VertexAIDeepSeekOCRConfig()
return VertexAIOCRConfig()

378
litellm/llms/vertex_ai/ocr/deepseek_transformation.py
View file

@ -1,378 +0,0 @@
"""
Vertex AI DeepSeek OCR transformation implementation.
"""
import json
from typing import TYPE_CHECKING, Any, Final
import httpx
from litellm._logging import verbose_logger
from litellm.llms.base_llm.ocr.transformation import (
BaseOCRConfig,
DocumentType,
OCRPage,
OCRRequestData,
OCRResponse,
OCRUsageInfo,
)
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
VERTEX_AI_DEEPSEEK_OCR_API_KEY_ENV_VAR: Final = "VERTEX_AI_API_KEY"
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
class VertexAIDeepSeekOCRConfig(BaseOCRConfig):
"""
Vertex AI DeepSeek OCR transformation configuration.
This transformation converts standard LiteLLM OCR requests to the
Vertex AI DeepSeek OCR OpenAPI endpoint shape and normalizes the response.
"""
def __init__(self) -> None:
super().__init__()
self.vertex_base = VertexBase()
def get_api_key_env_var(self) -> str | None:
return VERTEX_AI_DEEPSEEK_OCR_API_KEY_ENV_VAR
def validate_environment(
self,
headers: dict,
model: str,
api_key: str | None = None,
api_base: str | None = None,
litellm_params: dict | None = None,
**kwargs,
) -> dict:
"""
Validate environment and return headers for Vertex AI OCR.
Vertex AI uses Bearer token authentication with access token from credentials.
"""
if api_key is not None:
return {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
**headers,
}
# Extract Vertex AI parameters using safe helpers from VertexBase
# Use safe_get_* methods that don't mutate litellm_params dict
litellm_params = litellm_params or {}
vertex_project: Final = VertexBase.safe_get_vertex_ai_project(litellm_params=litellm_params)
vertex_credentials: Final = VertexBase.safe_get_vertex_ai_credentials(litellm_params=litellm_params)
# Get access token from Vertex credentials
access_token, project_id = self.vertex_base.get_access_token(
credentials=vertex_credentials,
project_id=vertex_project,
)
headers = {
"Authorization": f"Bearer {access_token}",
"Content-Type": "application/json",
**headers,
}
return headers
def get_complete_url(
self,
api_base: str | None,
model: str,
optional_params: dict,
litellm_params: dict | None = None,
**kwargs,
) -> str:
"""
Get complete URL for Vertex AI DeepSeek OCR endpoint.
Args:
api_base: Vertex AI API base URL (optional)
model: Model name (e.g., "deepseek-ai/deepseek-ocr-maas")
optional_params: Optional parameters
litellm_params: LiteLLM parameters containing vertex_project, vertex_location
Returns: Complete URL for Vertex AI OCR endpoint
"""
# Extract Vertex AI parameters using safe helpers from VertexBase
# Use safe_get_* methods that don't mutate litellm_params dict
litellm_params = litellm_params or {}
vertex_project: Final = VertexBase.safe_get_vertex_ai_project(litellm_params=litellm_params)
vertex_location = VertexBase.safe_get_vertex_ai_location(litellm_params=litellm_params)
if vertex_project is None:
raise ValueError(
"Missing vertex_project - Set VERTEXAI_PROJECT environment variable or pass vertex_project parameter"
)
if vertex_location is None:
vertex_location = "us-central1"
# Get API base URL
if api_base is None:
api_base = "https://aiplatform.googleapis.com"
# Ensure no trailing slash
api_base = api_base.rstrip("/")
return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/endpoints/openapi/chat/completions"
def transform_ocr_request(
self,
model: str,
document: DocumentType,
optional_params: dict,
headers: dict,
**kwargs,
) -> OCRRequestData:
"""
Transform OCR request for Vertex AI DeepSeek OCR.
Converts OCR document format to the Vertex AI DeepSeek OCR payload:
- Input: {"type": "image_url", "image_url": "gs://..."}
- Output: {"model": "deepseek-ai/deepseek-ocr-maas", "messages": [{"role": "user", "content": [{"type": "image_url", "image_url": "gs://..."}]}]}
Args:
model: Model name (e.g., "deepseek-ai/deepseek-ocr-maas")
document: Document dict from user (Mistral OCR format)
optional_params: Already mapped optional parameters
headers: Request headers
**kwargs: Additional arguments
Returns:
OCRRequestData with JSON data for the DeepSeek OCR endpoint
"""
verbose_logger.debug("Vertex AI DeepSeek OCR transform_ocr_request (sync) called")
if not isinstance(document, dict):
raise ValueError(f"Expected document dict, got {type(document)}")
# Extract document type and URL
doc_type: Final = document.get("type")
image_url = None
document_url = None
if doc_type == "image_url":
image_url = document.get("image_url", "")
elif doc_type == "document_url":
document_url = document.get("document_url", "")
else:
raise ValueError(f"Unsupported document type: {doc_type}. Expected 'image_url' or 'document_url'")
# Build DeepSeek OCR message content
content_item = {}
if image_url:
content_item = {"type": "image_url", "image_url": image_url}
elif document_url:
# For document URLs, we use image_url type as well (Vertex AI supports both)
content_item = {"type": "image_url", "image_url": document_url}
# Build DeepSeek OCR request
provider_model: Final = model if model.startswith("deepseek-ai/") else f"deepseek-ai/{model}"
data: Final = {
"model": provider_model,
"messages": [{"role": "user", "content": [content_item]}],
}
# Add optional parameters (stream, temperature, etc.)
deepseek_ocr_params: Final = {}
for key, value in optional_params.items():
if key in ["stream", "temperature", "max_tokens", "top_p", "n", "stop"]:
deepseek_ocr_params[key] = value
data.update(deepseek_ocr_params)
verbose_logger.debug("Vertex AI DeepSeek OCR: Transformed request")
return OCRRequestData(data=data, files=None)
async def async_transform_ocr_request(
self,
model: str,
document: DocumentType,
optional_params: dict,
headers: dict,
**kwargs,
) -> OCRRequestData:
"""
Transform OCR request for Vertex AI DeepSeek OCR (async).
Same as sync version - no async-specific logic needed.
Args:
model: Model name
document: Document dict from user
optional_params: Already mapped optional parameters
headers: Request headers
**kwargs: Additional arguments
Returns:
OCRRequestData with JSON data for the DeepSeek OCR endpoint
"""
return self.transform_ocr_request(
model=model,
document=document,
optional_params=optional_params,
headers=headers,
**kwargs,
)
def transform_ocr_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
**kwargs,
) -> OCRResponse:
"""
Transform Vertex AI DeepSeek OCR response to OCR format.
Vertex AI DeepSeek OCR returns an OpenAPI response:
{
"id": "...",
"choices": [{
"message": {
"role": "assistant",
"content": "<OCR result as JSON string or markdown>"
}
}],
"usage": {...}
}
We need to extract the content and convert it to OCRResponse format.
Args:
model: Model name
raw_response: Raw HTTP response from Vertex AI
logging_obj: Logging object
**kwargs: Additional arguments
Returns:
OCRResponse in standard format
"""
verbose_logger.debug("Vertex AI DeepSeek OCR transform_ocr_response called")
verbose_logger.debug("Raw response: %s", raw_response.text)
try:
response_json: Final = raw_response.json()
# Extract OCR content from provider response
choices: Final = response_json.get("choices", [])
if not choices:
raise ValueError("No choices in DeepSeek OCR response")
message: Final = choices[0].get("message", {})
content: Final = message.get("content", "")
if not content:
raise ValueError("No content in DeepSeek OCR response")
# Try to parse content as JSON (OCR result might be JSON string)
ocr_data = None
try:
# If content is a JSON string, parse it
if isinstance(content, str) and content.strip().startswith("{"):
ocr_data = json.loads(content)
elif isinstance(content, dict):
ocr_data = content
else:
# If content is markdown text, create a single page with the markdown
ocr_data = {
"pages": [{"index": 0, "markdown": content}],
"model": model,
"usage_info": response_json.get("usage", {}),
}
except json.JSONDecodeError:
# If JSON parsing fails, treat content as markdown
ocr_data = {
"pages": [{"index": 0, "markdown": content}],
"model": model,
"usage_info": response_json.get("usage", {}),
}
# Ensure we have the expected structure
if "pages" not in ocr_data:
# If OCR data doesn't have pages, wrap the content in a page
ocr_data = {
"pages": [
{
"index": 0,
"markdown": (content if isinstance(content, str) else json.dumps(content)),
}
],
"model": ocr_data.get("model", model),
"usage_info": ocr_data.get("usage_info", response_json.get("usage", {})),
}
# Convert usage info if present
usage_info = None
if "usage_info" in ocr_data:
usage_dict: Final = ocr_data["usage_info"]
if isinstance(usage_dict, dict):
usage_info = OCRUsageInfo(**usage_dict)
# Build OCRResponse
pages = []
for page_data in ocr_data.get("pages", []):
# Ensure page has required fields
if isinstance(page_data, dict):
page = OCRPage(
index=page_data.get("index", 0),
markdown=page_data.get("markdown", ""),
images=page_data.get("images"),
dimensions=page_data.get("dimensions"),
)
pages.append(page)
if not pages:
# Create a default page if none exist
pages = [OCRPage(index=0, markdown=content if isinstance(content, str) else "")]
return OCRResponse(
pages=pages,
model=ocr_data.get("model", model),
document_annotation=ocr_data.get("document_annotation"),
usage_info=usage_info,
object="ocr",
)
except Exception as e:
verbose_logger.error("Error parsing Vertex AI DeepSeek OCR response: %s", e)
raise e
async def async_transform_ocr_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
**kwargs,
) -> OCRResponse:
"""
Async transform Vertex AI DeepSeek OCR response to OCR format.
Same as sync version - no async-specific logic needed.
Args:
model: Model name
raw_response: Raw HTTP response
logging_obj: Logging object
**kwargs: Additional arguments
Returns:
OCRResponse in standard format
"""
return self.transform_ocr_response(
model=model,
raw_response=raw_response,
logging_obj=logging_obj,
**kwargs,
)

288
litellm/llms/vertex_ai/ocr/transformation.py
View file

@ -1,288 +0,0 @@
"""
Vertex AI Mistral OCR transformation implementation.
"""
from typing import Final
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.image_handling import (
async_convert_url_to_base64,
convert_url_to_base64,
)
from litellm.llms.base_llm.ocr.transformation import DocumentType, OCRRequestData
from litellm.llms.mistral.ocr.transformation import MistralOCRConfig
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
VERTEX_AI_OCR_API_KEY_ENV_VAR: Final = "VERTEX_AI_API_KEY"
class VertexAIOCRConfig(MistralOCRConfig):
"""
Vertex AI Mistral OCR transformation configuration.
Vertex AI uses Mistral's OCR API format through the Mistral publisher endpoint.
Inherits transformation logic from MistralOCRConfig since they use the same format.
Reference: Vertex AI Mistral OCR documentation
Important: Vertex AI OCR only supports base64 data URIs (data:image/..., data:application/pdf;base64,...).
Regular URLs are not supported.
"""
def __init__(self) -> None:
super().__init__()
self.vertex_base = VertexBase()
def get_api_key_env_var(self) -> str | None:
return VERTEX_AI_OCR_API_KEY_ENV_VAR
def validate_environment(
self,
headers: dict,
model: str,
api_key: str | None = None,
api_base: str | None = None,
litellm_params: dict | None = None,
**kwargs,
) -> dict:
"""
Validate environment and return headers for Vertex AI OCR.
Vertex AI uses Bearer token authentication with access token from credentials.
"""
if api_key is not None:
return {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
**headers,
}
# Extract Vertex AI parameters using safe helpers from VertexBase
# Use safe_get_* methods that don't mutate litellm_params dict
litellm_params = litellm_params or {}
vertex_project: Final = VertexBase.safe_get_vertex_ai_project(litellm_params=litellm_params)
vertex_credentials: Final = VertexBase.safe_get_vertex_ai_credentials(litellm_params=litellm_params)
# Get access token from Vertex credentials
access_token, project_id = self.vertex_base.get_access_token(
credentials=vertex_credentials,
project_id=vertex_project,
)
headers = {
"Authorization": f"Bearer {access_token}",
"Content-Type": "application/json",
**headers,
}
return headers
def get_complete_url(
self,
api_base: str | None,
model: str,
optional_params: dict,
litellm_params: dict | None = None,
**kwargs,
) -> str:
"""
Get complete URL for Vertex AI OCR endpoint.
Vertex AI endpoint format:
https://{location}-aiplatform.googleapis.com/v1/projects/{project}/locations/{location}/publishers/mistralai/ocr
Args:
api_base: Vertex AI API base URL (optional)
model: Model name (not used in URL construction)
optional_params: Optional parameters
litellm_params: LiteLLM parameters containing vertex_project, vertex_location
Returns: Complete URL for Vertex AI OCR endpoint
"""
# Extract Vertex AI parameters using safe helpers from VertexBase
# Use safe_get_* methods that don't mutate litellm_params dict
litellm_params = litellm_params or {}
vertex_project: Final = VertexBase.safe_get_vertex_ai_project(litellm_params=litellm_params)
vertex_location = VertexBase.safe_get_vertex_ai_location(litellm_params=litellm_params)
if vertex_project is None:
raise ValueError(
"Missing vertex_project - Set VERTEXAI_PROJECT environment variable or pass vertex_project parameter"
)
if vertex_location is None:
vertex_location = "us-central1"
# Get API base URL
if api_base is None:
api_base = get_vertex_base_url(vertex_location)
# Ensure no trailing slash
api_base = api_base.rstrip("/")
# Vertex AI OCR endpoint format for Mistral publisher
# Format: https://{region}-aiplatform.googleapis.com/v1/projects/{project}/locations/{region}/publishers/mistralai/models/{model}:rawPredict
return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/mistralai/models/{model}:rawPredict"
def _convert_url_to_data_uri_sync(self, url: str) -> str:
"""
Synchronously convert a URL to a base64 data URI.
Vertex AI OCR doesn't have internet access, so we need to fetch URLs
and convert them to base64 data URIs.
Args:
url: The URL to convert
Returns:
Base64 data URI string
"""
verbose_logger.debug("Vertex AI OCR: Converting URL to base64 data URI (sync): %s", url)
# Fetch and convert to base64 data URI
# convert_url_to_base64 already returns a full data URI like "data:image/jpeg;base64,..."
data_uri: Final = convert_url_to_base64(url=url)
verbose_logger.debug("Vertex AI OCR: Converted URL to data URI (length: %s)", len(data_uri))
return data_uri
async def _convert_url_to_data_uri_async(self, url: str) -> str:
"""
Asynchronously convert a URL to a base64 data URI.
Vertex AI OCR doesn't have internet access, so we need to fetch URLs
and convert them to base64 data URIs.
Args:
url: The URL to convert
Returns:
Base64 data URI string
"""
verbose_logger.debug("Vertex AI OCR: Converting URL to base64 data URI (async): %s", url)
# Fetch and convert to base64 data URI asynchronously
# async_convert_url_to_base64 already returns a full data URI like "data:image/jpeg;base64,..."
data_uri: Final = await async_convert_url_to_base64(url=url)
verbose_logger.debug("Vertex AI OCR: Converted URL to data URI (length: %s)", len(data_uri))
return data_uri
def transform_ocr_request(
self,
model: str,
document: DocumentType,
optional_params: dict,
headers: dict,
**kwargs,
) -> OCRRequestData:
"""
Transform OCR request for Vertex AI, converting URLs to base64 data URIs (sync).
Vertex AI OCR doesn't have internet access, so we automatically fetch
any URLs and convert them to base64 data URIs synchronously.
Args:
model: Model name
document: Document dict from user
optional_params: Already mapped optional parameters
headers: Request headers
**kwargs: Additional arguments
Returns:
OCRRequestData with JSON data
"""
verbose_logger.debug("Vertex AI OCR transform_ocr_request (sync) called")
if not isinstance(document, dict):
raise ValueError(f"Expected document dict, got {type(document)}")
# Check if we need to convert URL to base64
doc_type: Final = document.get("type")
transformed_document: Final = document.copy()
if doc_type == "document_url":
document_url: Final = document.get("document_url", "")
# If it's not already a data URI, convert it
if document_url and not document_url.startswith("data:"):
verbose_logger.debug("Vertex AI OCR: Converting document URL to base64 data URI (sync)")
data_uri = self._convert_url_to_data_uri_sync(url=document_url)
transformed_document["document_url"] = data_uri
elif doc_type == "image_url":
image_url: Final = document.get("image_url", "")
# If it's not already a data URI, convert it
if image_url and not image_url.startswith("data:"):
verbose_logger.debug("Vertex AI OCR: Converting image URL to base64 data URI (sync)")
data_uri = self._convert_url_to_data_uri_sync(url=image_url)
transformed_document["image_url"] = data_uri
# Call parent's transform to build the request
return super().transform_ocr_request(
model=model,
document=transformed_document,
optional_params=optional_params,
headers=headers,
**kwargs,
)
async def async_transform_ocr_request(
self,
model: str,
document: DocumentType,
optional_params: dict,
headers: dict,
**kwargs,
) -> OCRRequestData:
"""
Transform OCR request for Vertex AI, converting URLs to base64 data URIs (async).
Vertex AI OCR doesn't have internet access, so we automatically fetch
any URLs and convert them to base64 data URIs asynchronously.
Args:
model: Model name
document: Document dict from user
optional_params: Already mapped optional parameters
headers: Request headers
**kwargs: Additional arguments
Returns:
OCRRequestData with JSON data
"""
verbose_logger.debug("Vertex AI OCR async_transform_ocr_request - model: %s", model)
if not isinstance(document, dict):
raise ValueError(f"Expected document dict, got {type(document)}")
# Check if we need to convert URL to base64
doc_type: Final = document.get("type")
transformed_document: Final = document.copy()
if doc_type == "document_url":
document_url: Final = document.get("document_url", "")
# If it's not already a data URI, convert it
if document_url and not document_url.startswith("data:"):
verbose_logger.debug("Vertex AI OCR: Converting document URL to base64 data URI (async)")
data_uri = await self._convert_url_to_data_uri_async(url=document_url)
transformed_document["document_url"] = data_uri
elif doc_type == "image_url":
image_url: Final = document.get("image_url", "")
# If it's not already a data URI, convert it
if image_url and not image_url.startswith("data:"):
verbose_logger.debug("Vertex AI OCR: Converting image URL to base64 data URI (async)")
data_uri = await self._convert_url_to_data_uri_async(url=image_url)
transformed_document["image_url"] = data_uri
# Call parent's transform to build the request
return super().transform_ocr_request(
model=model,
document=transformed_document,
optional_params=optional_params,
headers=headers,
**kwargs,
)

23
litellm/ocr/dispatch.py
View file

@ -1,16 +1,15 @@
from collections.abc import Awaitable, Callable, Coroutine, Mapping
from typing import Final, cast # noqa: TID251 # native binding selects a sync result or an async awaitable
from collections.abc import Coroutine, Mapping
from typing import Final
import httpx
from litellm.llms.base_llm.ocr.transformation import OCRResponse
from litellm.ocr import main
from litellm.ocr.main import convert_file_document_to_url_document, get_mime_type
from litellm.rust_bridge import runtime
from litellm.rust_bridge.catalog import Route, RouteContext
from litellm.rust_bridge.dispatch import PublicDispatch, call_hook
from litellm.rust_bridge.ocr.entrypoints import NATIVE_AOCR, NATIVE_OCR, LiteLLMOcrRequest
__all__ = ("aocr", "convert_file_document_to_url_document", "get_mime_type", "ocr")
__all__ = ("aocr", "ocr")
def _bind_request(
@ -42,16 +41,6 @@ def _public_request(name: str, args: tuple[object, ...], kwargs: Mapping[str, ob
raise TypeError(str(error).replace("_bind_request()", f"{name}()")) from None
_PYTHON_OCR: Final = cast( # cast-ok: forward the original call shape through the Python @client decorator
Callable[..., OCRResponse | Coroutine[object, object, OCRResponse]],
main.ocr, # noqa: TID251 # dispatch boundary owns this Python fallback
)
_PYTHON_AOCR: Final = cast( # cast-ok: forward the original call shape through the Python @client decorator
Callable[..., Awaitable[OCRResponse]],
main.aocr, # noqa: TID251 # dispatch boundary owns this Python fallback
)
def _context(request: LiteLLMOcrRequest) -> RouteContext:
prefix, separator, _ = request.model.partition("/")
provider: Final = request.custom_llm_provider or (prefix if separator else None)
@ -79,7 +68,7 @@ def ocr(
return _DISPATCH.run(
args,
kwargs,
python=_PYTHON_OCR,
python=runtime.NO_PYTHON,
binding=NATIVE_OCR,
native=call_hook,
)
@ -89,7 +78,7 @@ async def aocr(*args: object, **kwargs: object) -> OCRResponse: # kwargs-ok: pr
return await _ADISPATCH.arun(
args,
kwargs,
python=_PYTHON_AOCR,
python=runtime.NO_PYTHON,
binding=NATIVE_AOCR,
native=call_hook,
)

421
litellm/ocr/main.py
View file

@ -1,421 +0,0 @@
"""
Main OCR function for LiteLLM.
"""
import asyncio
import base64
import mimetypes
import os
import re
from collections.abc import Coroutine, Mapping
from dataclasses import dataclass
from io import IOBase
from types import MappingProxyType
from typing import Final, Protocol, cast # noqa: TID251 # adapters preserve the legacy untyped contracts
import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.constants import request_timeout
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.ocr.transformation import (
OCR_REQUEST_FORMAT_PARAM,
BaseOCRConfig,
OCRResponse,
parse_ocr_request_format,
)
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import CustomPricingLiteLLMParams
from litellm.utils import ProviderConfigManager, client
base_llm_http_handler: Final = BaseLLMHTTPHandler()
class FileReader(Protocol):
def read(self) -> bytes | str: ...
@dataclass(frozen=True, slots=True)
class _PreparedOCRRequest:
model: str
document: Mapping[str, object]
api_key: str | None
api_base: str | None
custom_llm_provider: str
extra_headers: dict[str, object] | None
provider_config: BaseOCRConfig
optional_params: dict[str, object]
litellm_params: dict[str, object]
effective_timeout: float | httpx.Timeout
litellm_logging_obj: LiteLLMLoggingObj
def _prepare_ocr_request(
model: str,
document: Mapping[str, object],
api_key: str | None,
api_base: str | None,
timeout: float | httpx.Timeout | None,
custom_llm_provider: str | None,
extra_headers: dict[str, object] | None,
kwargs: dict[str, object],
) -> _PreparedOCRRequest:
litellm_logging_obj: Final = cast( # cast-ok: @client supplies the logging object; preserve legacy failure behavior
LiteLLMLoggingObj, kwargs.pop("litellm_logging_obj")
)
litellm_call_id: Final = cast( # cast-ok: @client supplies the call id without coercion
str | None, kwargs.get("litellm_call_id", None)
)
if not isinstance(document, dict):
raise litellm.BadRequestError(
message="document must be a dict with 'type' and URL/file field",
model=model,
llm_provider=_error_provider(model, custom_llm_provider) or "",
)
normalized_document: Final = (
convert_file_document_to_url_document(document) if document.get("type") == "file" else document
)
doc_type: Final = normalized_document.get("type")
if doc_type not in ("document_url", "image_url"):
raise litellm.BadRequestError(
message=f"Invalid document type: {doc_type}. Must be 'document_url', 'image_url', or 'file'",
model=model,
llm_provider=_error_provider(model, custom_llm_provider) or "",
)
if not normalized_document.get(doc_type):
raise litellm.BadRequestError(
message="Document URL is required",
model=model,
llm_provider=_error_provider(model, custom_llm_provider) or "",
)
(
model,
custom_llm_provider,
dynamic_api_key,
dynamic_api_base,
) = litellm.get_llm_provider(
model=model,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
api_key=api_key,
)
ocr_provider_config: Final = ProviderConfigManager.get_provider_ocr_config(
model=model,
provider=litellm.LlmProviders(custom_llm_provider),
)
if ocr_provider_config is None:
raise ValueError(f"OCR is not supported for provider: {custom_llm_provider}")
resolved_api_key, resolved_api_base = ocr_provider_config.resolve_connection_params(
api_key=api_key,
api_base=api_base,
dynamic_api_key=dynamic_api_key,
dynamic_api_base=dynamic_api_base,
)
verbose_logger.debug("OCR call - model: %s, provider: %s", model, custom_llm_provider)
litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs)
supported_params: Final = ocr_provider_config.get_supported_ocr_params(model=model)
requested_format: Final = kwargs.get(OCR_REQUEST_FORMAT_PARAM)
if requested_format is not None:
try:
parse_ocr_request_format(requested_format)
except ValueError as e:
raise litellm.exceptions.UnsupportedParamsError(
message=f"{e}", model=model, llm_provider=custom_llm_provider
) from e
non_default_params: Final = {param: kwargs.pop(param) for param in supported_params if param in kwargs}
try:
mapped_params: Final = ocr_provider_config.map_ocr_params(
non_default_params=non_default_params,
optional_params={},
model=model,
)
except ValueError as error:
raise litellm.BadRequestError(message=str(error), model=model, llm_provider=custom_llm_provider) from error
optional_params: Final = (
mapped_params if requested_format is None else {**mapped_params, OCR_REQUEST_FORMAT_PARAM: requested_format}
)
verbose_logger.debug("OCR optional_params after mapping: %s", optional_params)
effective_timeout: Final = timeout or request_timeout
litellm_logging_obj.update_from_kwargs(
kwargs=kwargs,
model=model,
optional_params=optional_params,
litellm_params={
"litellm_call_id": litellm_call_id,
"api_base": resolved_api_base,
**litellm_params.model_dump(include=frozenset(CustomPricingLiteLLMParams.model_fields), exclude_none=True),
},
custom_llm_provider=custom_llm_provider,
)
return _PreparedOCRRequest(
model=model,
document=normalized_document,
api_key=resolved_api_key,
api_base=resolved_api_base,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
provider_config=ocr_provider_config,
optional_params=cast(dict[str, object], optional_params),
litellm_params=dict(litellm_params),
effective_timeout=effective_timeout,
litellm_logging_obj=litellm_logging_obj,
)
def _error_provider(model: str, custom_llm_provider: str | None) -> str | None:
if custom_llm_provider is not None:
return custom_llm_provider
prefix: Final = model.partition("/")[0]
if prefix in ("mistral", "azure_ai", "vertex_ai"):
return prefix
return "mistral" if model.startswith("mistral-ocr") else None
@client
async def aocr(
model: str,
document: Mapping[str, object],
api_key: str | None = None,
api_base: str | None = None,
timeout: float | httpx.Timeout | None = None,
custom_llm_provider: str | None = None,
extra_headers: dict[str, object] | None = None,
**kwargs: object, # kwargs-ok: public OCR accepts provider-specific options
) -> OCRResponse:
completion_kwargs: Final[dict[str, object]] = {
"model": model,
"document": document,
"api_key": api_key,
"api_base": api_base,
"timeout": timeout,
"custom_llm_provider": custom_llm_provider,
"extra_headers": extra_headers,
"kwargs": kwargs,
}
try:
prepared: Final = _prepare_ocr_request(
model=model,
document=document,
api_key=api_key,
api_base=api_base,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
kwargs=kwargs,
)
model = prepared.model
custom_llm_provider = prepared.custom_llm_provider
completion_kwargs.update(model=model, custom_llm_provider=custom_llm_provider)
response = base_llm_http_handler.ocr(
model=prepared.model,
document=cast( # cast-ok: preserve legacy document fields for provider validation
dict[str, str], prepared.document
),
optional_params=prepared.optional_params,
timeout=prepared.effective_timeout,
logging_obj=prepared.litellm_logging_obj,
api_key=prepared.api_key,
api_base=prepared.api_base,
custom_llm_provider=prepared.custom_llm_provider,
aocr=True,
headers=prepared.extra_headers,
provider_config=prepared.provider_config,
litellm_params=prepared.litellm_params,
)
if asyncio.iscoroutine(response):
response = await response
if response is None:
raise ValueError(f"Got an unexpected None response from the OCR API: {response}")
return response
except Exception as e:
error_provider: Final = _error_provider(model, custom_llm_provider)
error_model: Final = model.removeprefix(f"{error_provider}/") if error_provider else model
raise litellm.exception_type(
model=error_model,
custom_llm_provider=error_provider,
original_exception=e,
completion_kwargs=completion_kwargs,
extra_kwargs=kwargs,
)
_MIME_PATTERN: Final = re.compile(r"^[\w.+-]+/[\w.+-]+$")
_MIME_TYPE_MAP: Final = MappingProxyType(
{
".pdf": "application/pdf",
".png": "image/png",
".jpg": "image/jpeg",
".jpeg": "image/jpeg",
".gif": "image/gif",
".webp": "image/webp",
".tiff": "image/tiff",
".tif": "image/tiff",
".bmp": "image/bmp",
}
)
def get_mime_type(file_path: str) -> str:
ext: Final = os.path.splitext(file_path)[1].lower()
mime: Final = _MIME_TYPE_MAP.get(ext)
if mime:
return mime
guessed, _ = mimetypes.guess_type(file_path)
return guessed or "application/octet-stream"
def _read_file(file_input: object) -> tuple[bytes, str, str | None]:
if isinstance(file_input, str):
raise ValueError(
"OCR file input does not accept bare str values. Pass bytes, "
"a pathlib.Path, or a file-like object. To OCR a local file "
"from a path, call open(path, 'rb') yourself."
)
if isinstance(file_input, os.PathLike):
file_path: Final = str(cast(object, file_input)) # cast-ok: preserve staging's str(PathLike) conversion
if not os.path.isfile(file_path):
raise FileNotFoundError(f"File not found: {file_path}")
mime_type: Final = get_mime_type(file_path)
with open(file_path, "rb") as stream:
return stream.read(), mime_type, os.path.basename(file_path)
if isinstance(file_input, bytes):
return file_input, "application/octet-stream", None
if isinstance(file_input, IOBase) or hasattr(file_input, "read"):
file_name: Final = cast( # cast-ok: retain legacy validation and errors for file-like metadata
str | None, getattr(file_input, "name", None)
)
inferred_mime: Final = get_mime_type(file_name) if file_name else "application/octet-stream"
reader: Final = cast(FileReader, file_input) # cast-ok: legacy accepts duck-typed file readers
content: Final = reader.read()
return content.encode("utf-8") if isinstance(content, str) else content, inferred_mime, file_name
raise ValueError(
f"Unsupported file input type: {type(file_input)}. Expected pathlib.Path, bytes, or a file-like object."
)
def convert_file_document_to_url_document(document: Mapping[str, object]) -> dict[str, str]:
file_input: Final = document.get("file")
if file_input is None:
raise ValueError(
"document with type='file' must include a 'file' field containing "
"a pathlib.Path, file-like object, or bytes"
)
file_bytes, inferred_mime, file_name = _read_file(file_input)
if not file_bytes:
raise ValueError("File is empty or could not be read")
mime_type: Final = cast( # cast-ok: keep staging's MIME validation errors
str, document.get("mime_type", inferred_mime)
)
if not _MIME_PATTERN.match(mime_type):
raise ValueError(f"Invalid MIME type: {mime_type}")
base64_data: Final = base64.b64encode(file_bytes).decode("utf-8")
data_uri: Final = f"data:{mime_type};base64,{base64_data}"
if mime_type.startswith("image/"):
verbose_logger.debug(
"OCR file input: Converted file to image_url data URI (mime=%s, size=%s bytes, name=%s)",
mime_type,
len(file_bytes),
file_name,
)
return {"type": "image_url", "image_url": data_uri}
verbose_logger.debug(
"OCR file input: Converted file to document_url data URI (mime=%s, size=%s bytes, name=%s)",
mime_type,
len(file_bytes),
file_name,
)
return {"type": "document_url", "document_url": data_uri}
@client
def ocr(
model: str,
document: Mapping[str, object],
api_key: str | None = None,
api_base: str | None = None,
timeout: float | httpx.Timeout | None = None,
custom_llm_provider: str | None = None,
extra_headers: dict[str, object] | None = None,
**kwargs: object, # kwargs-ok: public OCR accepts provider-specific options
) -> OCRResponse | Coroutine[object, object, OCRResponse]:
completion_kwargs: Final[dict[str, object]] = {
"model": model,
"document": document,
"api_key": api_key,
"api_base": api_base,
"timeout": timeout,
"custom_llm_provider": custom_llm_provider,
"extra_headers": extra_headers,
"kwargs": kwargs,
}
try:
_is_async: Final = kwargs.pop("aocr", False) is True
completion_kwargs["aocr"] = _is_async
prepared: Final = _prepare_ocr_request(
model=model,
document=document,
api_key=api_key,
api_base=api_base,
kwargs=kwargs,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
timeout=timeout,
)
model = prepared.model
custom_llm_provider = prepared.custom_llm_provider
completion_kwargs.update(model=model, custom_llm_provider=custom_llm_provider)
response: Final = base_llm_http_handler.ocr(
model=prepared.model,
document=cast( # cast-ok: preserve legacy document fields for provider validation
dict[str, str], prepared.document
),
optional_params=prepared.optional_params,
timeout=prepared.effective_timeout,
logging_obj=prepared.litellm_logging_obj,
api_key=prepared.api_key,
api_base=prepared.api_base,
custom_llm_provider=prepared.custom_llm_provider,
aocr=_is_async,
headers=prepared.extra_headers,
provider_config=prepared.provider_config,
litellm_params=prepared.litellm_params,
)
return response
except Exception as e:
error_provider: Final = _error_provider(model, custom_llm_provider)
error_model: Final = model.removeprefix(f"{error_provider}/") if error_provider else model
raise litellm.exception_type(
model=error_model,
custom_llm_provider=error_provider,
original_exception=e,
completion_kwargs=completion_kwargs,
extra_kwargs=kwargs,
)

24
litellm/proxy/ocr_endpoints/endpoints.py
View file

@ -1,5 +1,6 @@
#### OCR Endpoints #####
import io
import json
from collections.abc import Mapping
from typing import Final, cast
@ -15,7 +16,6 @@ from litellm.llms.base_llm.ocr.transformation import (
OCRResponse,
parse_ocr_request_format,
)
from litellm.ocr.main import convert_file_document_to_url_document, get_mime_type
from litellm.proxy._types import *
from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
@ -24,20 +24,24 @@ router: Final = APIRouter()
_MAX_FILE_BYTES: Final = 50 * 1024 * 1024
class _NamedUpload(io.BytesIO):
name: str | None
def __init__(self, content: bytes, name: str | None) -> None:
super().__init__(content)
self.name = name
def _build_document_from_upload(
file_content: bytes,
filename: str | None,
content_type: str | None,
) -> dict[str, str]:
) -> dict[str, object]:
supplied_mime: Final = content_type.split(";")[0].strip() if content_type else None
mime_type: Final = (
get_mime_type(filename)
if filename and (not supplied_mime or supplied_mime == "application/octet-stream")
else supplied_mime
)
return convert_file_document_to_url_document(
{"type": "file", "file": file_content, "mime_type": mime_type or "application/octet-stream"}
)
upload: Final = _NamedUpload(file_content, filename)
if supplied_mime and supplied_mime != "application/octet-stream":
return {"type": "file", "file": upload, "mime_type": supplied_mime}
return {"type": "file", "file": upload}
def _with_request_format(data: Mapping[str, object], request: Request) -> Mapping[str, object]:

4
litellm/rust_bridge/_native.pyi
View file

@ -46,6 +46,8 @@ def aocr(
args: tuple[object, ...],
kwargs: dict[str, object],
) -> Coroutine[object, object, OCRResponse]: ...
def ocr_health_check_document(model: str, custom_llm_provider: str | None) -> dict[str, object]: ...
def ocr_passthrough_response(model: str, endpoint: str, body: bytes) -> dict[str, object] | None: ...
def embedding(
request: LiteLLMEmbeddingRequest,
args: tuple[object, ...],
@ -417,6 +419,8 @@ __all__ = [
"gil_stats",
"messages",
"ocr",
"ocr_health_check_document",
"ocr_passthrough_response",
"process_state_started",
"reserve_process_for_forking",
"responses",

3
litellm/rust_bridge/catalog.py
View file

@ -108,8 +108,7 @@ RULES: Final[Rules] = (
LoggerRule(Rollout.RUST_OPT_IN),
RouteRule(Route.CHAT_COMPLETIONS, Rollout.PYTHON_ONLY),
RouteRule(Route.EMBEDDINGS, Rollout.PYTHON_ONLY),
RouteRule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset({"aws_textract"})),
RouteRule(Route.OCR, Rollout.RUST_OPT_OUT),
RouteRule(Route.OCR, Rollout.RUST_REQUIRED),
RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY),
RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY),
RouteRule(Route.TOKEN_COUNTER, Rollout.PYTHON_ONLY),

28
litellm/rust_bridge/ocr/entrypoints.py
View file

@ -6,7 +6,7 @@ from typing import Final, Protocol, cast # noqa: TID251 # validates dynamicall
import httpx
from litellm.llms.base_llm.ocr.transformation import OCRResponse
from litellm.llms.base_llm.ocr.transformation import DocumentType, OCRResponse
from litellm.rust_bridge.bindings import NativeBinding
@ -53,5 +53,31 @@ def _aocr_binding(value: object) -> NativeAocr | None:
return cast("NativeAocr", value) # cast-ok: callable validated at the native binding boundary
class NativeOcrHealthCheckDocument(Protocol):
def __call__(self, model: str, custom_llm_provider: str | None) -> DocumentType: ...
class NativeOcrPassthroughResponse(Protocol):
def __call__(self, model: str, endpoint: str, body: bytes) -> Mapping[str, object] | None: ...
def _health_check_document_binding(value: object) -> NativeOcrHealthCheckDocument | None:
if not callable(value):
return None
return cast("NativeOcrHealthCheckDocument", value) # cast-ok: callable validated at the native binding boundary
def _passthrough_response_binding(value: object) -> NativeOcrPassthroughResponse | None:
if not callable(value):
return None
return cast("NativeOcrPassthroughResponse", value) # cast-ok: callable validated at the native binding boundary
NATIVE_OCR: Final = NativeBinding("ocr", validate=_ocr_binding)
NATIVE_AOCR: Final = NativeBinding("aocr", validate=_aocr_binding)
NATIVE_OCR_HEALTH_CHECK_DOCUMENT: Final = NativeBinding(
"ocr_health_check_document", validate=_health_check_document_binding
)
NATIVE_OCR_PASSTHROUGH_RESPONSE: Final = NativeBinding(
"ocr_passthrough_response", validate=_passthrough_response_binding
)

47
litellm/utils.py
View file

@ -392,7 +392,6 @@ if TYPE_CHECKING:
from litellm.llms.base_llm.image_variations.transformation import (
BaseImageVariationConfig,
)
from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
from litellm.llms.base_llm.realtime.http_transformation import (
BaseRealtimeHTTPConfig,
@ -430,7 +429,6 @@ if TYPE_CHECKING:
)
from litellm.llms.cohere.common_utils import CohereModelInfo
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.mistral.ocr.transformation import MistralOCRConfig
from litellm.proxy._types import AllowedModelRegion
from litellm.router_utils.get_retry_from_policy import (
get_num_retries_from_retry_policy,
@ -9758,51 +9756,6 @@ class ProviderConfigManager:
return get_openrouter_image_edit_config(model)
return None
@staticmethod
def get_provider_ocr_config(
model: str,
provider: LlmProviders,
) -> BaseOCRConfig | None:
"""
Get OCR configuration for a given provider.
"""
from litellm.llms.vertex_ai.ocr.transformation import VertexAIOCRConfig
# Special handling for Azure AI - distinguish between Mistral OCR and Document Intelligence
if provider == litellm.LlmProviders.AZURE_AI:
from litellm.llms.azure_ai.ocr.common_utils import get_azure_ai_ocr_config
return get_azure_ai_ocr_config(model=model)
if provider == litellm.LlmProviders.VERTEX_AI:
from litellm.llms.vertex_ai.ocr.common_utils import get_vertex_ai_ocr_config
return get_vertex_ai_ocr_config(model=model)
if provider == litellm.LlmProviders.COHERE:
from litellm.llms.cohere.ocr.transformation import CohereParseConfig
return CohereParseConfig()
if provider == litellm.LlmProviders.REDUCTO:
from litellm.llms.reducto.ocr.transformation import (
ReductoParseLegacyConfig,
ReductoParseV3Config,
)
if model == "parse-legacy":
return ReductoParseLegacyConfig()
return ReductoParseV3Config()
MistralOCRConfig: Final = litellm_utils.MistralOCRConfig
PROVIDER_TO_CONFIG_MAP: Final = {
litellm.LlmProviders.MISTRAL: MistralOCRConfig,
}
config_class: Final = PROVIDER_TO_CONFIG_MAP.get(provider, None)
if config_class is None:
return None
return config_class()
@staticmethod
def get_provider_search_config(
provider: SearchProviders,

2
ruff-strict.toml
View file

@ -65,7 +65,5 @@ max-args = 5
"litellm.responses.main.aresponses".msg = "Import litellm.responses.dispatch.aresponses so the call routes through dispatch."
"litellm.llms.anthropic.experimental_pass_through.messages.handler.anthropic_messages".msg = "Import litellm.messages.anthropic_messages so the call routes through dispatch."
"litellm.llms.anthropic.experimental_pass_through.messages.handler.anthropic_messages_handler".msg = "Import litellm.messages.anthropic_messages_handler so the call routes through dispatch."
"litellm.ocr.main.ocr".msg = "Import litellm.ocr.dispatch.ocr so the call routes through dispatch."
"litellm.ocr.main.aocr".msg = "Import litellm.ocr.dispatch.aocr so the call routes through dispatch."
"litellm.main.completion".msg = "Import litellm.completion so the call routes through dispatch."
"litellm.main.acompletion".msg = "Import litellm.acompletion so the call routes through dispatch."

14
tests/ocr_tests/conftest.py
View file

@ -5,8 +5,6 @@
# Vertex AI OCR) are replayed for 24h. See tests/llm_translation/Readme.md
# for the design overview.
from typing import Final
import pytest
@ -24,13 +22,6 @@ from tests._vcr_conftest_common import ( # noqa: E402,F401
vcr_config_dict,
)
_VCR_INCOMPATIBLE_NODEID_SUFFIXES: Final[tuple[str, ...]] = (
"test_rust_bridge.py::test_native_public_ocr_matches_python[mistral/mistral-ocr-latest-False]",
"test_rust_bridge.py::test_native_public_ocr_matches_python[mistral/mistral-ocr-latest-True]",
"test_rust_bridge.py::test_native_public_ocr_matches_python[azure_ai/doc-intelligence/prebuilt-read-False]",
"test_rust_bridge.py::test_native_public_ocr_matches_python[azure_ai/doc-intelligence/prebuilt-read-True]",
)
_verbose_state = VerboseReporterState()
@ -67,10 +58,7 @@ def pytest_runtest_logreport(report):
def pytest_collection_modifyitems(config, items):
apply_vcr_auto_marker_to_items(
items,
skip_nodeid_suffixes=_VCR_INCOMPATIBLE_NODEID_SUFFIXES,
)
apply_vcr_auto_marker_to_items(items)
def pytest_terminal_summary(terminalreporter, exitstatus, config):

128
tests/ocr_tests/test_ocr_azure_document_intelligence.py
View file

@ -1,128 +0,0 @@
"""Azure Document Intelligence request transformation: Mistral-shaped `pages` to Azure's query string."""
import pytest
from litellm.constants import AZURE_DOCUMENT_INTELLIGENCE_API_VERSION
from litellm.llms.azure_ai.ocr.document_intelligence.transformation import (
AzureDocumentIntelligenceOCRConfig,
)
class TestAzureDocumentIntelligencePagesParam:
"""
Unit tests for the Mistral-compatible `pages` parameter translation to
Azure Document Intelligence's `pages` query string.
These tests exercise the transformation layer directly and do not
require Azure credentials or a network call.
"""
@pytest.fixture
def cfg(self) -> AzureDocumentIntelligenceOCRConfig:
return AzureDocumentIntelligenceOCRConfig()
def test_get_supported_ocr_params_includes_pages_and_features(self, cfg):
assert cfg.get_supported_ocr_params("prebuilt-layout") == ["pages", "features", "req_format"]
def test_map_ocr_params_mistral_zero_based_int_list(self, cfg):
mapped = cfg.map_ocr_params({"pages": [0, 1, 2]}, {}, "prebuilt-layout")
assert mapped == {"pages": "1,2,3"}
def test_map_ocr_params_dedupes_and_sorts(self, cfg):
mapped = cfg.map_ocr_params({"pages": [2, 0, 0, 1]}, {}, "prebuilt-layout")
assert mapped == {"pages": "1,2,3"}
def test_map_ocr_params_empty_list_omits_pages(self, cfg):
mapped = cfg.map_ocr_params({"pages": []}, {}, "prebuilt-layout")
assert mapped == {}
def test_map_ocr_params_azure_native_string_range(self, cfg):
mapped = cfg.map_ocr_params({"pages": "3-9"}, {}, "prebuilt-layout")
assert mapped == {"pages": "3-9"}
def test_map_ocr_params_azure_native_string_with_spaces_stripped(self, cfg):
mapped = cfg.map_ocr_params({"pages": "1-3, 5"}, {}, "prebuilt-layout")
assert mapped == {"pages": "1-3,5"}
def test_map_ocr_params_list_of_string_tokens(self, cfg):
mapped = cfg.map_ocr_params({"pages": ["1", "3-5"]}, {}, "prebuilt-layout")
assert mapped == {"pages": "1,3-5"}
def test_map_ocr_params_invalid_string_raises(self, cfg):
with pytest.raises(ValueError, match="Invalid `pages` string"):
cfg.map_ocr_params({"pages": "a,b"}, {}, "prebuilt-layout")
def test_map_ocr_params_negative_index_raises(self, cfg):
with pytest.raises(ValueError, match="must be >= 0"):
cfg.map_ocr_params({"pages": [-1]}, {}, "prebuilt-layout")
def test_map_ocr_params_bool_list_raises(self, cfg):
with pytest.raises(ValueError, match="must be integers, not booleans"):
cfg.map_ocr_params({"pages": [True, False]}, {}, "prebuilt-layout")
def test_map_ocr_params_unsupported_type_raises(self, cfg):
with pytest.raises(ValueError, match="based, Mistral-style\\) or a string like"):
cfg.map_ocr_params({"pages": 5}, {}, "prebuilt-layout")
def test_get_complete_url_appends_pages_query(self, cfg):
url = cfg.get_complete_url(
api_base="https://example.cognitiveservices.azure.com/",
model="azure_ai/doc-intelligence/prebuilt-layout",
optional_params={"pages": "1-3,5"},
)
assert f"api-version={AZURE_DOCUMENT_INTELLIGENCE_API_VERSION}" in url, url
assert "pages=1-3,5" in url, url
assert "/documentintelligence/documentModels/prebuilt-layout:analyze" in url
def test_get_complete_url_no_pages_when_optional_params_empty(self, cfg):
url = cfg.get_complete_url(
api_base="https://example.cognitiveservices.azure.com",
model="prebuilt-layout",
optional_params={},
)
assert "pages=" not in url
def test_transform_ocr_request_does_not_put_pages_in_body(self, cfg):
req = cfg.transform_ocr_request(
model="prebuilt-layout",
document={
"type": "document_url",
"document_url": "https://example.com/x.pdf",
},
optional_params={"pages": "1,2,3"},
headers={},
)
assert req.data is not None
assert "pages" not in req.data
assert req.data.get("urlSource") == "https://example.com/x.pdf"
def test_end_to_end_mistral_shape_to_azure_query(self, cfg):
"""
Caller sends Mistral-style `pages: [2,3,4,5,6,7,8]` (0-based,
meaning human pages 3-9). LiteLLM should turn that into Azure's
`&pages=3,4,5,6,7,8,9` on the analyze URL, and the body should
still only contain urlSource.
"""
non_default_params = {"pages": [2, 3, 4, 5, 6, 7, 8]}
optional_params = cfg.map_ocr_params(
non_default_params=non_default_params,
optional_params={},
model="prebuilt-layout",
)
url = cfg.get_complete_url(
api_base="https://example.cognitiveservices.azure.com",
model="prebuilt-layout",
optional_params=optional_params,
)
req = cfg.transform_ocr_request(
model="prebuilt-layout",
document={
"type": "document_url",
"document_url": "https://example.com/x.pdf",
},
optional_params=optional_params,
headers={},
)
assert "pages=3,4,5,6,7,8,9" in url
assert req.data == {"urlSource": "https://example.com/x.pdf"}

48
tests/ocr_tests/test_ocr_vertex_ai.py
View file

@ -1,48 +0,0 @@
"""Vertex AI OCR config routing and DeepSeek request shaping (no network)."""
from typing import Final
import pytest
def test_vertex_ai_ocr_routing():
"""
Test that Vertex AI OCR routing correctly selects the right config based on model name.
"""
from litellm.llms.vertex_ai.ocr.common_utils import get_vertex_ai_ocr_config
from litellm.llms.vertex_ai.ocr.deepseek_transformation import (
VertexAIDeepSeekOCRConfig,
)
from litellm.llms.vertex_ai.ocr.transformation import VertexAIOCRConfig
# Test DeepSeek OCR routing
deepseek_config = get_vertex_ai_ocr_config("vertex_ai/deepseek-ocr-maas")
assert isinstance(deepseek_config, VertexAIDeepSeekOCRConfig), (
"DeepSeek model should route to VertexAIDeepSeekOCRConfig"
)
# Test Mistral OCR routing (should use default VertexAIOCRConfig)
mistral_config = get_vertex_ai_ocr_config("vertex_ai/mistral-ocr-2505")
assert isinstance(mistral_config, VertexAIOCRConfig), "Mistral model should route to VertexAIOCRConfig"
# Test other DeepSeek variants
deepseek_variant = get_vertex_ai_ocr_config("vertex_ai/deepseek-ocr-maas")
assert isinstance(deepseek_variant, VertexAIDeepSeekOCRConfig), (
"DeepSeek variant should route to VertexAIDeepSeekOCRConfig"
)
@pytest.mark.parametrize("model", ("deepseek-ocr-maas", "deepseek-ai/deepseek-ocr-maas"))
def test_deepseek_request_uses_single_provider_namespace(model: str) -> None:
from litellm.llms.vertex_ai.ocr.deepseek_transformation import (
VertexAIDeepSeekOCRConfig,
)
request: Final = VertexAIDeepSeekOCRConfig().transform_ocr_request(
model=model,
document={"type": "image_url", "image_url": "data:image/png;base64,AA=="},
optional_params={},
headers={},
)
assert request.data["model"] == "deepseek-ai/deepseek-ocr-maas"

23
tests/rust-python-harness/shared/unit_runners/contracts.py
View file

@ -85,25 +85,4 @@ class UnitTestContract(_ContractModel):
rust: RustUnitSpec
OCR_CONTRACT: Final = UnitTestContract(
unit_parity=UnitParitySpec(
python_selectors=(
"tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py",
"tests/test_litellm/llms/mistral/ocr",
"tests/test_litellm/llms/ocr",
"tests/test_litellm/ocr",
),
exclusions=(
UnitParityExclusionSpec(
nodeid="tests/test_litellm/ocr/test_rust_bridge.py::test_rust_toggles_flag",
reason="This test asserts the process-level backend flag selected by the parity runner.",
),
),
),
rust=RustUnitSpec(
cargo_manifest="litellm-rust/Cargo.toml",
cargo_filter="ocr",
),
)
UNIT_TEST_CONTRACTS: Final[Mapping[SdkFunction, UnitTestContract]] = MappingProxyType({"ocr": OCR_CONTRACT})
UNIT_TEST_CONTRACTS: Final[Mapping[SdkFunction, UnitTestContract]] = MappingProxyType({})

11
tests/rust-python-harness/strategies/e2e_parity/__init__.py
View file

@ -1,7 +1,7 @@
from pathlib import Path
from typing import Final
from ...shared.reporting.models import SURFACES, Coverage
from ...shared.reporting.models import SURFACES
from ...shared.reporting.strategy import (
CaseDefinition,
ModuleCaseSpec,
@ -14,14 +14,7 @@ from .runner import run_e2e_cases
CASES: Final[tuple[CaseDefinition, ...]] = (
CaseDefinition(
"ocr",
ModuleCaseSpec(
coverage=Coverage.PARTIAL,
module="tests.rust-python-harness.strategies.e2e_parity.sdk.ocr.test_sdk_parity",
note=(
"Recorded sync/async SDK parity with focused success/error callback profiles; "
"Reducto lacks a Rust contract, and known provider parity gaps remain."
),
),
NotImplementedCaseSpec(reason="OCR has no Python implementation left to compare against."),
surface="sdk",
),
CaseDefinition(

43
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/conftest.py
View file

@ -1,43 +0,0 @@
from __future__ import annotations
from typing import Final
import pytest
from .....shared.parity.fixtures.pytest_support import parametrize_recorded_fixtures
from .....shared.parity.fixtures.store import fixture_id
from .fixtures.config import DEFAULT_FIXTURE_DIRECTORY, FIXTURE_DIR_ENV
from .fixtures.models import OcrParityCase
def ocr_fixture_id(fixture: OcrParityCase) -> str:
case_input: Final = fixture.litellm_input
provider: Final = case_input.custom_llm_provider
prefix: Final = f"{provider}/{case_input.model}" if provider else case_input.model
return fixture_id(case_input, prefix)
def ocr_fixture_marks(fixture: OcrParityCase) -> tuple[pytest.MarkDecorator, ...]:
if fixture.litellm_input.contract not in {"reducto_v3", "reducto_legacy"}:
return ()
return (
pytest.mark.xfail(
reason="Reducto does not have a Rust OCR contract",
strict=False,
),
)
def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
parametrize_recorded_fixtures(
metafunc,
fixture_name="ocr_fixture",
case_type=OcrParityCase,
env_var=FIXTURE_DIR_ENV,
default_directory=DEFAULT_FIXTURE_DIRECTORY,
regeneration_command=(
f"uv run python -m tests.rust-python-harness.strategies.e2e_parity.sdk.ocr.fixtures.record --fixture-dir {DEFAULT_FIXTURE_DIRECTORY}"
),
id_builder=ocr_fixture_id,
marks_builder=ocr_fixture_marks,
)

74
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/README.md
View file

@ -1,74 +0,0 @@
# OCR parity fixtures
The recording command runs four stages:
1. Generate deterministic SDK inputs for every configured OCR target
2. Build target-scoped, deduplicated recording jobs
3. Record upstream responses through one globally bounded worker pool
4. Persist each fixture and report whether it was recorded, cached, or failed
Run it with:
```shell
uv run python -m tests.rust-python-harness.strategies.e2e_parity.sdk.ocr.fixtures.record --examples 1000
```
`--concurrency` defaults to 2 and caps active recording jobs across all targets. Increase it explicitly when provider
quotas permit. Concurrency limits do not guarantee a request-per-minute quota; HTTP 408, 429, and 5xx responses fail
recording without being saved. Rerunning retries missing fixtures and reuses successful recordings
New recordings are VCR YAML cassettes. The corpus retains the original 31 migrated cassettes and adds live recordings
for all four providers. Original response bytes, statuses, headers, and recording timestamps are preserved
For Vertex, authenticate and select a project once:
```shell
gcloud auth login
gcloud config set project YOUR_PROJECT_ID
```
The recording command reads the project from `VERTEXAI_PROJECT`, `VERTEX_PROJECT`, or the active gcloud configuration,
then gets an OAuth access token with `gcloud auth print-access-token`. Tokens stay in memory and are removed from
recorded headers. `VERTEX_AI_ACCESS_TOKEN` or the legacy `VERTEX_AI_API_KEY` can override token lookup. The
`VERTEXT_API_KEY` express-mode key is not used as a Bearer token. Mistral defaults to `us-central1`; DeepSeek defaults
to the global host and `global` location. `VERTEX_DEEPSEEK_LOCATION` and `VERTEX_DEEPSEEK_API_BASE` override the latter
Azure accepts `AZURE_KEY` and `AZURE_ENDPOINT` as fallbacks for both Azure OCR contracts. Provider-specific variables
take precedence. Set `AZURE_DEPLOYMENT_NAME=mistral-ocr-4-0` to record that deployment instead of the default
`mistral-document-ai-2512`. Mistral and Reducto use `MISTRAL_API_KEY` and `REDUCTO_API_KEY`
To migrate an existing JSON fixture directory locally:
```shell
uv run python -m tests.rust-python-harness.strategies.e2e_parity.sdk.ocr.fixtures.migrate --fixture-dir tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data
```
The migration replays each old response through the Python SDK to reconstruct missing requests, writes and validates
the YAML cassette, then removes its JSON predecessor. It calls only local recording/replay servers and needs no provider
credentials. Reconstructed requests are labeled `python_replay`; they are not historical wire captures. Filenames use
the current normalized SDK input hash, including the fixture contract
OCR strategies generate public `litellm.ocr()` and `litellm.aocr()` inputs. Every case contains the normalized model,
document, optional provider override, and LiteLLM keyword arguments. The fixture-only `contract` literal selects the
input schema and is removed before calling the SDK. Strategies never build provider wire payloads
Each contract's strategy contains baselines and cases for its supported top-level OCR parameters. The
contracts are Mistral, Azure-hosted Mistral, Vertex-hosted Mistral, Azure Document Intelligence, Vertex DeepSeek,
Reducto v3, and Reducto legacy. Credentials and endpoints only control target discovery, so a machine records the
contracts it has configured and skips the rest
`--examples 1000` exhausts the current finite strategies: 32 Mistral, 15 Azure Mistral, 17 Azure Document Intelligence,
16 Vertex Mistral, 2 Vertex DeepSeek, 55 Reducto v3, and 3 Reducto legacy cases, including fixed rejected inputs.
This covers the defined strategy choices, not every possible value accepted by the schemas. The live run recorded
139 of these 140 cases. Vertex Mistral's standalone `document_annotation_format` case repeatedly returned HTTP 500
and remains pending. Its bounding-box annotation, annotation-prompt, and confidence cases record upstream 404/422
rejections; schema acceptance does not imply support by the hosted model
Reducto fixtures record upload and parse responses. Their parity cases remain non-strict expected failures until the
Rust OCR bridge supports Reducto. Azure and Vertex generation paths are unit-tested without credentials in CI, so the
committed corpus does not need live recordings for every target
Every recording target owns a small fixed provider-rejected corpus, independent of replay implementation support.
Those inputs are recorded separately from generated valid inputs. Local validation failures use no recorded response;
the parity suite checks those unsupported providers and models, malformed documents, invalid request formats, invalid
Azure Document Intelligence parameters, and invalid headers in sync and async SDK calls

1
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/__init__.py
View file

@ -1 +0,0 @@

216
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/azure.py
View file

@ -1,216 +0,0 @@
from __future__ import annotations
from collections.abc import Mapping
from typing import Final, Literal, cast
from hypothesis import strategies as st
from hypothesis.strategies import SearchStrategy
from pydantic import StrictInt, StrictStr, TypeAdapter, field_validator
from ......shared.parity.fixtures.recording import UpstreamEndpoint
from .base import OcrDocument, OcrSdkInputBase
from .common import (
OcrFixtureClient,
OcrRecordingTarget,
image_document,
invoke_with_api_key,
pdf_document,
)
from .mistral import (
MistralCompatibleOcrSdkInput,
mistral_input_values_strategy,
)
AzureMistralModel = Literal["azure_ai/mistral-document-ai-2512", "azure_ai/mistral-ocr-4-0"]
AzureMistralFixtureModel = AzureMistralModel | Literal["azure_ai/invalid-ocr-model-for-parity"]
AzureDocumentIntelligenceModel = Literal[
"azure_ai/doc-intelligence/prebuilt-read",
"azure_ai/doc-intelligence/prebuilt-layout",
"azure_ai/doc-intelligence/prebuilt-document",
]
AzureDocumentIntelligenceFixtureModel = (
AzureDocumentIntelligenceModel | Literal["azure_ai/doc-intelligence/invalid-ocr-model-for-parity"]
)
AZURE_MISTRAL_MODELS: Final[tuple[AzureMistralModel, ...]] = ("azure_ai/mistral-document-ai-2512",)
AZURE_DOCUMENT_INTELLIGENCE_MODELS: Final[tuple[AzureDocumentIntelligenceModel, ...]] = (
"azure_ai/doc-intelligence/prebuilt-read",
"azure_ai/doc-intelligence/prebuilt-layout",
"azure_ai/doc-intelligence/prebuilt-document",
)
# API v4 replaces prebuilt-document with prebuilt-layout plus keyValuePairs. Keep
# the broader fixture model above so existing recordings remain loadable.
AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS: Final[tuple[AzureDocumentIntelligenceModel, ...]] = (
"azure_ai/doc-intelligence/prebuilt-read",
"azure_ai/doc-intelligence/prebuilt-layout",
)
class AzureMistralOcrSdkInput(MistralCompatibleOcrSdkInput):
contract: Literal["azure_mistral"] = "azure_mistral"
model: AzureMistralFixtureModel
custom_llm_provider: Literal["azure_ai"] | None = None
@field_validator("model")
@classmethod
def validate_model_namespace(cls, model: str) -> str:
if not model.startswith("azure_ai/"):
raise ValueError("Azure Mistral models must use the azure_ai/ LiteLLM namespace")
return model
class AzureDocumentIntelligenceOcrSdkInput(OcrSdkInputBase):
contract: Literal["azure_document_intelligence"] = "azure_document_intelligence"
model: AzureDocumentIntelligenceFixtureModel
document: OcrDocument
custom_llm_provider: Literal["azure_ai"] | None = None
pages: str | list[StrictInt] | list[StrictStr] | None = None
features: str | list[str] | None = None
req_format: Literal["litellm"] = "litellm"
AZURE_MISTRAL_PROVIDER_REJECTED_INPUTS: Final[tuple[AzureMistralOcrSdkInput, ...]] = (
AzureMistralOcrSdkInput(
model="azure_ai/invalid-ocr-model-for-parity",
document=pdf_document(),
),
)
AZURE_DOCUMENT_INTELLIGENCE_PROVIDER_REJECTED_INPUTS: Final[tuple[AzureDocumentIntelligenceOcrSdkInput, ...]] = (
AzureDocumentIntelligenceOcrSdkInput(
model="azure_ai/doc-intelligence/invalid-ocr-model-for-parity",
document=pdf_document(),
),
)
def _azure_mistral_input(values: dict[str, object], model: AzureMistralModel) -> AzureMistralOcrSdkInput:
return AzureMistralOcrSdkInput.model_validate({**values, "model": model})
def azure_mistral_input_strategy(
inline_image_data_uri: str,
models: tuple[AzureMistralModel, ...] = AZURE_MISTRAL_MODELS,
) -> SearchStrategy[AzureMistralOcrSdkInput]:
# Foundry's active gateway schema rejects 2512-only controls and
# document_annotation_prompt, even though native Mistral accepts them.
return st.builds(
_azure_mistral_input,
values=mistral_input_values_strategy("2505", inline_image_data_uri, include_document_annotation_prompt=False),
model=st.sampled_from(models),
)
_AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL: Final[AzureDocumentIntelligenceModel] = (
"azure_ai/doc-intelligence/prebuilt-layout"
)
def _document_intelligence_input(
model: AzureDocumentIntelligenceModel,
document: OcrDocument,
optional_params: Mapping[str, object] | None = None,
) -> AzureDocumentIntelligenceOcrSdkInput:
return AzureDocumentIntelligenceOcrSdkInput.model_validate(
{"model": model, "document": document, **(optional_params or {})}
)
def azure_document_intelligence_input_strategy() -> SearchStrategy[AzureDocumentIntelligenceOcrSdkInput]:
document: Final = pdf_document()
pages: Final = st.one_of(
st.sampled_from(((0,), (2, 0, 0, 1))).map(list),
st.just(["1", "2-4"]),
st.just("1-4, 5"),
).map(lambda value: {"pages": value})
features: Final = st.one_of(
st.sampled_from(
(
("languages",),
("ocrHighResolution",),
("barcodes",),
("formulas",),
("styleFont",),
("keyValuePairs",),
)
).map(list),
st.just("languages, styleFont"),
).map(lambda value: {"features": value})
combined_query: Final = st.just({"pages": (0, 1), "features": ("languages", "styleFont")})
return st.one_of(
st.sampled_from(AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS).map(
lambda model: _document_intelligence_input(model, document)
),
st.just(
_document_intelligence_input(
_AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL,
image_document("invoice 123", 24),
)
),
pages.map(
lambda optional_params: _document_intelligence_input(
_AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL, document, optional_params
)
),
features.map(
lambda optional_params: _document_intelligence_input(
_AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL, document, optional_params
)
),
combined_query.map(
lambda optional_params: _document_intelligence_input(
_AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL, document, optional_params
)
),
st.just(
_document_intelligence_input(
_AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL,
document,
{"req_format": "litellm"},
)
),
)
def azure_mistral_recording_targets(
environ: Mapping[str, str], client: OcrFixtureClient, inline_image_data_uri: str
) -> tuple[OcrRecordingTarget, ...]:
api_key: Final = environ.get("AZURE_AI_API_KEY") or environ.get("AZURE_KEY")
base_url: Final = environ.get("AZURE_AI_API_BASE") or environ.get("AZURE_ENDPOINT")
if not api_key or not base_url:
return ()
deployment: Final = environ.get("AZURE_DEPLOYMENT_NAME")
models: Final = (
(TypeAdapter(AzureMistralModel).validate_python(f"azure_ai/{deployment.removeprefix('azure_ai/')}"),)
if deployment
else AZURE_MISTRAL_MODELS
)
return (
OcrRecordingTarget(
name="azure-mistral",
upstream=UpstreamEndpoint(base_url=base_url.rstrip("/")),
strategy=cast(
SearchStrategy[OcrSdkInputBase],
azure_mistral_input_strategy(inline_image_data_uri, models),
),
invocation=invoke_with_api_key(client, api_key),
required_inputs=AZURE_MISTRAL_PROVIDER_REJECTED_INPUTS,
),
)
def azure_document_intelligence_recording_targets(
environ: Mapping[str, str], client: OcrFixtureClient
) -> tuple[OcrRecordingTarget, ...]:
api_key: Final = environ.get("AZURE_DOCUMENT_INTELLIGENCE_API_KEY") or environ.get("AZURE_KEY")
base_url: Final = environ.get("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT") or environ.get("AZURE_ENDPOINT")
if not api_key or not base_url:
return ()
return (
OcrRecordingTarget(
name="azure-document-intelligence",
upstream=UpstreamEndpoint(base_url=base_url.rstrip("/")),
strategy=cast(SearchStrategy[OcrSdkInputBase], azure_document_intelligence_input_strategy()),
invocation=invoke_with_api_key(client, api_key),
required_inputs=AZURE_DOCUMENT_INTELLIGENCE_PROVIDER_REJECTED_INPUTS,
),
)

48
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/base.py
View file

@ -1,48 +0,0 @@
from __future__ import annotations
from typing import Annotated, Literal
from pydantic import Field
from ......shared.parity.fixture_models import (
FixtureModel,
JsonSchemaDefinition,
JsonSchemaResponseFormat,
SdkInputBase,
)
__all__ = (
"DocumentUrlDocument",
"ImageUrlDocument",
"ImageUrlValue",
"JsonSchemaDefinition",
"JsonSchemaResponseFormat",
"OcrDocument",
"OcrSdkInputBase",
)
class OcrSdkInputBase(SdkInputBase):
fixture_only_fields = ("contract",)
class ImageUrlValue(FixtureModel):
url: str
detail: Literal["low", "auto", "high"] | None = None
class ImageUrlDocument(FixtureModel):
type: Literal["image_url"]
image_url: str | ImageUrlValue
class DocumentUrlDocument(FixtureModel):
type: Literal["document_url"]
document_url: str
document_name: str | None = None
OcrDocument = Annotated[
ImageUrlDocument | DocumentUrlDocument,
Field(discriminator="type"),
]

101
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/common.py
View file

@ -1,101 +0,0 @@
from __future__ import annotations
from dataclasses import dataclass, field
from functools import cache
from typing import Final, Literal, Protocol
from hypothesis import strategies as st
from hypothesis.strategies import SearchStrategy
from ......shared.parity.fixtures.pipeline import RecordingTarget
from ......shared.parity.fixtures.media import dummy_image_url, structured_pdf_data_uri
from .base import (
DocumentUrlDocument,
ImageUrlDocument,
JsonSchemaDefinition,
JsonSchemaResponseFormat,
OcrSdkInputBase,
)
OcrRecordingTarget = RecordingTarget[OcrSdkInputBase]
class OcrFixtureClient(Protocol):
def execute(self, api_base: str, api_key: str, case_input: OcrSdkInputBase) -> None: ...
class OcrSdkCall(Protocol):
def __call__(self, **kwargs: object) -> object: ...
@dataclass(frozen=True, slots=True)
class ApiKeyOcrInvocation:
client: OcrFixtureClient
api_key: str = field(repr=False)
def execute(self, provider_url: str, case_input: OcrSdkInputBase) -> None:
self.client.execute(provider_url, self.api_key, case_input)
def image_document(text: str, font_size: int) -> ImageUrlDocument:
return ImageUrlDocument(type="image_url", image_url=dummy_image_url(text, font_size))
def image_data_document(data_uri: str) -> ImageUrlDocument:
return ImageUrlDocument(type="image_url", image_url=data_uri)
@cache
def remote_pdf_document() -> DocumentUrlDocument:
return DocumentUrlDocument(
type="document_url",
document_url="https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf",
)
@cache
def pdf_document() -> DocumentUrlDocument:
return DocumentUrlDocument(type="document_url", document_url=structured_pdf_data_uri())
def document_transport_strategy(inline_image_data_uri: str) -> SearchStrategy[ImageUrlDocument | DocumentUrlDocument]:
transports: Final[tuple[Literal["remote_image", "inline_image", "remote_pdf", "inline_pdf"], ...]] = (
"remote_image",
"inline_image",
"remote_pdf",
"inline_pdf",
)
def as_document(
transport: Literal["remote_image", "inline_image", "remote_pdf", "inline_pdf"],
) -> ImageUrlDocument | DocumentUrlDocument:
if transport == "remote_image":
return image_document("invoice 123", 24)
if transport == "inline_image":
return image_data_document(inline_image_data_uri)
if transport == "remote_pdf":
return remote_pdf_document()
return pdf_document()
return st.sampled_from(transports).map(as_document)
def annotation_format(name: str) -> JsonSchemaResponseFormat:
return JsonSchemaResponseFormat(
type="json_schema",
json_schema=JsonSchemaDefinition(
name=name,
description="Extract the visible document fields",
schema={
"type": "object",
"properties": {"title": {"type": "string"}},
"required": ["title"],
"additionalProperties": False,
},
strict=True,
),
)
def invoke_with_api_key(client: OcrFixtureClient, api_key: str) -> ApiKeyOcrInvocation:
return ApiKeyOcrInvocation(client=client, api_key=api_key)

45
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/config.py
View file

@ -1,45 +0,0 @@
from __future__ import annotations
import os
import subprocess
from collections.abc import Callable, Mapping
from pathlib import Path
from typing import Final
from ......shared.parity.fixtures.store import fixture_directory
FIXTURE_DIR_ENV: Final = "LITELLM_OCR_FIXTURE_DIR"
DEFAULT_FIXTURE_DIRECTORY: Final = Path(__file__).with_name("data")
def read_gcloud(arguments: tuple[str, ...]) -> str:
try:
result: Final = subprocess.run(("gcloud", *arguments), capture_output=True, text=True, timeout=45, check=False)
except (OSError, subprocess.TimeoutExpired):
return ""
return result.stdout.strip() if result.returncode == 0 else ""
def recording_environment(
environ: Mapping[str, str],
command_reader: Callable[[tuple[str, ...]], str] = read_gcloud,
) -> Mapping[str, str]:
project: Final = (
environ.get("VERTEXAI_PROJECT")
or environ.get("VERTEX_PROJECT")
or command_reader(("config", "get-value", "project"))
)
if not project or project == "(unset)":
return environ
token: Final = (
environ.get("VERTEX_AI_ACCESS_TOKEN")
or environ.get("VERTEX_AI_API_KEY")
or command_reader(("auth", "print-access-token"))
)
if not token:
raise SystemExit("Vertex OCR needs OAuth credentials. Run `gcloud auth login` or set VERTEX_AI_ACCESS_TOKEN")
return {**environ, "VERTEXAI_PROJECT": project, "VERTEX_AI_API_KEY": token}
def configured_fixture_directory() -> Path:
return fixture_directory(None, os.environ.get(FIXTURE_DIR_ENV), DEFAULT_FIXTURE_DIRECTORY)

349
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-document-intelligence/15e3f0c2fc4b3745a43ef5fea1d6322e06e90c4e67fab951b77d3804d641946a.yaml
View file

File diff suppressed because one or more lines are too long

283
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-document-intelligence/3b762f4e7105daf31553730a814850e20aecfca328d558dd4ee885150517b883.yaml
View file

File diff suppressed because one or more lines are too long

348
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-document-intelligence/3d023ad4964ce5494b82fa65cbe4cca193095563b871ebc9429603689eb43a5f.yaml
View file

File diff suppressed because one or more lines are too long

170
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-document-intelligence/3d3312cb0c9b2d437bfd85e261dda2f68b620acd7c0141ae8388aeddd130566d.yaml
View file

File diff suppressed because one or more lines are too long

413
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-document-intelligence/4e4d7210fcf07ee5e1c1cb1c4c05ea915600e298b818877f66cb85a3bec066da.yaml
View file

File diff suppressed because one or more lines are too long

349
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-document-intelligence/4f206b5c9c5a6dcf71f26f7243ec8f584f6df1a12313b173f35fc03b24c4f4c7.yaml
View file

File diff suppressed because one or more lines are too long

347
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-document-intelligence/5220af64dcc05df1f086d83616e61091b1173a1e46e2d21d70793e158e213b3b.yaml
View file

File diff suppressed because one or more lines are too long

253
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-document-intelligence/56c320ac28e0a6d0babd23d01e250d2c3bee841589f7161eb6bd3e9f7e198cd2.yaml
View file

File diff suppressed because one or more lines are too long

297
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-document-intelligence/5ffef0f53acf1c10801929c54b345ebbfb17c79ce0ae1b50e90d3ef5f3cad255.yaml
View file

File diff suppressed because one or more lines are too long

348
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-document-intelligence/83f063eda07b9ec10f975fabcbba64b77e86621ae57498d70c66cfb9b1ae1a8c.yaml
View file

File diff suppressed because one or more lines are too long

349
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-document-intelligence/95d5e32a3f3ee6a80d7d156a4310f9672bce0aa3d69a47cca82dd402dca07156.yaml
View file

File diff suppressed because one or more lines are too long

354
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-document-intelligence/b70ee878c8f769527c7e0b1c148efd461cf737b67b63cc18ca127c39c228aee1.yaml
View file

File diff suppressed because one or more lines are too long

360
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-document-intelligence/c16e0aa1a19d5a1816b16f4eddcbebdee92edd7c36cd6efed21995a13e5e4a10.yaml
View file

File diff suppressed because one or more lines are too long

359
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-document-intelligence/ce83e3c90d022c67bb250d24ffd954b33e196aefdd4e4af57bd6ed0a1f1bb771.yaml
View file

File diff suppressed because one or more lines are too long

52
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-document-intelligence/d3a84b92e66a8daa8129daccea438d781b6885e0638f670314ad064b13d73b70.yaml
View file

File diff suppressed because one or more lines are too long

314
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-document-intelligence/e41209d72675cad6ede447d370a1b20eb8f7d8022478869e32ae2a0ec2e84a51.yaml
View file

File diff suppressed because one or more lines are too long

116
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-document-intelligence/ed4c810b78a758aab2475b1eb7c9a5db92e60bc4711fb001fbb30616af660edf.yaml
View file

@ -1,116 +0,0 @@
interactions:
- request:
body: '{"urlSource":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"}'
headers:
Accept:
- '*/*'
Content-Type:
- application/json
User-Agent:
- litellm/1.101.0
method: POST
uri: http://parity-provider.invalid/documentintelligence/documentModels/prebuilt-layout:analyze?api-version=2024-11-30
response:
body:
string: ''
headers:
Date:
- Wed, 02 Sep 2026 22:36:01 GMT
Operation-Location:
- http://parity-provider.invalid/documentintelligence/documentModels/prebuilt-layout/analyzeResults/42554c0a-83ec-4db4-8cf8-e3ff9cd9dd2a?api-version=2024-11-30
Server:
- istio-envoy
Strict-Transport-Security:
- max-age=31536000; includeSubDomains; preload
X-Content-Type-Options:
- nosniff
apim-request-id:
- 42554c0a-83ec-4db4-8cf8-e3ff9cd9dd2a
x-envoy-upstream-service-time:
- '670'
x-ms-region:
- West US 3
status:
code: 202
message: ''
- request:
body: ''
headers:
Accept:
- '*/*'
User-Agent:
- litellm/1.101.0
method: GET
uri: http://parity-provider.invalid/documentintelligence/documentModels/prebuilt-layout/analyzeResults/42554c0a-83ec-4db4-8cf8-e3ff9cd9dd2a?api-version=2024-11-30
response:
body:
string: '{"status":"running","createdDateTime":"2026-09-02T22:36:01Z","lastUpdatedDateTime":"2026-09-02T22:36:01Z"}'
headers:
Content-Type:
- application/json; charset=utf-8
Date:
- Wed, 02 Sep 2026 22:36:01 GMT
Retry-After:
- '4'
Server:
- istio-envoy
Strict-Transport-Security:
- max-age=31536000; includeSubDomains; preload
X-Content-Type-Options:
- nosniff
apim-request-id:
- 011f699b-8d28-4ecb-917b-24cd42ffee29
x-envoy-upstream-service-time:
- '12'
x-ms-region:
- West US 3
status:
code: 200
message: ''
- request:
body: ''
headers:
Accept:
- '*/*'
User-Agent:
- litellm/1.101.0
method: GET
uri: http://parity-provider.invalid/documentintelligence/documentModels/prebuilt-layout/analyzeResults/42554c0a-83ec-4db4-8cf8-e3ff9cd9dd2a?api-version=2024-11-30
response:
body:
string: '{"status":"succeeded","createdDateTime":"2026-09-02T22:36:01Z","lastUpdatedDateTime":"2026-09-02T22:36:02Z","analyzeResult":{"apiVersion":"2024-11-30","modelId":"prebuilt-layout","stringIndexType":"textElements","content":"invoice
123","pages":[{"pageNumber":1,"angle":-0.12780000269412994,"width":800,"height":300,"unit":"pixel","words":[{"content":"invoice","polygon":[338,138,418,139,418,160,338,160],"confidence":0.994,"span":{"offset":0,"length":7}},{"content":"123","polygon":[424,139,461,138,461,160,424,160],"confidence":0.999,"span":{"offset":8,"length":3}}],"lines":[{"content":"invoice
123","polygon":[338,138,460,138,461,160,338,160],"spans":[{"offset":0,"length":11}]}],"spans":[{"offset":0,"length":11}]}],"tables":[],"paragraphs":[{"spans":[{"offset":0,"length":11}],"boundingRegions":[{"pageNumber":1,"polygon":[338,138,460,138,461,160,338,160]}],"role":"title","content":"invoice
123"}],"styles":[],"contentFormat":"text","sections":[{"spans":[{"offset":0,"length":11}],"elements":["/paragraphs/0"]}]}}'
headers:
Content-Type:
- application/json; charset=utf-8
Date:
- Wed, 02 Sep 2026 22:36:06 GMT
Server:
- istio-envoy
Strict-Transport-Security:
- max-age=31536000; includeSubDomains; preload
X-Content-Type-Options:
- nosniff
apim-request-id:
- 8940c1a9-9c86-4188-86bf-8f34bb8fb537
x-envoy-upstream-service-time:
- '32'
x-ms-region:
- West US 3
status:
code: 200
message: ''
recorded_at: '2026-09-02T22:36:06.585674+00:00'
ttl_seconds: 0
version: 1
x-litellm:
case:
litellm_input:
document:
image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24
type: image_url
model: azure_ai/doc-intelligence/prebuilt-layout
request_source: recorded
schema_version: 1

474
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-mistral/0fe41827087ef249498dd70a85575abae2292d009027135599ee05789a7d3885.yaml
View file

File diff suppressed because one or more lines are too long

217
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-mistral/14bf631fc3fd377ab96a27a6a135a20e0b1efdf8c12c66ea93f1f81e0ebab9f5.yaml
View file

File diff suppressed because one or more lines are too long

218
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-mistral/308e3da5b40ae506ea9bcdb1fe72e41eaeaa0a4780107c10ee1d4e6d5fcf8887.yaml
View file

File diff suppressed because one or more lines are too long

218
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-mistral/36a2fec0e0b8c4fd64dc8ffe1d0bbb4fcc93d18612764adfd0e5d9fbca0313a9.yaml
View file

File diff suppressed because one or more lines are too long

216
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-mistral/4253ab0886f20d2ac400b9be2c6b884c94751dd16123b2bccb53f97069034e57.yaml
View file

File diff suppressed because one or more lines are too long

242
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-mistral/498db31114af91033c68d473c84969a342678d6d83f7061d32a4751de19ea114.yaml
View file

File diff suppressed because one or more lines are too long

92
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-mistral/68c08454d47ce41b9d61148737eadf6cfb15c7ec9785f57f8500861c675a0f3f.yaml
View file

File diff suppressed because one or more lines are too long

92
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-mistral/ae44127be834dc1735e216817683ff12a45fb429bffdef1c3960cd68a4d1f00b.yaml
View file

File diff suppressed because one or more lines are too long

92
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-mistral/b2aa9df433e9509c5665a180e8ba4f5ab8601cac8646bab83f819020512e2b94.yaml
View file

@ -1,92 +0,0 @@
interactions:
- request:
body: '{"model":"mistral-ocr-4-0","document":{"type":"image_url","image_url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAUAAAABQCAIAAADnUzvSAAABfUlEQVR4nO3dwY2DMBRAQbza/ltmK+BAhNd+YaYAcEBPPnwC4zzPA2j6Wb0A4HMChjABQ5iAIUzAECZgCBMwhAkYwgQMYQKGMAFD2O9TBxpjHDM99cz2U+vcbT1XdnvW/er3zl7nWHTfZ99fOzCECRjCBAxhAoYwAUOYgCFMwBD22Bx41bz06viz5427rWc3s+efd897Lrrvs++vHRjCBAxhAoYwAUOYgCFMwBAmYAibPgfmne7Ow2ef91vZgSFMwBAmYAgTMIQJGMIEDGEChjBzYNjof8V32YEhTMAQJmAIEzCECRjCBAxhAoYwc2BebSx6X/dTx7cDQ5iAIUzAECZgCBMwhAkYwgQMYebAvML40u8z24EhTMAQJmAIEzCECRjCBAxhAoawsdt7bq/sts7d1nNl1Zxz1XUek6/nbt9DtgNDmIAhTMAQJmAIEzCECRjCBAxhj82Bgf9nB4YwAUOYgCFMwBAmYAgTMIQJGMIEDGEChjABQ5iA4ej6A6GiUaF+b+RaAAAAAElFTkSuQmCC"}}'
headers:
Accept:
- '*/*'
Content-Type:
- application/json
User-Agent:
- litellm/1.101.0
method: POST
uri: http://parity-provider.invalid/providers/mistral/azure/ocr
response:
body:
string: '{"pages":[{"index":0,"markdown":"$$\\text{DOC } 123$$","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":80,"width":320},"confidence_scores":null,"blocks":[{"top_left_x":0,"top_left_y":0,"bottom_right_x":320,"bottom_right_y":80,"content":"$$\\text{DOC
} 123$$","type":"equation"}]}],"model":"mistral-ocr-4-0","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":438}}'
headers:
Content-Type:
- application/json
Date:
- Wed, 02 Sep 2026 22:38:07 GMT
Server:
- istio-envoy
Strict-Transport-Security:
- max-age=31536000; includeSubDomains; preload
X-Content-Type-Options:
- nosniff
apim-request-id:
- 3795070f-a4a7-4ad6-b31d-fe51fdcf29d4
azureai-fe-is-streaming:
- 'False'
azureai-fe-offer-type:
- paygo
azureai-processed-tier:
- default
azureai-requested-tier:
- default
azureml-destination-deployment:
- d20260831221917-7d0e5d7f
azureml-destination-endpoint:
- mo4-use-h1-oai-oe-nvisoc
azureml-destination-model-group:
- aoai-balanced
azureml-destination-region:
- eastus
azureml-model-deployment:
- d20260831221917-7d0e5d7f
azureml-model-group:
- aoai-balanced
azureml-model-session:
- d20260831221917-7d0e5d7f
azureml-ms-model-error-comprehensive:
- --
azureml-served-by-cluster:
- hyena-westus3-01
x-ms-client-request-id:
- 3795070f-a4a7-4ad6-b31d-fe51fdcf29d4
x-ms-is-spilled-over:
- 'false'
x-ms-region:
- West US 3
x-pages-processed:
- '1'
x-ratelimit-abusepenalty-active:
- 'False'
x-ratelimit-key:
- mistral-ocr-4-0
x-ratelimit-limit-requests:
- '10'
x-ratelimit-remaining-requests:
- '8'
x-ratelimit-renewalperiod-requests:
- '60'
x-ratelimit-reset-requests:
- '14'
x-request-id:
- 7a885df6-7464-4f05-b091-1a154c180886
status:
code: 200
message: ''
recorded_at: '2026-09-02T22:38:08.503737+00:00'
ttl_seconds: 0
version: 1
x-litellm:
case:
litellm_input:
document:
image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAUAAAABQCAIAAADnUzvSAAABfUlEQVR4nO3dwY2DMBRAQbza/ltmK+BAhNd+YaYAcEBPPnwC4zzPA2j6Wb0A4HMChjABQ5iAIUzAECZgCBMwhAkYwgQMYQKGMAFD2O9TBxpjHDM99cz2U+vcbT1XdnvW/er3zl7nWHTfZ99fOzCECRjCBAxhAoYwAUOYgCFMwBD22Bx41bz06viz5427rWc3s+efd897Lrrvs++vHRjCBAxhAoYwAUOYgCFMwBAmYAibPgfmne7Ow2ef91vZgSFMwBAmYAgTMIQJGMIEDGEChjBzYNjof8V32YEhTMAQJmAIEzCECRjCBAxhAoYwc2BebSx6X/dTx7cDQ5iAIUzAECZgCBMwhAkYwgQMYebAvML40u8z24EhTMAQJmAIEzCECRjCBAxhAoawsdt7bq/sts7d1nNl1Zxz1XUek6/nbt9DtgNDmIAhTMAQJmAIEzCECRjCBAxhj82Bgf9nB4YwAUOYgCFMwBAmYAgTMIQJGMIEDGEChjABQ5iA4ej6A6GiUaF+b+RaAAAAAElFTkSuQmCC
type: image_url
model: azure_ai/mistral-ocr-4-0
request_source: recorded
schema_version: 1

121
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-mistral/b2e663c74c352dfeca7547535ecdd5e29036ce1e5be4065da1e18cd6a8fde622.yaml
View file

File diff suppressed because one or more lines are too long

145
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-mistral/b6efd962a7fbd486ea42080a1b72df37a37552c0b910190877b0ce56c25b359b.yaml
View file

File diff suppressed because one or more lines are too long

216
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-mistral/d89a8fe095a79d39ba5d796a4ec9cd572747c3cbe4a4e2fefebabacbb36cb80b.yaml
View file

File diff suppressed because one or more lines are too long

218
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-mistral/de7d0818d171507c1ecdd77b4b90adb4c4cc86e1a2376cc8625913aa5b9a5a3d.yaml
View file

File diff suppressed because one or more lines are too long

53
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-mistral/ea745741b68da1e42ada189795fd7ab875d1430d6ab5accdd987dbbfd03a58b0.yaml
View file

File diff suppressed because one or more lines are too long

235
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/azure-mistral/f69c443523a0dbd89d873e6cb5726c3223b63e8cdaa1f397ea19182bd9956704.yaml
View file

File diff suppressed because one or more lines are too long

123
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/00e40122a146d5fd663c145054ae744eab2f1613e845de12ab931c7c36a76a2a.yaml
View file

File diff suppressed because one or more lines are too long

123
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/06548a2b80868916f18a71e91a6ff17459a6e60da3c4b2e8c2529a3373c5f2c2.yaml
View file

File diff suppressed because one or more lines are too long

68
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/0b234402bdfd3be223e731051202f2996578fff02d93ec3011031ce2b4c511fc.yaml
View file

@ -1,68 +0,0 @@
interactions:
- request:
body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"image_limit":1}'
headers:
Accept:
- '*/*'
Content-Type:
- application/json
User-Agent:
- litellm/1.101.0
method: POST
uri: http://parity-provider.invalid/v1/ocr
response:
body:
string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice
123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}'
headers:
CF-RAY:
- a346d3637dc2c090-SJC
Content-Type:
- application/json
Date:
- Tue, 01 Sep 2026 19:54:15 GMT
Server:
- cloudflare
Strict-Transport-Security:
- max-age=15552000; includeSubDomains; preload
X-Content-Type-Options:
- nosniff
access-control-allow-origin:
- '*'
alt-svc:
- h3=":443"; ma=86400
cf-cache-status:
- DYNAMIC
mistral-correlation-id:
- 01a05e89-5a87-7148-8ae4-4e14967bebf3
x-envoy-upstream-service-time:
- '226'
x-kong-proxy-latency:
- '19'
x-kong-request-id:
- 01a05e89-5a87-7148-8ae4-4e14967bebf3
x-kong-upstream-latency:
- '227'
x-ratelimit-limit-ocr-pages-minute:
- '60'
x-ratelimit-ocr-pages-query-cost:
- '1'
x-ratelimit-remaining-ocr-pages-minute:
- '56'
status:
code: 200
message: ''
recorded_at: '2026-09-01T19:54:15.394028+00:00'
ttl_seconds: 0
version: 1
x-litellm:
case:
litellm_input:
contract: mistral
document:
image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24
type: image_url
image_limit: 1
model: mistral/mistral-ocr-latest
request_source: python_replay
schema_version: 1

60
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/1f3001924ef2c46d3eccdedd808c7ded7fea981215a478a12d8636eff8eb1157.yaml
View file

File diff suppressed because one or more lines are too long

69
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/1f38e1eab857c22107df833aa63b119bced6537f45cf08b27cbec08bb8f45951.yaml
View file

@ -1,69 +0,0 @@
interactions:
- request:
body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"pages":[0]}'
headers:
Accept:
- '*/*'
Content-Type:
- application/json
User-Agent:
- litellm/1.101.0
method: POST
uri: http://parity-provider.invalid/v1/ocr
response:
body:
string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice
123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}'
headers:
CF-RAY:
- a346d35d1fbbebe5-SJC
Content-Type:
- application/json
Date:
- Tue, 01 Sep 2026 19:54:14 GMT
Server:
- cloudflare
Strict-Transport-Security:
- max-age=15552000; includeSubDomains; preload
X-Content-Type-Options:
- nosniff
access-control-allow-origin:
- '*'
alt-svc:
- h3=":443"; ma=86400
cf-cache-status:
- DYNAMIC
mistral-correlation-id:
- 01a05e89-5685-747f-b76d-dab242ea7512
x-envoy-upstream-service-time:
- '179'
x-kong-proxy-latency:
- '12'
x-kong-request-id:
- 01a05e89-5685-747f-b76d-dab242ea7512
x-kong-upstream-latency:
- '180'
x-ratelimit-limit-ocr-pages-minute:
- '60'
x-ratelimit-ocr-pages-query-cost:
- '1'
x-ratelimit-remaining-ocr-pages-minute:
- '58'
status:
code: 200
message: ''
recorded_at: '2026-09-01T19:54:14.385374+00:00'
ttl_seconds: 0
version: 1
x-litellm:
case:
litellm_input:
contract: mistral
document:
image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24
type: image_url
model: mistral/mistral-ocr-latest
pages:
- 0
request_source: python_replay
schema_version: 1

207
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/2700b798fa2f8d5f93496e026a16dd99c8b344209a377fe13cc63c9894fc316d.yaml
View file

File diff suppressed because one or more lines are too long

192
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/27fb5f73a7e247d431fcd64805a4d7981c5e459339edf7910e4315d4514bfa69.yaml
View file

File diff suppressed because one or more lines are too long

59
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/288d25b041e6fae7816c1bc8e21d5d427bce5d6e181efa2e9d087c94ca4ce145.yaml
View file

File diff suppressed because one or more lines are too long

85
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/38dbc05d8508355879de894ecc0f46da887570de720a142e9df297bdd280608d.yaml
View file

@ -1,85 +0,0 @@
interactions:
- request:
body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"document_annotation_format":{"type":"json_schema","json_schema":{"name":"document_title","description":"Extract
the visible document fields","schema":{"additionalProperties":false,"properties":{"title":{"type":"string"}},"required":["title"],"type":"object"},"strict":true}},"document_annotation_prompt":"Extract
the visible title"}'
headers:
Accept:
- '*/*'
Content-Type:
- application/json
User-Agent:
- litellm/1.101.0
method: POST
uri: http://parity-provider.invalid/v1/ocr
response:
body:
string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice
123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":"{\"title\":
\"invoice 123\"}","usage_info":{"pages_processed":1,"doc_size_bytes":4124}}'
headers:
CF-RAY:
- a346d376581f74f9-SJC
Content-Type:
- application/json
Date:
- Tue, 01 Sep 2026 19:54:18 GMT
Server:
- cloudflare
Strict-Transport-Security:
- max-age=15552000; includeSubDomains; preload
X-Content-Type-Options:
- nosniff
access-control-allow-origin:
- '*'
alt-svc:
- h3=":443"; ma=86400
cf-cache-status:
- DYNAMIC
mistral-correlation-id:
- 01a05e89-664a-740d-a112-0a1183c302b3
x-envoy-upstream-service-time:
- '402'
x-kong-proxy-latency:
- '20'
x-kong-request-id:
- 01a05e89-664a-740d-a112-0a1183c302b3
x-kong-upstream-latency:
- '403'
x-ratelimit-limit-ocr-pages-minute:
- '60'
x-ratelimit-ocr-pages-query-cost:
- '1'
x-ratelimit-remaining-ocr-pages-minute:
- '52'
status:
code: 200
message: ''
recorded_at: '2026-09-01T19:54:18.915814+00:00'
ttl_seconds: 0
version: 1
x-litellm:
case:
litellm_input:
contract: mistral
document:
image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24
type: image_url
document_annotation_format:
json_schema:
description: Extract the visible document fields
name: document_title
schema:
additionalProperties: false
properties:
title:
type: string
required:
- title
type: object
strict: true
type: json_schema
document_annotation_prompt: Extract the visible title
model: mistral/mistral-ocr-latest
request_source: python_replay
schema_version: 1

209
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/3a6e57388e0c0962a17ea366a6696c0795736f6e2e1c203df2446a7318913f50.yaml
View file

File diff suppressed because one or more lines are too long

123
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/3d1273437ef7a6e55a97a6ddbbebb2d794090691d99dd7c6565c010018032cef.yaml
View file

File diff suppressed because one or more lines are too long

68
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/3d5f42192461789f547b42c8f1b625aa688e6cbd69e01f94d1f3205ef41ab830.yaml
View file

@ -1,68 +0,0 @@
interactions:
- request:
body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"extract_header":true}'
headers:
Accept:
- '*/*'
Content-Type:
- application/json
User-Agent:
- litellm/1.101.0
method: POST
uri: http://parity-provider.invalid/v1/ocr
response:
body:
string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice
123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}'
headers:
CF-RAY:
- a346d37c9944d8a7-SJC
Content-Type:
- application/json
Date:
- Tue, 01 Sep 2026 19:54:19 GMT
Server:
- cloudflare
Strict-Transport-Security:
- max-age=15552000; includeSubDomains; preload
X-Content-Type-Options:
- nosniff
access-control-allow-origin:
- '*'
alt-svc:
- h3=":443"; ma=86400
cf-cache-status:
- DYNAMIC
mistral-correlation-id:
- 01a05e89-6a3a-76c7-85b8-d40a7156155e
x-envoy-upstream-service-time:
- '230'
x-kong-proxy-latency:
- '16'
x-kong-request-id:
- 01a05e89-6a3a-76c7-85b8-d40a7156155e
x-kong-upstream-latency:
- '230'
x-ratelimit-limit-ocr-pages-minute:
- '60'
x-ratelimit-ocr-pages-query-cost:
- '1'
x-ratelimit-remaining-ocr-pages-minute:
- '51'
status:
code: 200
message: ''
recorded_at: '2026-09-01T19:54:19.420285+00:00'
ttl_seconds: 0
version: 1
x-litellm:
case:
litellm_input:
contract: mistral
document:
image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24
type: image_url
extract_header: true
model: mistral/mistral-ocr-latest
request_source: python_replay
schema_version: 1

122
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/49f28a2c1d134cd0db0dff94f573d059706f6222542e400d101b59541a55e7b1.yaml
View file

File diff suppressed because one or more lines are too long

191
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/4b2d15e61e7ddfdd02aa7463edfa4f628f4544f79ab1a98692fde6e4bfa03802.yaml
View file

File diff suppressed because one or more lines are too long

124
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/4b628172d5b53e8d1795487bade4ae02170297d2f5ed69b27bc0c8b7ce0fa896.yaml
View file

File diff suppressed because one or more lines are too long

192
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/4e4d87350d1af889a5ab6d26260a9b75c1ef71dc72e524adfb1479b68725d659.yaml
View file

File diff suppressed because one or more lines are too long

191
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/531b8ac261099643b357fef2064627387abf747c80ee95733af00b3d1957bb09.yaml
View file

File diff suppressed because one or more lines are too long

67
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/54ce0991a21ea668e2373d2b05801606b7039583f55a0f85bfe38c7c15712adf.yaml
View file

@ -1,67 +0,0 @@
interactions:
- request:
body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"}}'
headers:
Accept:
- '*/*'
Content-Type:
- application/json
User-Agent:
- litellm/1.101.0
method: POST
uri: http://parity-provider.invalid/v1/ocr
response:
body:
string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice
123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}'
headers:
CF-RAY:
- a346d356fb3698ce-SJC
Content-Type:
- application/json
Date:
- Tue, 01 Sep 2026 19:54:13 GMT
Server:
- cloudflare
Strict-Transport-Security:
- max-age=15552000; includeSubDomains; preload
X-Content-Type-Options:
- nosniff
access-control-allow-origin:
- '*'
alt-svc:
- h3=":443"; ma=86400
cf-cache-status:
- DYNAMIC
mistral-correlation-id:
- 01a05e89-52b4-7e7b-983e-5bc4e5469b39
x-envoy-upstream-service-time:
- '554'
x-kong-proxy-latency:
- '13'
x-kong-request-id:
- 01a05e89-52b4-7e7b-983e-5bc4e5469b39
x-kong-upstream-latency:
- '557'
x-ratelimit-limit-ocr-pages-minute:
- '60'
x-ratelimit-ocr-pages-query-cost:
- '1'
x-ratelimit-remaining-ocr-pages-minute:
- '59'
status:
code: 200
message: ''
recorded_at: '2026-09-01T19:54:13.880852+00:00'
ttl_seconds: 0
version: 1
x-litellm:
case:
litellm_input:
contract: mistral
document:
image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24
type: image_url
model: mistral/mistral-ocr-latest
request_source: python_replay
schema_version: 1

122
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/6288bbc7fa109d331374994937107ce19497c2230a0dde199f0aa2cdc7375a1b.yaml
View file

File diff suppressed because one or more lines are too long

146
tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/62a60a5c6797e4c8e01f2d7b408bb555da76bcfb03780a0e02f767f5dabdb9e3.yaml
View file

File diff suppressed because one or more lines are too long

Some files were not shown because too many files have changed in this diff Show more