feat(vertex_ai): route Mistral OCR through Rust

This commit is contained in:
Yujong Lee 2026-09-09 18:29:48 -07:00
parent 26ba34dbba
commit f5ff89dab9
12 changed files with 321 additions and 19 deletions

View file

@ -638,6 +638,79 @@ async fn azure_ai_mistral_ocr_sends_api_key_as_bearer() {
);
}
#[tokio::test]
async fn vertex_mistral_ocr_uses_rust_auth_url_and_body() {
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("test listener binds");
let addr = listener.local_addr().expect("listener has local addr");
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.expect("accepts request");
let request = read_http_request(&mut socket).await;
let response_body = r#"{"pages":[{"index":0,"markdown":"ok"}],"model":"mistral-ocr-2505"}"#;
let response = format!(
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
response_body.len(),
response_body
);
socket
.write_all(response.as_bytes())
.await
.expect("writes response");
request
});
let api_base = format!("http://{addr}");
let optional_params = json!({
"vertex_project": "project-1",
"vertex_location": "us-central1",
"include_image_base64": true
})
.as_object()
.expect("object")
.clone();
let response = ocr(OcrRequest {
model: "mistral-ocr-2505",
document: json!({
"type": "document_url",
"document_url": "data:application/pdf;base64,YWJj"
}),
api_key: Some("vertex-token"),
api_base: Some(&api_base),
custom_llm_provider: Some("vertex_ai"),
extra_headers: None,
optional_params,
timeout: Some(Duration::from_secs(5)),
callbacks: Vec::new(),
guardrails: Vec::new(),
request_metadata: RequestMetadata::default(),
litellm_call_id: None,
})
.await
.expect("Vertex OCR request succeeds");
assert_eq!(response["pages"][0]["markdown"], "ok");
let request = server.await.expect("server task completes");
assert!(
request.starts_with(
"POST /v1/projects/project-1/locations/us-central1/publishers/mistralai/models/mistral-ocr-2505:rawPredict "
),
"{request}"
);
assert!(
request
.to_ascii_lowercase()
.contains("authorization: bearer vertex-token"),
"{request}"
);
assert!(
request.contains(r#""include_image_base64":true"#),
"{request}"
);
assert!(!request.contains("vertex_project"), "{request}");
assert!(!request.contains("vertex_location"), "{request}");
}
#[tokio::test]
async fn azure_ai_entra_token_survives_pre_call_guardrail_mapping() {
let listener = TcpListener::bind("127.0.0.1:0")

View file

@ -62,3 +62,4 @@ pub(crate) const AZURE_AI_OCR_PATH: &str = "/providers/mistral/azure/ocr";
pub(crate) const MISTRAL_OCR_API_BASE: &str = "https://api.mistral.ai/v1";
pub(crate) const REDUCTO_OCR_API_BASE: &str = "https://platform.reducto.ai";
pub(crate) const REDUCTO_ID_PREFIX: &str = "reducto://";
pub(crate) const VERTEX_OCR_DEFAULT_LOCATION: &str = "us-central1";

View file

@ -2,6 +2,7 @@ pub(crate) mod azure_ai;
pub(crate) mod azure_document_intelligence;
pub(crate) mod mistral;
pub(crate) mod reducto;
pub(crate) mod vertex_ai;
use serde_json::{Map, Value};

View file

@ -0,0 +1,49 @@
use serde_json::{Map, Value};
use crate::Error;
use crate::constants::VERTEX_OCR_DEFAULT_LOCATION;
use crate::ocr::backends::OcrBackend;
use crate::ocr::error::OcrError;
use crate::ocr::types::OcrConnection;
use crate::providers::vertex_ai::auth::{self, VertexAuthInputs};
#[derive(Clone, Debug)]
pub struct VertexBackend;
impl OcrBackend for VertexBackend {
type Config = VertexAuthInputs;
const PROVIDER: crate::ocr::registry::OcrProvider = crate::ocr::registry::OcrProvider::VertexAi;
fn decode_config(params: &Map<String, Value>) -> Result<Self::Config, Error> {
Ok(VertexAuthInputs::from_optional_params(params)?)
}
}
pub(crate) fn resolve_integration(
_model: &crate::ocr::registry::OcrModel,
) -> crate::ocr::registry::OcrIntegrationKind {
crate::ocr::registry::OcrIntegrationKind::VertexMistral
}
pub(crate) async fn authenticate(
connection: &OcrConnection,
config: &VertexAuthInputs,
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
) -> Result<auth::VertexAuthentication, OcrError> {
Ok(auth::authenticate(
connection.extra_headers.clone(),
connection.api_key.as_deref(),
config,
None,
env_lookup,
)
.await?)
}
pub(crate) fn location(
config: &VertexAuthInputs,
env_lookup: &dyn Fn(&str) -> Option<String>,
) -> String {
auth::resolve_location(config, env_lookup)
.unwrap_or_else(|| VERTEX_OCR_DEFAULT_LOCATION.to_string())
}

View file

@ -14,6 +14,7 @@ mod azure_mistral;
mod mistral_direct;
mod reducto_legacy;
mod reducto_v3;
mod vertex_mistral;
pub use super::backends::reducto::ReductoUpload;
pub use super::document::{DocumentPreparation, PassThrough, RequireInline};
@ -22,6 +23,7 @@ pub use azure_mistral::AzureMistral;
pub use mistral_direct::MistralDirect;
pub use reducto_legacy::ReductoLegacy;
pub use reducto_v3::ReductoV3;
pub use vertex_mistral::VertexMistral;
#[derive(Clone, Copy, PartialEq, Eq)]
pub enum GuardrailStage {
@ -128,6 +130,7 @@ macro_rules! for_each_ocr_integration {
Mistral, $crate::ocr::integrations::MistralDirect, $crate::ocr::integrations::MistralDirect, Mistral;
AzureMistral, $crate::ocr::integrations::AzureMistral, $crate::ocr::integrations::AzureMistral, AzureAi;
AzureDocumentIntelligence, $crate::ocr::integrations::AzureDocumentIntelligence, $crate::ocr::integrations::AzureDocumentIntelligence, AzureAi;
VertexMistral, $crate::ocr::integrations::VertexMistral, $crate::ocr::integrations::VertexMistral, VertexAi;
ReductoV3, $crate::ocr::integrations::ReductoV3, $crate::ocr::integrations::ReductoV3, Reducto;
ReductoLegacy, $crate::ocr::integrations::ReductoLegacy, $crate::ocr::integrations::ReductoLegacy, Reducto;
}

View file

@ -0,0 +1,62 @@
use crate::ocr::backends::PreparedOcrBackend;
use crate::ocr::backends::vertex_ai::{self, VertexBackend};
use crate::ocr::document::InlineOcrDocument;
use crate::ocr::error::{OcrError, OcrRequestError};
use crate::ocr::formats::mistral::{MistralOcrFormat, types::MistralOcrParams};
use crate::ocr::types::OcrConnection;
use crate::url_utils::ApiUrl;
use super::{BackendConfig, OcrIntegration};
#[derive(Clone, Debug)]
pub struct VertexMistral;
impl OcrIntegration for VertexMistral {
type Backend = VertexBackend;
type Format = MistralOcrFormat;
type DocumentPreparation = super::RequireInline;
async fn prepare(
&self,
connection: &OcrConnection,
config: &BackendConfig<Self>,
model: &str,
_params: &MistralOcrParams,
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
) -> Result<PreparedOcrBackend, OcrError> {
let authentication = vertex_ai::authenticate(connection, config, env_lookup).await?;
let location = vertex_ai::location(config, env_lookup);
let default_base = format!("https://{location}-aiplatform.googleapis.com");
let base = connection.api_base.as_deref().unwrap_or(&default_base);
let prediction = format!("{model}:rawPredict");
let path = [
"v1",
"projects",
authentication.project_id.as_str(),
"locations",
location.as_str(),
"publishers",
"mistralai",
"models",
prediction.as_str(),
];
let url = ApiUrl::parse(base)
.and_then(|url| url.complete_path(&path))
.map(|url| url.into_string())
.map_err(|_| OcrRequestError::RequestField {
path: "api_base".into(),
})?;
Ok(PreparedOcrBackend {
url,
headers: authentication.headers,
})
}
fn validate_request_body(
&self,
body: &crate::ocr::formats::mistral::types::MistralOcrRequest,
) -> Result<(), OcrRequestError> {
InlineOcrDocument::try_from(body.document.clone())?;
Ok(())
}
}

View file

@ -32,3 +32,6 @@ pub(crate) mod test_support;
#[cfg(test)]
#[path = "../../tests/ocr.rs"]
pub(crate) mod tests;
#[cfg(test)]
#[path = "../../tests/vertex_ai_ocr.rs"]
mod vertex_ai_tests;

View file

@ -6,7 +6,7 @@ use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}
pub use super::backends::OcrBackend;
pub use super::integrations::{
AzureDocumentIntelligence, AzureMistral, MistralDirect, ReductoLegacy, ReductoV3,
AzureDocumentIntelligence, AzureMistral, MistralDirect, ReductoLegacy, ReductoV3, VertexMistral,
};
use super::integrations::{BackendConfig, InputParams, OcrIntegration};
@ -62,6 +62,7 @@ super::integrations::for_each_ocr_integration!(define_integration_types);
pub enum OcrProvider {
Mistral,
AzureAi,
VertexAi,
Reducto,
}
@ -70,6 +71,7 @@ impl OcrProvider {
match self {
Self::Mistral => "mistral",
Self::AzureAi => "azure_ai",
Self::VertexAi => "vertex_ai",
Self::Reducto => "reducto",
}
}
@ -100,6 +102,7 @@ pub fn resolve_ocr_integration(provider: OcrProvider, model: &OcrModel) -> OcrIn
match provider {
OcrProvider::Mistral => OcrIntegrationKind::Mistral,
OcrProvider::AzureAi => super::backends::azure_ai::resolve_integration(model),
OcrProvider::VertexAi => super::backends::vertex_ai::resolve_integration(model),
OcrProvider::Reducto => super::backends::reducto::resolve_integration(model),
}
}

View file

@ -0,0 +1,78 @@
use crate::ocr::integrations::OcrIntegration;
use crate::ocr::integrations::{MistralDirect as MISTRAL, VertexMistral as VERTEX_MISTRAL};
use crate::ocr::test_support::body;
use crate::ocr::types::OcrConnection;
use crate::providers::vertex_ai::auth::VertexAuthInputs;
use serde_json::json;
#[tokio::test]
async fn vertex_mistral_reuses_mistral_body_transform() {
let doc = json!({"type":"image_url","image_url":"data:image/png;base64,YWJj"});
assert_eq!(
body(
&VERTEX_MISTRAL,
"model",
doc.clone(),
json!({"extract_footer":true})
)
.await
.unwrap(),
body(&MISTRAL, "model", doc, json!({"extract_footer":true}))
.await
.unwrap()
);
}
#[tokio::test]
async fn vertex_mistral_url_uses_project_location_and_model() {
let connection = OcrConnection {
api_key: Some("token".into()),
..Default::default()
};
let config = VertexAuthInputs::from_optional_params(
json!({"vertex_project":"proj-1","vertex_location":"europe-west4"})
.as_object()
.unwrap(),
)
.unwrap();
let prepared = OcrIntegration::prepare(
&VERTEX_MISTRAL,
&connection,
&config,
"mistral-ocr-maas",
&Default::default(),
&|_| None,
)
.await
.unwrap();
assert_eq!(
prepared.url,
"https://europe-west4-aiplatform.googleapis.com/v1/projects/proj-1/locations/europe-west4/publishers/mistralai/models/mistral-ocr-maas:rawPredict"
);
}
#[tokio::test]
async fn vertex_mistral_resolves_provider_environment_in_the_backend() {
let connection = OcrConnection {
api_key: Some("token".into()),
..Default::default()
};
let prepared = OcrIntegration::prepare(
&VERTEX_MISTRAL,
&connection,
&Default::default(),
"model",
&Default::default(),
&|name| match name {
"VERTEXAI_PROJECT" => Some("environment-project".into()),
"VERTEXAI_LOCATION" => Some("asia-east1".into()),
_ => None,
},
)
.await
.unwrap();
assert_eq!(
prepared.url,
"https://asia-east1-aiplatform.googleapis.com/v1/projects/environment-project/locations/asia-east1/publishers/mistralai/models/model:rawPredict"
);
}

View file

@ -58,6 +58,7 @@ class _PreparedOCRRequest:
_RUST_OCR_PROVIDERS: Final = {
"mistral",
"azure_ai",
"vertex_ai",
}
_AZURE_RUST_AUTH_PARAMS: Final = (
@ -71,6 +72,15 @@ _AZURE_RUST_AUTH_PARAMS: Final = (
"azure_federated_token_file",
)
_VERTEX_RUST_PARAMS: Final = (
"vertex_project",
"vertex_ai_project",
"vertex_location",
"vertex_ai_location",
"vertex_credentials",
"vertex_ai_credentials",
)
def _prepare_ocr_request(
model: str,
@ -226,9 +236,15 @@ def _rust_bridge_optional_params(
if prepared_request.custom_llm_provider == "azure_ai"
else {}
)
vertex_params: Final = (
{name: raw_optional_params[name] for name in _VERTEX_RUST_PARAMS if name in raw_optional_params}
if prepared_request.custom_llm_provider == "vertex_ai"
else {}
)
return {
**provider_params,
**azure_auth_params,
**vertex_params,
}

View file

@ -40,19 +40,17 @@ def load_vertex_ai_credentials():
service_account_key_data = {}
# Update the service_account_key_data with environment variables
private_key_id = os.environ.get("VERTEX_AI_PRIVATE_KEY_ID", "")
private_key = os.environ.get("VERTEX_AI_PRIVATE_KEY", "")
private_key = private_key.replace("\\n", "\n")
service_account_key_data["private_key_id"] = private_key_id
service_account_key_data["private_key"] = private_key
private_key_id = os.environ.get("VERTEX_AI_PRIVATE_KEY_ID")
private_key = os.environ.get("VERTEX_AI_PRIVATE_KEY")
if private_key_id:
service_account_key_data["private_key_id"] = private_key_id
if private_key:
service_account_key_data["private_key"] = private_key.replace("\\n", "\n")
# Create a temporary file
with tempfile.NamedTemporaryFile(mode="w+", delete=False) as temp_file:
# Write the updated content to the temporary files
json.dump(service_account_key_data, temp_file, indent=2)
# Export the temporary file as GOOGLE_APPLICATION_CREDENTIALS
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.abspath(temp_file.name)
if service_account_key_data.get("private_key"):
with tempfile.NamedTemporaryFile(mode="w+", delete=False) as temp_file:
json.dump(service_account_key_data, temp_file, indent=2)
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.abspath(temp_file.name)
class TestVertexAIMistralOCR(BaseOCRTest):

View file

@ -690,6 +690,27 @@ def test_ocr_routes_azure_entra_inputs_to_rust_without_python_auth(fake_bridge):
}
def test_ocr_routes_vertex_auth_inputs_to_rust(fake_bridge):
response = litellm.ocr(
model="vertex_ai/mistral-ocr-maas",
document=DOCUMENT,
vertex_project="project-1",
vertex_location="us-central1",
vertex_credentials={"type": "service_account", "private_key": "secret"},
include_image_base64=True,
)
assert isinstance(response, OCRResponse)
assert fake_bridge.calls[0]["api_key"] is None
assert fake_bridge.calls[0]["custom_llm_provider"] == "vertex_ai"
assert fake_bridge.calls[0]["optional_params"] == {
"vertex_project": "project-1",
"vertex_location": "us-central1",
"vertex_credentials": {"type": "service_account", "private_key": "secret"},
"include_image_base64": True,
}
def test_ocr_rust_path_converts_file_document_before_bridge(fake_bridge):
response = litellm.ocr(
model=MODEL,
@ -792,12 +813,6 @@ def test_ocr_does_not_route_to_rust_when_disabled():
assert bridge.calls == []
def test_vertex_ocr_stays_on_python_until_its_stack_boundary():
prepared = build_prepared_request(custom_llm_provider="vertex_ai")
assert ocr_main._rust_ocr_supported(prepared) is False
def test_ocr_falls_back_to_python_when_bridge_unavailable(monkeypatch):
"""Rust enabled but no bridge available (no injected impl, no compiled wheel):
ocr() must degrade to the Python HTTP handler instead of raising."""