diff --git a/litellm-rust/crates/ai-gateway/tests/ocr_lifecycle.rs b/litellm-rust/crates/ai-gateway/tests/ocr_lifecycle.rs index 465d3b93901..905760a192e 100644 --- a/litellm-rust/crates/ai-gateway/tests/ocr_lifecycle.rs +++ b/litellm-rust/crates/ai-gateway/tests/ocr_lifecycle.rs @@ -638,6 +638,79 @@ async fn azure_ai_mistral_ocr_sends_api_key_as_bearer() { ); } +#[tokio::test] +async fn vertex_mistral_ocr_uses_rust_auth_url_and_body() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("test listener binds"); + let addr = listener.local_addr().expect("listener has local addr"); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accepts request"); + let request = read_http_request(&mut socket).await; + let response_body = r#"{"pages":[{"index":0,"markdown":"ok"}],"model":"mistral-ocr-2505"}"#; + let response = format!( + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", + response_body.len(), + response_body + ); + socket + .write_all(response.as_bytes()) + .await + .expect("writes response"); + request + }); + let api_base = format!("http://{addr}"); + let optional_params = json!({ + "vertex_project": "project-1", + "vertex_location": "us-central1", + "include_image_base64": true + }) + .as_object() + .expect("object") + .clone(); + + let response = ocr(OcrRequest { + model: "mistral-ocr-2505", + document: json!({ + "type": "document_url", + "document_url": "data:application/pdf;base64,YWJj" + }), + api_key: Some("vertex-token"), + api_base: Some(&api_base), + custom_llm_provider: Some("vertex_ai"), + extra_headers: None, + optional_params, + timeout: Some(Duration::from_secs(5)), + callbacks: Vec::new(), + guardrails: Vec::new(), + request_metadata: RequestMetadata::default(), + litellm_call_id: None, + }) + .await + .expect("Vertex OCR request succeeds"); + + assert_eq!(response["pages"][0]["markdown"], "ok"); + let request = server.await.expect("server task completes"); + assert!( + request.starts_with( + "POST /v1/projects/project-1/locations/us-central1/publishers/mistralai/models/mistral-ocr-2505:rawPredict " + ), + "{request}" + ); + assert!( + request + .to_ascii_lowercase() + .contains("authorization: bearer vertex-token"), + "{request}" + ); + assert!( + request.contains(r#""include_image_base64":true"#), + "{request}" + ); + assert!(!request.contains("vertex_project"), "{request}"); + assert!(!request.contains("vertex_location"), "{request}"); +} + #[tokio::test] async fn azure_ai_entra_token_survives_pre_call_guardrail_mapping() { let listener = TcpListener::bind("127.0.0.1:0") diff --git a/litellm-rust/crates/core/src/constants.rs b/litellm-rust/crates/core/src/constants.rs index 885a9895696..174368193a7 100644 --- a/litellm-rust/crates/core/src/constants.rs +++ b/litellm-rust/crates/core/src/constants.rs @@ -62,3 +62,4 @@ pub(crate) const AZURE_AI_OCR_PATH: &str = "/providers/mistral/azure/ocr"; pub(crate) const MISTRAL_OCR_API_BASE: &str = "https://api.mistral.ai/v1"; pub(crate) const REDUCTO_OCR_API_BASE: &str = "https://platform.reducto.ai"; pub(crate) const REDUCTO_ID_PREFIX: &str = "reducto://"; +pub(crate) const VERTEX_OCR_DEFAULT_LOCATION: &str = "us-central1"; diff --git a/litellm-rust/crates/core/src/ocr/backends/mod.rs b/litellm-rust/crates/core/src/ocr/backends/mod.rs index f56513b0e95..595176f4903 100644 --- a/litellm-rust/crates/core/src/ocr/backends/mod.rs +++ b/litellm-rust/crates/core/src/ocr/backends/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod azure_ai; pub(crate) mod azure_document_intelligence; pub(crate) mod mistral; pub(crate) mod reducto; +pub(crate) mod vertex_ai; use serde_json::{Map, Value}; diff --git a/litellm-rust/crates/core/src/ocr/backends/vertex_ai.rs b/litellm-rust/crates/core/src/ocr/backends/vertex_ai.rs new file mode 100644 index 00000000000..ee772236039 --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/backends/vertex_ai.rs @@ -0,0 +1,49 @@ +use serde_json::{Map, Value}; + +use crate::Error; +use crate::constants::VERTEX_OCR_DEFAULT_LOCATION; +use crate::ocr::backends::OcrBackend; +use crate::ocr::error::OcrError; +use crate::ocr::types::OcrConnection; +use crate::providers::vertex_ai::auth::{self, VertexAuthInputs}; + +#[derive(Clone, Debug)] +pub struct VertexBackend; + +impl OcrBackend for VertexBackend { + type Config = VertexAuthInputs; + const PROVIDER: crate::ocr::registry::OcrProvider = crate::ocr::registry::OcrProvider::VertexAi; + + fn decode_config(params: &Map) -> Result { + Ok(VertexAuthInputs::from_optional_params(params)?) + } +} + +pub(crate) fn resolve_integration( + _model: &crate::ocr::registry::OcrModel, +) -> crate::ocr::registry::OcrIntegrationKind { + crate::ocr::registry::OcrIntegrationKind::VertexMistral +} + +pub(crate) async fn authenticate( + connection: &OcrConnection, + config: &VertexAuthInputs, + env_lookup: &(dyn Fn(&str) -> Option + Sync), +) -> Result { + Ok(auth::authenticate( + connection.extra_headers.clone(), + connection.api_key.as_deref(), + config, + None, + env_lookup, + ) + .await?) +} + +pub(crate) fn location( + config: &VertexAuthInputs, + env_lookup: &dyn Fn(&str) -> Option, +) -> String { + auth::resolve_location(config, env_lookup) + .unwrap_or_else(|| VERTEX_OCR_DEFAULT_LOCATION.to_string()) +} diff --git a/litellm-rust/crates/core/src/ocr/integrations/mod.rs b/litellm-rust/crates/core/src/ocr/integrations/mod.rs index 4ec04fec4f1..7fe280c2523 100644 --- a/litellm-rust/crates/core/src/ocr/integrations/mod.rs +++ b/litellm-rust/crates/core/src/ocr/integrations/mod.rs @@ -14,6 +14,7 @@ mod azure_mistral; mod mistral_direct; mod reducto_legacy; mod reducto_v3; +mod vertex_mistral; pub use super::backends::reducto::ReductoUpload; pub use super::document::{DocumentPreparation, PassThrough, RequireInline}; @@ -22,6 +23,7 @@ pub use azure_mistral::AzureMistral; pub use mistral_direct::MistralDirect; pub use reducto_legacy::ReductoLegacy; pub use reducto_v3::ReductoV3; +pub use vertex_mistral::VertexMistral; #[derive(Clone, Copy, PartialEq, Eq)] pub enum GuardrailStage { @@ -128,6 +130,7 @@ macro_rules! for_each_ocr_integration { Mistral, $crate::ocr::integrations::MistralDirect, $crate::ocr::integrations::MistralDirect, Mistral; AzureMistral, $crate::ocr::integrations::AzureMistral, $crate::ocr::integrations::AzureMistral, AzureAi; AzureDocumentIntelligence, $crate::ocr::integrations::AzureDocumentIntelligence, $crate::ocr::integrations::AzureDocumentIntelligence, AzureAi; + VertexMistral, $crate::ocr::integrations::VertexMistral, $crate::ocr::integrations::VertexMistral, VertexAi; ReductoV3, $crate::ocr::integrations::ReductoV3, $crate::ocr::integrations::ReductoV3, Reducto; ReductoLegacy, $crate::ocr::integrations::ReductoLegacy, $crate::ocr::integrations::ReductoLegacy, Reducto; } diff --git a/litellm-rust/crates/core/src/ocr/integrations/vertex_mistral.rs b/litellm-rust/crates/core/src/ocr/integrations/vertex_mistral.rs new file mode 100644 index 00000000000..7c7bb488dac --- /dev/null +++ b/litellm-rust/crates/core/src/ocr/integrations/vertex_mistral.rs @@ -0,0 +1,62 @@ +use crate::ocr::backends::PreparedOcrBackend; +use crate::ocr::backends::vertex_ai::{self, VertexBackend}; +use crate::ocr::document::InlineOcrDocument; +use crate::ocr::error::{OcrError, OcrRequestError}; +use crate::ocr::formats::mistral::{MistralOcrFormat, types::MistralOcrParams}; +use crate::ocr::types::OcrConnection; +use crate::url_utils::ApiUrl; + +use super::{BackendConfig, OcrIntegration}; + +#[derive(Clone, Debug)] +pub struct VertexMistral; + +impl OcrIntegration for VertexMistral { + type Backend = VertexBackend; + type Format = MistralOcrFormat; + type DocumentPreparation = super::RequireInline; + + async fn prepare( + &self, + connection: &OcrConnection, + config: &BackendConfig, + model: &str, + _params: &MistralOcrParams, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + ) -> Result { + let authentication = vertex_ai::authenticate(connection, config, env_lookup).await?; + let location = vertex_ai::location(config, env_lookup); + let default_base = format!("https://{location}-aiplatform.googleapis.com"); + let base = connection.api_base.as_deref().unwrap_or(&default_base); + let prediction = format!("{model}:rawPredict"); + let path = [ + "v1", + "projects", + authentication.project_id.as_str(), + "locations", + location.as_str(), + "publishers", + "mistralai", + "models", + prediction.as_str(), + ]; + let url = ApiUrl::parse(base) + .and_then(|url| url.complete_path(&path)) + .map(|url| url.into_string()) + .map_err(|_| OcrRequestError::RequestField { + path: "api_base".into(), + })?; + Ok(PreparedOcrBackend { + url, + headers: authentication.headers, + }) + } + + fn validate_request_body( + &self, + body: &crate::ocr::formats::mistral::types::MistralOcrRequest, + ) -> Result<(), OcrRequestError> { + InlineOcrDocument::try_from(body.document.clone())?; + Ok(()) + } +} diff --git a/litellm-rust/crates/core/src/ocr/mod.rs b/litellm-rust/crates/core/src/ocr/mod.rs index 0ed3a9cf325..bf10dd5f7fc 100644 --- a/litellm-rust/crates/core/src/ocr/mod.rs +++ b/litellm-rust/crates/core/src/ocr/mod.rs @@ -32,3 +32,6 @@ pub(crate) mod test_support; #[cfg(test)] #[path = "../../tests/ocr.rs"] pub(crate) mod tests; +#[cfg(test)] +#[path = "../../tests/vertex_ai_ocr.rs"] +mod vertex_ai_tests; diff --git a/litellm-rust/crates/core/src/ocr/registry.rs b/litellm-rust/crates/core/src/ocr/registry.rs index d3dbe9c3fd5..0028616adaa 100644 --- a/litellm-rust/crates/core/src/ocr/registry.rs +++ b/litellm-rust/crates/core/src/ocr/registry.rs @@ -6,7 +6,7 @@ use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider} pub use super::backends::OcrBackend; pub use super::integrations::{ - AzureDocumentIntelligence, AzureMistral, MistralDirect, ReductoLegacy, ReductoV3, + AzureDocumentIntelligence, AzureMistral, MistralDirect, ReductoLegacy, ReductoV3, VertexMistral, }; use super::integrations::{BackendConfig, InputParams, OcrIntegration}; @@ -62,6 +62,7 @@ super::integrations::for_each_ocr_integration!(define_integration_types); pub enum OcrProvider { Mistral, AzureAi, + VertexAi, Reducto, } @@ -70,6 +71,7 @@ impl OcrProvider { match self { Self::Mistral => "mistral", Self::AzureAi => "azure_ai", + Self::VertexAi => "vertex_ai", Self::Reducto => "reducto", } } @@ -100,6 +102,7 @@ pub fn resolve_ocr_integration(provider: OcrProvider, model: &OcrModel) -> OcrIn match provider { OcrProvider::Mistral => OcrIntegrationKind::Mistral, OcrProvider::AzureAi => super::backends::azure_ai::resolve_integration(model), + OcrProvider::VertexAi => super::backends::vertex_ai::resolve_integration(model), OcrProvider::Reducto => super::backends::reducto::resolve_integration(model), } } diff --git a/litellm-rust/crates/core/tests/vertex_ai_ocr.rs b/litellm-rust/crates/core/tests/vertex_ai_ocr.rs new file mode 100644 index 00000000000..e04fb388a91 --- /dev/null +++ b/litellm-rust/crates/core/tests/vertex_ai_ocr.rs @@ -0,0 +1,78 @@ +use crate::ocr::integrations::OcrIntegration; +use crate::ocr::integrations::{MistralDirect as MISTRAL, VertexMistral as VERTEX_MISTRAL}; +use crate::ocr::test_support::body; +use crate::ocr::types::OcrConnection; +use crate::providers::vertex_ai::auth::VertexAuthInputs; +use serde_json::json; + +#[tokio::test] +async fn vertex_mistral_reuses_mistral_body_transform() { + let doc = json!({"type":"image_url","image_url":"data:image/png;base64,YWJj"}); + assert_eq!( + body( + &VERTEX_MISTRAL, + "model", + doc.clone(), + json!({"extract_footer":true}) + ) + .await + .unwrap(), + body(&MISTRAL, "model", doc, json!({"extract_footer":true})) + .await + .unwrap() + ); +} +#[tokio::test] +async fn vertex_mistral_url_uses_project_location_and_model() { + let connection = OcrConnection { + api_key: Some("token".into()), + ..Default::default() + }; + let config = VertexAuthInputs::from_optional_params( + json!({"vertex_project":"proj-1","vertex_location":"europe-west4"}) + .as_object() + .unwrap(), + ) + .unwrap(); + let prepared = OcrIntegration::prepare( + &VERTEX_MISTRAL, + &connection, + &config, + "mistral-ocr-maas", + &Default::default(), + &|_| None, + ) + .await + .unwrap(); + assert_eq!( + prepared.url, + "https://europe-west4-aiplatform.googleapis.com/v1/projects/proj-1/locations/europe-west4/publishers/mistralai/models/mistral-ocr-maas:rawPredict" + ); +} + +#[tokio::test] +async fn vertex_mistral_resolves_provider_environment_in_the_backend() { + let connection = OcrConnection { + api_key: Some("token".into()), + ..Default::default() + }; + let prepared = OcrIntegration::prepare( + &VERTEX_MISTRAL, + &connection, + &Default::default(), + "model", + &Default::default(), + &|name| match name { + "VERTEXAI_PROJECT" => Some("environment-project".into()), + "VERTEXAI_LOCATION" => Some("asia-east1".into()), + _ => None, + }, + ) + .await + .unwrap(); + + assert_eq!( + prepared.url, + "https://asia-east1-aiplatform.googleapis.com/v1/projects/environment-project/locations/asia-east1/publishers/mistralai/models/model:rawPredict" + ); +} diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 7a3c4419de3..2179e7883b2 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -58,6 +58,7 @@ class _PreparedOCRRequest: _RUST_OCR_PROVIDERS: Final = { "mistral", "azure_ai", + "vertex_ai", } _AZURE_RUST_AUTH_PARAMS: Final = ( @@ -71,6 +72,15 @@ _AZURE_RUST_AUTH_PARAMS: Final = ( "azure_federated_token_file", ) +_VERTEX_RUST_PARAMS: Final = ( + "vertex_project", + "vertex_ai_project", + "vertex_location", + "vertex_ai_location", + "vertex_credentials", + "vertex_ai_credentials", +) + def _prepare_ocr_request( model: str, @@ -226,9 +236,15 @@ def _rust_bridge_optional_params( if prepared_request.custom_llm_provider == "azure_ai" else {} ) + vertex_params: Final = ( + {name: raw_optional_params[name] for name in _VERTEX_RUST_PARAMS if name in raw_optional_params} + if prepared_request.custom_llm_provider == "vertex_ai" + else {} + ) return { **provider_params, **azure_auth_params, + **vertex_params, } diff --git a/tests/ocr_tests/test_ocr_vertex_ai.py b/tests/ocr_tests/test_ocr_vertex_ai.py index 1842eb063a5..eb089fa84cd 100644 --- a/tests/ocr_tests/test_ocr_vertex_ai.py +++ b/tests/ocr_tests/test_ocr_vertex_ai.py @@ -40,19 +40,17 @@ def load_vertex_ai_credentials(): service_account_key_data = {} # Update the service_account_key_data with environment variables - private_key_id = os.environ.get("VERTEX_AI_PRIVATE_KEY_ID", "") - private_key = os.environ.get("VERTEX_AI_PRIVATE_KEY", "") - private_key = private_key.replace("\\n", "\n") - service_account_key_data["private_key_id"] = private_key_id - service_account_key_data["private_key"] = private_key + private_key_id = os.environ.get("VERTEX_AI_PRIVATE_KEY_ID") + private_key = os.environ.get("VERTEX_AI_PRIVATE_KEY") + if private_key_id: + service_account_key_data["private_key_id"] = private_key_id + if private_key: + service_account_key_data["private_key"] = private_key.replace("\\n", "\n") - # Create a temporary file - with tempfile.NamedTemporaryFile(mode="w+", delete=False) as temp_file: - # Write the updated content to the temporary files - json.dump(service_account_key_data, temp_file, indent=2) - - # Export the temporary file as GOOGLE_APPLICATION_CREDENTIALS - os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.abspath(temp_file.name) + if service_account_key_data.get("private_key"): + with tempfile.NamedTemporaryFile(mode="w+", delete=False) as temp_file: + json.dump(service_account_key_data, temp_file, indent=2) + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.abspath(temp_file.name) class TestVertexAIMistralOCR(BaseOCRTest): diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py index 2f9293bfd48..cfb1f4ca866 100644 --- a/tests/test_litellm/ocr/test_rust_bridge.py +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -690,6 +690,27 @@ def test_ocr_routes_azure_entra_inputs_to_rust_without_python_auth(fake_bridge): } +def test_ocr_routes_vertex_auth_inputs_to_rust(fake_bridge): + response = litellm.ocr( + model="vertex_ai/mistral-ocr-maas", + document=DOCUMENT, + vertex_project="project-1", + vertex_location="us-central1", + vertex_credentials={"type": "service_account", "private_key": "secret"}, + include_image_base64=True, + ) + + assert isinstance(response, OCRResponse) + assert fake_bridge.calls[0]["api_key"] is None + assert fake_bridge.calls[0]["custom_llm_provider"] == "vertex_ai" + assert fake_bridge.calls[0]["optional_params"] == { + "vertex_project": "project-1", + "vertex_location": "us-central1", + "vertex_credentials": {"type": "service_account", "private_key": "secret"}, + "include_image_base64": True, + } + + def test_ocr_rust_path_converts_file_document_before_bridge(fake_bridge): response = litellm.ocr( model=MODEL, @@ -792,12 +813,6 @@ def test_ocr_does_not_route_to_rust_when_disabled(): assert bridge.calls == [] -def test_vertex_ocr_stays_on_python_until_its_stack_boundary(): - prepared = build_prepared_request(custom_llm_provider="vertex_ai") - - assert ocr_main._rust_ocr_supported(prepared) is False - - def test_ocr_falls_back_to_python_when_bridge_unavailable(monkeypatch): """Rust enabled but no bridge available (no injected impl, no compiled wheel): ocr() must degrade to the Python HTTP handler instead of raising."""