mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
feat(vertex_ai): route Mistral OCR through Rust
This commit is contained in:
parent
26ba34dbba
commit
f5ff89dab9
12 changed files with 321 additions and 19 deletions
|
|
@ -638,6 +638,79 @@ async fn azure_ai_mistral_ocr_sends_api_key_as_bearer() {
|
|||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn vertex_mistral_ocr_uses_rust_auth_url_and_body() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("test listener binds");
|
||||
let addr = listener.local_addr().expect("listener has local addr");
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.expect("accepts request");
|
||||
let request = read_http_request(&mut socket).await;
|
||||
let response_body = r#"{"pages":[{"index":0,"markdown":"ok"}],"model":"mistral-ocr-2505"}"#;
|
||||
let response = format!(
|
||||
"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
|
||||
response_body.len(),
|
||||
response_body
|
||||
);
|
||||
socket
|
||||
.write_all(response.as_bytes())
|
||||
.await
|
||||
.expect("writes response");
|
||||
request
|
||||
});
|
||||
let api_base = format!("http://{addr}");
|
||||
let optional_params = json!({
|
||||
"vertex_project": "project-1",
|
||||
"vertex_location": "us-central1",
|
||||
"include_image_base64": true
|
||||
})
|
||||
.as_object()
|
||||
.expect("object")
|
||||
.clone();
|
||||
|
||||
let response = ocr(OcrRequest {
|
||||
model: "mistral-ocr-2505",
|
||||
document: json!({
|
||||
"type": "document_url",
|
||||
"document_url": "data:application/pdf;base64,YWJj"
|
||||
}),
|
||||
api_key: Some("vertex-token"),
|
||||
api_base: Some(&api_base),
|
||||
custom_llm_provider: Some("vertex_ai"),
|
||||
extra_headers: None,
|
||||
optional_params,
|
||||
timeout: Some(Duration::from_secs(5)),
|
||||
callbacks: Vec::new(),
|
||||
guardrails: Vec::new(),
|
||||
request_metadata: RequestMetadata::default(),
|
||||
litellm_call_id: None,
|
||||
})
|
||||
.await
|
||||
.expect("Vertex OCR request succeeds");
|
||||
|
||||
assert_eq!(response["pages"][0]["markdown"], "ok");
|
||||
let request = server.await.expect("server task completes");
|
||||
assert!(
|
||||
request.starts_with(
|
||||
"POST /v1/projects/project-1/locations/us-central1/publishers/mistralai/models/mistral-ocr-2505:rawPredict "
|
||||
),
|
||||
"{request}"
|
||||
);
|
||||
assert!(
|
||||
request
|
||||
.to_ascii_lowercase()
|
||||
.contains("authorization: bearer vertex-token"),
|
||||
"{request}"
|
||||
);
|
||||
assert!(
|
||||
request.contains(r#""include_image_base64":true"#),
|
||||
"{request}"
|
||||
);
|
||||
assert!(!request.contains("vertex_project"), "{request}");
|
||||
assert!(!request.contains("vertex_location"), "{request}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn azure_ai_entra_token_survives_pre_call_guardrail_mapping() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0")
|
||||
|
|
|
|||
|
|
@ -62,3 +62,4 @@ pub(crate) const AZURE_AI_OCR_PATH: &str = "/providers/mistral/azure/ocr";
|
|||
pub(crate) const MISTRAL_OCR_API_BASE: &str = "https://api.mistral.ai/v1";
|
||||
pub(crate) const REDUCTO_OCR_API_BASE: &str = "https://platform.reducto.ai";
|
||||
pub(crate) const REDUCTO_ID_PREFIX: &str = "reducto://";
|
||||
pub(crate) const VERTEX_OCR_DEFAULT_LOCATION: &str = "us-central1";
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ pub(crate) mod azure_ai;
|
|||
pub(crate) mod azure_document_intelligence;
|
||||
pub(crate) mod mistral;
|
||||
pub(crate) mod reducto;
|
||||
pub(crate) mod vertex_ai;
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
|
|
|
|||
49
litellm-rust/crates/core/src/ocr/backends/vertex_ai.rs
Normal file
49
litellm-rust/crates/core/src/ocr/backends/vertex_ai.rs
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::Error;
|
||||
use crate::constants::VERTEX_OCR_DEFAULT_LOCATION;
|
||||
use crate::ocr::backends::OcrBackend;
|
||||
use crate::ocr::error::OcrError;
|
||||
use crate::ocr::types::OcrConnection;
|
||||
use crate::providers::vertex_ai::auth::{self, VertexAuthInputs};
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct VertexBackend;
|
||||
|
||||
impl OcrBackend for VertexBackend {
|
||||
type Config = VertexAuthInputs;
|
||||
const PROVIDER: crate::ocr::registry::OcrProvider = crate::ocr::registry::OcrProvider::VertexAi;
|
||||
|
||||
fn decode_config(params: &Map<String, Value>) -> Result<Self::Config, Error> {
|
||||
Ok(VertexAuthInputs::from_optional_params(params)?)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn resolve_integration(
|
||||
_model: &crate::ocr::registry::OcrModel,
|
||||
) -> crate::ocr::registry::OcrIntegrationKind {
|
||||
crate::ocr::registry::OcrIntegrationKind::VertexMistral
|
||||
}
|
||||
|
||||
pub(crate) async fn authenticate(
|
||||
connection: &OcrConnection,
|
||||
config: &VertexAuthInputs,
|
||||
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
|
||||
) -> Result<auth::VertexAuthentication, OcrError> {
|
||||
Ok(auth::authenticate(
|
||||
connection.extra_headers.clone(),
|
||||
connection.api_key.as_deref(),
|
||||
config,
|
||||
None,
|
||||
env_lookup,
|
||||
)
|
||||
.await?)
|
||||
}
|
||||
|
||||
pub(crate) fn location(
|
||||
config: &VertexAuthInputs,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> String {
|
||||
auth::resolve_location(config, env_lookup)
|
||||
.unwrap_or_else(|| VERTEX_OCR_DEFAULT_LOCATION.to_string())
|
||||
}
|
||||
|
|
@ -14,6 +14,7 @@ mod azure_mistral;
|
|||
mod mistral_direct;
|
||||
mod reducto_legacy;
|
||||
mod reducto_v3;
|
||||
mod vertex_mistral;
|
||||
|
||||
pub use super::backends::reducto::ReductoUpload;
|
||||
pub use super::document::{DocumentPreparation, PassThrough, RequireInline};
|
||||
|
|
@ -22,6 +23,7 @@ pub use azure_mistral::AzureMistral;
|
|||
pub use mistral_direct::MistralDirect;
|
||||
pub use reducto_legacy::ReductoLegacy;
|
||||
pub use reducto_v3::ReductoV3;
|
||||
pub use vertex_mistral::VertexMistral;
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub enum GuardrailStage {
|
||||
|
|
@ -128,6 +130,7 @@ macro_rules! for_each_ocr_integration {
|
|||
Mistral, $crate::ocr::integrations::MistralDirect, $crate::ocr::integrations::MistralDirect, Mistral;
|
||||
AzureMistral, $crate::ocr::integrations::AzureMistral, $crate::ocr::integrations::AzureMistral, AzureAi;
|
||||
AzureDocumentIntelligence, $crate::ocr::integrations::AzureDocumentIntelligence, $crate::ocr::integrations::AzureDocumentIntelligence, AzureAi;
|
||||
VertexMistral, $crate::ocr::integrations::VertexMistral, $crate::ocr::integrations::VertexMistral, VertexAi;
|
||||
ReductoV3, $crate::ocr::integrations::ReductoV3, $crate::ocr::integrations::ReductoV3, Reducto;
|
||||
ReductoLegacy, $crate::ocr::integrations::ReductoLegacy, $crate::ocr::integrations::ReductoLegacy, Reducto;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,62 @@
|
|||
use crate::ocr::backends::PreparedOcrBackend;
|
||||
use crate::ocr::backends::vertex_ai::{self, VertexBackend};
|
||||
use crate::ocr::document::InlineOcrDocument;
|
||||
use crate::ocr::error::{OcrError, OcrRequestError};
|
||||
use crate::ocr::formats::mistral::{MistralOcrFormat, types::MistralOcrParams};
|
||||
use crate::ocr::types::OcrConnection;
|
||||
use crate::url_utils::ApiUrl;
|
||||
|
||||
use super::{BackendConfig, OcrIntegration};
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct VertexMistral;
|
||||
|
||||
impl OcrIntegration for VertexMistral {
|
||||
type Backend = VertexBackend;
|
||||
type Format = MistralOcrFormat;
|
||||
type DocumentPreparation = super::RequireInline;
|
||||
|
||||
async fn prepare(
|
||||
&self,
|
||||
connection: &OcrConnection,
|
||||
config: &BackendConfig<Self>,
|
||||
model: &str,
|
||||
_params: &MistralOcrParams,
|
||||
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
|
||||
) -> Result<PreparedOcrBackend, OcrError> {
|
||||
let authentication = vertex_ai::authenticate(connection, config, env_lookup).await?;
|
||||
let location = vertex_ai::location(config, env_lookup);
|
||||
let default_base = format!("https://{location}-aiplatform.googleapis.com");
|
||||
let base = connection.api_base.as_deref().unwrap_or(&default_base);
|
||||
let prediction = format!("{model}:rawPredict");
|
||||
let path = [
|
||||
"v1",
|
||||
"projects",
|
||||
authentication.project_id.as_str(),
|
||||
"locations",
|
||||
location.as_str(),
|
||||
"publishers",
|
||||
"mistralai",
|
||||
"models",
|
||||
prediction.as_str(),
|
||||
];
|
||||
let url = ApiUrl::parse(base)
|
||||
.and_then(|url| url.complete_path(&path))
|
||||
.map(|url| url.into_string())
|
||||
.map_err(|_| OcrRequestError::RequestField {
|
||||
path: "api_base".into(),
|
||||
})?;
|
||||
Ok(PreparedOcrBackend {
|
||||
url,
|
||||
headers: authentication.headers,
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_request_body(
|
||||
&self,
|
||||
body: &crate::ocr::formats::mistral::types::MistralOcrRequest,
|
||||
) -> Result<(), OcrRequestError> {
|
||||
InlineOcrDocument::try_from(body.document.clone())?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
@ -32,3 +32,6 @@ pub(crate) mod test_support;
|
|||
#[cfg(test)]
|
||||
#[path = "../../tests/ocr.rs"]
|
||||
pub(crate) mod tests;
|
||||
#[cfg(test)]
|
||||
#[path = "../../tests/vertex_ai_ocr.rs"]
|
||||
mod vertex_ai_tests;
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}
|
|||
|
||||
pub use super::backends::OcrBackend;
|
||||
pub use super::integrations::{
|
||||
AzureDocumentIntelligence, AzureMistral, MistralDirect, ReductoLegacy, ReductoV3,
|
||||
AzureDocumentIntelligence, AzureMistral, MistralDirect, ReductoLegacy, ReductoV3, VertexMistral,
|
||||
};
|
||||
use super::integrations::{BackendConfig, InputParams, OcrIntegration};
|
||||
|
||||
|
|
@ -62,6 +62,7 @@ super::integrations::for_each_ocr_integration!(define_integration_types);
|
|||
pub enum OcrProvider {
|
||||
Mistral,
|
||||
AzureAi,
|
||||
VertexAi,
|
||||
Reducto,
|
||||
}
|
||||
|
||||
|
|
@ -70,6 +71,7 @@ impl OcrProvider {
|
|||
match self {
|
||||
Self::Mistral => "mistral",
|
||||
Self::AzureAi => "azure_ai",
|
||||
Self::VertexAi => "vertex_ai",
|
||||
Self::Reducto => "reducto",
|
||||
}
|
||||
}
|
||||
|
|
@ -100,6 +102,7 @@ pub fn resolve_ocr_integration(provider: OcrProvider, model: &OcrModel) -> OcrIn
|
|||
match provider {
|
||||
OcrProvider::Mistral => OcrIntegrationKind::Mistral,
|
||||
OcrProvider::AzureAi => super::backends::azure_ai::resolve_integration(model),
|
||||
OcrProvider::VertexAi => super::backends::vertex_ai::resolve_integration(model),
|
||||
OcrProvider::Reducto => super::backends::reducto::resolve_integration(model),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
78
litellm-rust/crates/core/tests/vertex_ai_ocr.rs
Normal file
78
litellm-rust/crates/core/tests/vertex_ai_ocr.rs
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
use crate::ocr::integrations::OcrIntegration;
|
||||
use crate::ocr::integrations::{MistralDirect as MISTRAL, VertexMistral as VERTEX_MISTRAL};
|
||||
use crate::ocr::test_support::body;
|
||||
use crate::ocr::types::OcrConnection;
|
||||
use crate::providers::vertex_ai::auth::VertexAuthInputs;
|
||||
use serde_json::json;
|
||||
|
||||
#[tokio::test]
|
||||
async fn vertex_mistral_reuses_mistral_body_transform() {
|
||||
let doc = json!({"type":"image_url","image_url":"data:image/png;base64,YWJj"});
|
||||
assert_eq!(
|
||||
body(
|
||||
&VERTEX_MISTRAL,
|
||||
"model",
|
||||
doc.clone(),
|
||||
json!({"extract_footer":true})
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
body(&MISTRAL, "model", doc, json!({"extract_footer":true}))
|
||||
.await
|
||||
.unwrap()
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn vertex_mistral_url_uses_project_location_and_model() {
|
||||
let connection = OcrConnection {
|
||||
api_key: Some("token".into()),
|
||||
..Default::default()
|
||||
};
|
||||
let config = VertexAuthInputs::from_optional_params(
|
||||
json!({"vertex_project":"proj-1","vertex_location":"europe-west4"})
|
||||
.as_object()
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let prepared = OcrIntegration::prepare(
|
||||
&VERTEX_MISTRAL,
|
||||
&connection,
|
||||
&config,
|
||||
"mistral-ocr-maas",
|
||||
&Default::default(),
|
||||
&|_| None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
prepared.url,
|
||||
"https://europe-west4-aiplatform.googleapis.com/v1/projects/proj-1/locations/europe-west4/publishers/mistralai/models/mistral-ocr-maas:rawPredict"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn vertex_mistral_resolves_provider_environment_in_the_backend() {
|
||||
let connection = OcrConnection {
|
||||
api_key: Some("token".into()),
|
||||
..Default::default()
|
||||
};
|
||||
let prepared = OcrIntegration::prepare(
|
||||
&VERTEX_MISTRAL,
|
||||
&connection,
|
||||
&Default::default(),
|
||||
"model",
|
||||
&Default::default(),
|
||||
&|name| match name {
|
||||
"VERTEXAI_PROJECT" => Some("environment-project".into()),
|
||||
"VERTEXAI_LOCATION" => Some("asia-east1".into()),
|
||||
_ => None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
prepared.url,
|
||||
"https://asia-east1-aiplatform.googleapis.com/v1/projects/environment-project/locations/asia-east1/publishers/mistralai/models/model:rawPredict"
|
||||
);
|
||||
}
|
||||
|
|
@ -58,6 +58,7 @@ class _PreparedOCRRequest:
|
|||
_RUST_OCR_PROVIDERS: Final = {
|
||||
"mistral",
|
||||
"azure_ai",
|
||||
"vertex_ai",
|
||||
}
|
||||
|
||||
_AZURE_RUST_AUTH_PARAMS: Final = (
|
||||
|
|
@ -71,6 +72,15 @@ _AZURE_RUST_AUTH_PARAMS: Final = (
|
|||
"azure_federated_token_file",
|
||||
)
|
||||
|
||||
_VERTEX_RUST_PARAMS: Final = (
|
||||
"vertex_project",
|
||||
"vertex_ai_project",
|
||||
"vertex_location",
|
||||
"vertex_ai_location",
|
||||
"vertex_credentials",
|
||||
"vertex_ai_credentials",
|
||||
)
|
||||
|
||||
|
||||
def _prepare_ocr_request(
|
||||
model: str,
|
||||
|
|
@ -226,9 +236,15 @@ def _rust_bridge_optional_params(
|
|||
if prepared_request.custom_llm_provider == "azure_ai"
|
||||
else {}
|
||||
)
|
||||
vertex_params: Final = (
|
||||
{name: raw_optional_params[name] for name in _VERTEX_RUST_PARAMS if name in raw_optional_params}
|
||||
if prepared_request.custom_llm_provider == "vertex_ai"
|
||||
else {}
|
||||
)
|
||||
return {
|
||||
**provider_params,
|
||||
**azure_auth_params,
|
||||
**vertex_params,
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -40,19 +40,17 @@ def load_vertex_ai_credentials():
|
|||
service_account_key_data = {}
|
||||
|
||||
# Update the service_account_key_data with environment variables
|
||||
private_key_id = os.environ.get("VERTEX_AI_PRIVATE_KEY_ID", "")
|
||||
private_key = os.environ.get("VERTEX_AI_PRIVATE_KEY", "")
|
||||
private_key = private_key.replace("\\n", "\n")
|
||||
service_account_key_data["private_key_id"] = private_key_id
|
||||
service_account_key_data["private_key"] = private_key
|
||||
private_key_id = os.environ.get("VERTEX_AI_PRIVATE_KEY_ID")
|
||||
private_key = os.environ.get("VERTEX_AI_PRIVATE_KEY")
|
||||
if private_key_id:
|
||||
service_account_key_data["private_key_id"] = private_key_id
|
||||
if private_key:
|
||||
service_account_key_data["private_key"] = private_key.replace("\\n", "\n")
|
||||
|
||||
# Create a temporary file
|
||||
with tempfile.NamedTemporaryFile(mode="w+", delete=False) as temp_file:
|
||||
# Write the updated content to the temporary files
|
||||
json.dump(service_account_key_data, temp_file, indent=2)
|
||||
|
||||
# Export the temporary file as GOOGLE_APPLICATION_CREDENTIALS
|
||||
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.abspath(temp_file.name)
|
||||
if service_account_key_data.get("private_key"):
|
||||
with tempfile.NamedTemporaryFile(mode="w+", delete=False) as temp_file:
|
||||
json.dump(service_account_key_data, temp_file, indent=2)
|
||||
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.abspath(temp_file.name)
|
||||
|
||||
|
||||
class TestVertexAIMistralOCR(BaseOCRTest):
|
||||
|
|
|
|||
|
|
@ -690,6 +690,27 @@ def test_ocr_routes_azure_entra_inputs_to_rust_without_python_auth(fake_bridge):
|
|||
}
|
||||
|
||||
|
||||
def test_ocr_routes_vertex_auth_inputs_to_rust(fake_bridge):
|
||||
response = litellm.ocr(
|
||||
model="vertex_ai/mistral-ocr-maas",
|
||||
document=DOCUMENT,
|
||||
vertex_project="project-1",
|
||||
vertex_location="us-central1",
|
||||
vertex_credentials={"type": "service_account", "private_key": "secret"},
|
||||
include_image_base64=True,
|
||||
)
|
||||
|
||||
assert isinstance(response, OCRResponse)
|
||||
assert fake_bridge.calls[0]["api_key"] is None
|
||||
assert fake_bridge.calls[0]["custom_llm_provider"] == "vertex_ai"
|
||||
assert fake_bridge.calls[0]["optional_params"] == {
|
||||
"vertex_project": "project-1",
|
||||
"vertex_location": "us-central1",
|
||||
"vertex_credentials": {"type": "service_account", "private_key": "secret"},
|
||||
"include_image_base64": True,
|
||||
}
|
||||
|
||||
|
||||
def test_ocr_rust_path_converts_file_document_before_bridge(fake_bridge):
|
||||
response = litellm.ocr(
|
||||
model=MODEL,
|
||||
|
|
@ -792,12 +813,6 @@ def test_ocr_does_not_route_to_rust_when_disabled():
|
|||
assert bridge.calls == []
|
||||
|
||||
|
||||
def test_vertex_ocr_stays_on_python_until_its_stack_boundary():
|
||||
prepared = build_prepared_request(custom_llm_provider="vertex_ai")
|
||||
|
||||
assert ocr_main._rust_ocr_supported(prepared) is False
|
||||
|
||||
|
||||
def test_ocr_falls_back_to_python_when_bridge_unavailable(monkeypatch):
|
||||
"""Rust enabled but no bridge available (no injected impl, no compiled wheel):
|
||||
ocr() must degrade to the Python HTTP handler instead of raising."""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue