refactor(rust): extract inference-ocr crate (#44832)

Co-authored-by: Yujong Lee <yujong@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm-rust/Cargo.lock

@@ -3838,6 +3838,7 @@ dependencies = [
  "litellm-inference",
  "litellm-inference-chat",
  "litellm-inference-messages",
+ "litellm-inference-ocr",
  "litellm-inference-responses",
  "litellm-inference-transcription",
  "litellm-llms",
@@ -4095,6 +4096,36 @@ dependencies = [
  "wiremock",
 ]
 
+[[package]]
+name = "litellm-inference-ocr"
+version = "0.1.0"
+dependencies = [
+ "base64 0.22.1",
+ "bytes",
+ "futures-util",
+ "litellm-auth",
+ "litellm-auth-aws",
+ "litellm-auth-gcp",
+ "litellm-core-utils",
+ "litellm-host",
+ "litellm-host-native",
+ "litellm-http",
+ "litellm-inference",
+ "litellm-llms",
+ "litellm-llms-types",
+ "litellm-secrets",
+ "litellm-tracing",
+ "mime_guess",
+ "rand 0.8.7",
+ "rstest",
+ "serde",
+ "serde_json",
+ "time",
+ "tokio",
+ "tracing",
+ "wiremock",
+]
+
 [[package]]
 name = "litellm-inference-responses"
 version = "0.1.0"
@@ -4226,6 +4257,7 @@ dependencies = [
  "litellm-inference",
  "litellm-inference-chat",
  "litellm-inference-messages",
+ "litellm-inference-ocr",
  "litellm-inference-responses",
  "litellm-inference-transcription",
  "litellm-llms",

litellm-rust/Cargo.toml

@@ -21,6 +21,7 @@ litellm-inference-transcription = { path = "crates/inference-transcription" }
 litellm-inference-responses = { path = "crates/inference-responses" }
 litellm-inference-messages = { path = "crates/inference-messages" }
 litellm-inference-chat = { path = "crates/inference-chat" }
+litellm-inference-ocr = { path = "crates/inference-ocr" }
 litellm-gateway-mcp = { path = "crates/gateway-mcp" }
 litellm-gateway = { path = "crates/gateway" }
 litellm-gateway-inference = { path = "crates/gateway-inference" }

litellm-rust/crates/gateway-inference/Cargo.toml

@@ -17,6 +17,7 @@ litellm-inference-transcription.workspace = true
 litellm-inference-responses.workspace = true
 litellm-inference-messages.workspace = true
 litellm-inference-chat.workspace = true
+litellm-inference-ocr.workspace = true
 litellm-host-http.workspace = true
 litellm-host.workspace = true
 litellm-http.workspace = true

litellm-rust/crates/gateway-inference/src/lib.rs

@@ -16,9 +16,10 @@ use std::sync::Arc;
 
 use axum::{Router, routing::post};
 use litellm_http::{ClientVariant, HttpClientConfig, media::UrlPolicy};
-use litellm_inference::{ocr::OcrRoute, resources::CoreResources};
+use litellm_inference::resources::CoreResources;
 use litellm_inference_chat::ChatCompletionsRoute;
 use litellm_inference_messages::MessagesRoute;
+use litellm_inference_ocr::OcrRoute;
 use litellm_inference_responses::ResponsesRoute;
 use litellm_inference_transcription::AudioTranscriptionRoute;
 use litellm_llms::base_llm::ocr::{handler::OcrClient, settings::OcrSettings};

litellm-rust/crates/gateway-inference/src/ocr.rs

@@ -3,7 +3,7 @@ use std::sync::Arc;
 
 use axum::{Json, extract::State, http::HeaderMap, response::IntoResponse};
 use litellm_auth::SecretValue;
-use litellm_inference::ocr::types::{LiteLLMOcrRequest, OcrConnectionInputs, OcrDocumentInput};
+use litellm_inference_ocr::types::{LiteLLMOcrRequest, OcrConnectionInputs, OcrDocumentInput};
 use litellm_llms::base_llm::ocr::transformation::decode_request_value;
 use litellm_llms_types::formats::ocr::OcrDocument;
 use serde_json::Value;

litellm-rust/crates/inference-ocr/Cargo.toml

@@ -0,0 +1,36 @@
+[package]
+name = "litellm-inference-ocr"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+base64.workspace = true
+bytes.workspace = true
+futures-util.workspace = true
+litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] }
+litellm-core-utils.workspace = true
+litellm-host.workspace = true
+litellm-http.workspace = true
+litellm-inference.workspace = true
+litellm-llms.workspace = true
+litellm-llms-types.workspace = true
+litellm-secrets.workspace = true
+litellm-tracing.workspace = true
+mime_guess = "2.0.5"
+serde.workspace = true
+serde_json = { workspace = true, features = ["preserve_order"] }
+tokio.workspace = true
+tracing.workspace = true
+
+[dev-dependencies]
+litellm-auth-aws.workspace = true
+litellm-auth-gcp.workspace = true
+litellm-host-native.workspace = true
+litellm-http = { workspace = true, features = ["test-support"] }
+litellm-inference = { workspace = true, features = ["test-support"] }
+rand.workspace = true
+rstest.workspace = true
+time.workspace = true
+wiremock.workspace = true

litellm-rust/crates/inference-ocr/src/client.rs

@@ -47,7 +47,7 @@ impl OcrRoute {
         interceptors: &impl Interceptors<Error>,
         observers: Option<&ObservationSender>,
     ) -> Result<LiteLLMOcrResponse, Error> {
-        crate::diagnostic::unary(async {
+        litellm_inference::diagnostic::unary(async {
             let caller_document = matches!(&request.document, OcrDocumentInput::Document(_));
             let prepared = prepare_request_document(request).await?;
             let execute: futures_util::future::BoxFuture<'_, Result<LiteLLMOcrResponse, Error>> =

litellm-rust/crates/inference-ocr/src/document.rs

@@ -4,7 +4,7 @@ use base64::{Engine, engine::general_purpose::STANDARD};
 use litellm_llms::base_llm::ocr::{error::Error, transformation::OCR_INLINE_MAX_BYTES};
 use litellm_llms_types::formats::ocr::OcrDocument;
 
-use crate::ocr::types::OcrDocumentInput;
+use crate::types::OcrDocumentInput;
 
 pub fn prepare_document(input: OcrDocumentInput) -> Result<OcrDocument, Error> {
     match input {

litellm-rust/crates/inference-ocr/src/handler.rs

@@ -10,7 +10,7 @@ use litellm_llms_types::formats::ocr::LiteLLMOcrResponse;
 use serde_json::Value;
 
 use super::{arguments::is_secret_param, prepare::prepare_request, provider_config::OcrConfigKind};
-use crate::ocr::types::ResolvedOcrRequest;
+use crate::types::ResolvedOcrRequest;
 
 pub(crate) async fn perform_ocr_request(
     client: &OcrClient,

litellm-rust/crates/inference-ocr/src/prepare.rs

@@ -6,7 +6,7 @@ use litellm_llms::base_llm::ocr::{
 };
 use litellm_secrets::source::Secrets;
 
-use crate::ocr::types::{LiteLLMOcrRequest, ResolvedOcrRequest};
+use crate::types::{LiteLLMOcrRequest, ResolvedOcrRequest};
 
 pub(crate) fn prepare_request(
     request: ResolvedOcrRequest,
@@ -95,7 +95,7 @@ mod tests {
     use serde_json::{Value, json};
 
     use super::*;
-    use crate::ocr::{
+    use crate::{
         document::prepare_document,
         types::LiteLLMOcrRequest,
         wire::{OcrWireRequest, decode_request},

litellm-rust/crates/inference-ocr/src/route.rs

@@ -9,7 +9,7 @@ use litellm_host::{
 use litellm_llms::base_llm::ocr::error::Error;
 use litellm_llms_types::formats::ocr::LiteLLMOcrResponse;
 
-use crate::ocr::types::{LiteLLMOcrRequest, OcrDocumentInput};
+use crate::types::{LiteLLMOcrRequest, OcrDocumentInput};
 
 pub enum OcrOp {
     AcquireAzureAdToken(Reply<ResolvedCredential>),
@@ -49,7 +49,7 @@ fn caller_token_provider(services: HostServices<Ocr>) -> TokenProviderHandle {
     })
 }
 
-impl crate::ocr::OcrRoute {
+impl crate::OcrRoute {
     pub fn machine(self, request: OcrCall, observers: Option<ObservationSender>) -> OcrMachine {
         hosted_call(
             request,

litellm-rust/crates/inference-ocr/src/wire.rs

@@ -6,13 +6,13 @@ use litellm_llms_types::formats::ocr::OcrDocument;
 use serde::Deserialize;
 use serde_json::{Map, Value};
 
-use crate::ocr::types::{LiteLLMOcrRequest, OcrConnectionInputs, OcrDocumentInput};
+use crate::types::{LiteLLMOcrRequest, OcrConnectionInputs, OcrDocumentInput};
 
 pub fn consumed_optional_params(
     model: &str,
     provider: Option<&str>,
 ) -> Result<Vec<litellm_core_utils::call_arguments::ArgumentSpec>, Error> {
-    let specs = crate::ocr::arguments::consumed_optional_params(model, provider)?;
+    let specs = crate::arguments::consumed_optional_params(model, provider)?;
     Ok(consumed_optional_param_names(model, provider)?
         .into_iter()
         .map(|name| litellm_core_utils::call_arguments::ArgumentSpec {
@@ -26,7 +26,7 @@ pub fn consumed_optional_param_names(
     model: &str,
     provider: Option<&str>,
 ) -> Result<Vec<&'static str>, Error> {
-    let names = crate::ocr::arguments::consumed_optional_param_names(model, provider)?;
+    let names = crate::arguments::consumed_optional_param_names(model, provider)?;
     let (_, config) = super::provider_config::resolve_provider_config(model, provider)?;
     if config == super::provider_config::OcrConfigKind::VertexDeepSeek {
         return Ok(names
@@ -109,7 +109,7 @@ mod tests {
     use serde_json::json;
 
     use super::*;
-    use crate::ocr::arguments::is_supported_request;
+    use crate::arguments::is_supported_request;
 
     #[rstest]
     #[case::omitted(json!({"type":"document_url", "document_url":"https://example.com/a.pdf"}))]

litellm-rust/crates/inference-ocr/tests/ocr/documents.rs

@@ -1,7 +1,7 @@
 use base64::Engine;
 use litellm_host::interceptors::WireRequest;
-use litellm_inference::ocr::types::OcrDocumentInput;
 use litellm_inference::test_support::{http_config, no_secrets, resources};
+use litellm_inference_ocr::types::OcrDocumentInput;
 use rstest::rstest;
 use wiremock::{Mock, matchers::any};
 

litellm-rust/crates/inference-ocr/tests/ocr/lifecycle.rs

@@ -9,7 +9,7 @@ use litellm_host::{
     interceptors::{RequestContext, WireRequest},
     lifecycle::CallEvent,
 };
-use litellm_inference::ocr::{
+use litellm_inference_ocr::{
     route::{Ocr, OcrCall, OcrOp},
     types::OcrDocumentInput,
 };

litellm-rust/crates/inference-ocr/tests/ocr/machine.rs

@@ -12,7 +12,7 @@ use litellm_host::{
     protocol::{HostRequest, InterceptRequest},
 };
 use litellm_host_native::services::HostCallHandler;
-use litellm_inference::ocr::{
+use litellm_inference_ocr::{
     route::{OcrCall, OcrMachine, OcrOp},
     types::OcrDocumentInput,
 };

litellm-rust/crates/inference-ocr/tests/ocr/main.rs

@@ -2,14 +2,14 @@ use litellm_host::{
     interceptors::{RequestContext, WireRequest},
     lifecycle::CallEvent,
 };
-use litellm_inference::ocr::{
+use litellm_inference::test_support::{http_config, no_secrets, resources};
+use litellm_inference_ocr::{
     OcrRoute,
     document::prepare_document,
     route::{Ocr, OcrCall, OcrOp},
     types::{LiteLLMOcrRequest, OcrDocumentInput},
     wire::{OcrWireRequest, decode_request},
 };
-use litellm_inference::test_support::{http_config, no_secrets, resources};
 use litellm_llms::base_llm::ocr::{error::Error, settings::OcrSettings};
 use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrDocument};
 use serde_json::{Map, Value, json};

litellm-rust/crates/inference-ocr/tests/ocr/vertex_ai.rs

@@ -1,5 +1,5 @@
 use litellm_auth::{InputSource, Sourced};
-use litellm_inference::ocr::arguments::is_supported_request;
+use litellm_inference_ocr::arguments::is_supported_request;
 use litellm_llms::base_llm::ocr::settings::OcrSettings;
 use rstest::rstest;
 

litellm-rust/crates/inference-ocr/tests/resources.rs

@@ -10,11 +10,9 @@ use litellm_auth_gcp::{
     CredentialSource, VertexAuth, VertexAuthFuture, VertexProviderLoader, VertexTokenSource,
 };
 use litellm_http::{HttpSettings, Resolution};
+use litellm_inference::resources::CoreResources;
 use litellm_inference::test_support::{RecordingSecrets, http_pool};
-use litellm_inference::{
-    ocr::wire::{OcrWireRequest, decode_request},
-    resources::CoreResources,
-};
+use litellm_inference_ocr::wire::{OcrWireRequest, decode_request};
 use litellm_llms::base_llm::ocr::settings::OcrSettings;
 use rstest::{fixture, rstest};
 use serde_json::json;

litellm-rust/crates/inference-ocr/tests/support/mod.rs

@@ -22,8 +22,8 @@ pub fn build_ocr_route(
     url_policy: litellm_http::media::UrlPolicy,
     settings: litellm_llms::base_llm::ocr::settings::OcrSettings,
     secrets: Arc<dyn SecretSource>,
-) -> litellm_inference::ocr::OcrRoute {
-    litellm_inference::ocr::OcrRoute::new(
+) -> litellm_inference_ocr::OcrRoute {
+    litellm_inference_ocr::OcrRoute::new(
         litellm_llms::base_llm::ocr::handler::OcrClient::new(
             &resources.pool,
             config,

litellm-rust/crates/inference/src/lib.rs

@@ -4,7 +4,6 @@ pub mod diagnostic;
 pub mod caching;
 pub mod constants;
 pub mod error;
-pub mod ocr;
 pub mod outbound;
 pub mod provider;
 pub mod resources;

litellm-rust/crates/llms-types/AGENTS.md

@@ -1,4 +1,4 @@
-The same ownership rule applies to Messages, Responses, Chat Completions, OCR, and other API formats. This crate owns their shared API data contracts. Adapter contracts and shared transformation machinery belong in `llms/src/base_llm/<format>/`, provider policy in `llms/src/<provider>/<format>/`, and call orchestration in `core/src/<format>/`. A provider originating a format, or several providers using a type, does not change these responsibilities. Existing model locations outside this crate are not exceptions to this rule for new shared API contracts
+The same ownership rule applies to Messages, Responses, Chat Completions, OCR, and other API formats. This crate owns their shared API data contracts. Adapter contracts and shared transformation machinery belong in `llms/src/base_llm/<format>/`, provider policy in `llms/src/<provider>/<format>/`, and call orchestration in `inference-<format>`. A provider originating a format, or several providers using a type, does not change these responsibilities. Existing model locations outside this crate are not exceptions to this rule for new shared API contracts
 
 - `litellm-llms-types` owns shared API data contracts and their serialization
   - A type belongs here when it describes a request, response, event, or value that consumers must agree on independently of how a call executes

litellm-rust/crates/llms/AGENTS.md

@@ -1,4 +1,4 @@
-litellm-llms mirrors `litellm/llms/`: base config traits, provider transformations, and the OCR request handler in `base_llm/ocr/handler.rs`. Transport code (clients, media fetching, header helpers, transport errors) lives in `litellm-http`. See `../core/AGENTS.md` for how the crates layer.
+litellm-llms mirrors `litellm/llms/`: base config traits, provider transformations, and the OCR request handler in `base_llm/ocr/handler.rs`. Transport code (clients, media fetching, header helpers, transport errors) lives in `litellm-http`. See `../inference/AGENTS.md` for how the crates layer.
 
 ## Python/Rust transformation pairs
 
@@ -22,7 +22,7 @@ Azure Messages maps to `llms/azure_ai/anthropic/messages_transformation.py`; Bed
 
 ## Provider and format boundaries
 
-The same ownership rule applies to Messages, Responses, Chat Completions, OCR, and other API formats. `litellm-llms-types` owns shared API data contracts. `llms/src/base_llm/<format>/` owns provider adapter contracts and shared transformation machinery. `llms/src/<provider>/<format>/` owns provider implementations and policy. `core/src/<format>/` owns call orchestration. Repeating a format name identifies the API each layer handles, not duplicate ownership of its schema. These boundaries also apply between modules in the same crate
+The same ownership rule applies to Messages, Responses, Chat Completions, OCR, and other API formats. `litellm-llms-types` owns shared API data contracts. `llms/src/base_llm/<format>/` owns provider adapter contracts and shared transformation machinery. `llms/src/<provider>/<format>/` owns provider implementations and policy. `inference-<format>` owns call orchestration. Repeating a format name identifies the API each layer handles, not duplicate ownership of its schema. These boundaries also apply between modules in the same crate
 
 A provider adapter may explicitly reuse another provider's transformation helper when that policy applies to its backend, such as Bedrock's Claude adapter using Anthropic payload shaping. Reuse across hosts of the same model family does not make the policy format-wide. Keep provider policy out of shared trait defaults and generic normalization, and keep shared execution contexts limited to inputs the adapter contract actually needs. Pure payload rewrites belong with transformations, not transport handlers
 

litellm-rust/crates/python-bridge/Cargo.toml

@@ -41,6 +41,7 @@ litellm-inference-transcription.workspace = true
 litellm-inference-responses.workspace = true
 litellm-inference-messages.workspace = true
 litellm-inference-chat.workspace = true
+litellm-inference-ocr.workspace = true
 litellm-core-utils.workspace = true
 litellm-http.workspace = true
 litellm-llms.workspace = true

litellm-rust/crates/python-bridge/src/routes/ocr/document.rs

@@ -1,7 +1,7 @@
 use std::path::PathBuf;
 
 use litellm_host_python::{PythonFileReader, py_bytes};
-use litellm_inference::ocr::types::OcrDocumentInput;
+use litellm_inference_ocr::types::OcrDocumentInput;
 use pyo3::{
     exceptions::PyValueError,
     prelude::*,

litellm-rust/crates/python-bridge/src/routes/ocr/host.rs

@@ -1,7 +1,7 @@
 use litellm_auth::ResolvedCredential;
 use litellm_host_python::{InvokeError, PythonBinding, missing_state, to_py};
 use litellm_host_python::{PythonHostCalls, PythonOwned};
-use litellm_inference::ocr::route::{Ocr, OcrCall, OcrOp};
+use litellm_inference_ocr::route::{Ocr, OcrCall, OcrOp};
 use litellm_llms::base_llm::ocr::error::Error;
 use litellm_llms_types::formats::ocr::LiteLLMOcrResponse;
 use pyo3::{

litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs

@@ -7,7 +7,7 @@ use host::OcrPythonHost;
 use litellm_callbacks_legacy_python::LoggingOperation;
 use litellm_core_utils::settings::ProcessEnvironment;
 use litellm_host_python::to_py;
-use litellm_inference::ocr::provider_config;
+use litellm_inference_ocr::provider_config;
 use litellm_llms::base_llm::ocr::settings::OcrSettings;
 use pyo3::{
     prelude::*,
@@ -58,7 +58,7 @@ fn run_ocr(
                 crate::secrets::source(py)?,
             )
             .map_err(http::client_error)?;
-            let route = litellm_inference::ocr::OcrRoute::new(client);
+            let route = litellm_inference_ocr::OcrRoute::new(client);
             Ok(route.machine(request, None))
         },
         OcrPythonHost::new(request.unbind()),

litellm-rust/crates/python-bridge/src/routes/ocr/project.rs

@@ -1,6 +1,6 @@
 use litellm_auth::SecretValue;
 use litellm_host_python::from_py;
-use litellm_inference::ocr::{
+use litellm_inference_ocr::{
     types::{LiteLLMOcrRequest, OcrDocumentInput},
     wire::{OcrWireRequest, consumed_optional_params, decode_document, decode_request_input},
 };