refactor(rust): hand the OCR route host its Python helpers instead of importing the package

The OCR route host looked up litellm.rust_bridge.ocr.route_host and
litellm.rust_bridge.timeouts by module path at call time, a back-edge
from the extension into package internals that unit tests had to
stub through sys.modules. Python now passes an OcrHostHelpers value
(response, map_failure, timeout_to_seconds) as the hook's fourth
argument, the host keeps that handle for the call and traverses it
for GC, and the bridge imports nothing from the package.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Yujong Lee 2026-09-17 20:18:50 -07:00
parent 86e98ce3a0
commit dd7b1ba757
10 changed files with 105 additions and 50 deletions

View file

@ -37,13 +37,6 @@ fn optional_object(
required_object(name, from_py_argument(value)?).map(Some)
}
pub(crate) fn python_timeout_seconds(py: Python<'_>, timeout: Py<PyAny>) -> PyResult<Option<f64>> {
py.import("litellm.rust_bridge.timeouts")?
.getattr("timeout_to_seconds")?
.call1((timeout,))?
.extract()
}
pub(crate) fn project_optional_fields(
kwargs: &Bound<'_, PyDict>,
names: &[&str],

View file

@ -16,16 +16,19 @@ enum OcrHostData {
}
/// The Python side of the OCR route: projects the prepared arguments, reads file-like
/// documents, acquires Azure AD tokens, and builds the public response and exception.
/// documents, acquires Azure AD tokens, and builds the public response and exception
/// through the helpers the caller handed in.
pub(super) struct OcrRouteHost {
request: Py<PyAny>,
helpers: Py<PyAny>,
data: OcrHostData,
}
impl OcrRouteHost {
pub(super) fn new(request: Py<PyAny>) -> Self {
pub(super) fn new(request: Py<PyAny>, helpers: Py<PyAny>) -> Self {
Self {
request,
helpers,
data: OcrHostData::Unprojected,
}
}
@ -68,7 +71,8 @@ impl RouteHost for OcrRouteHost {
let OcrHostData::Unprojected = self.data else {
return Err(missing_state());
};
let (request, handles) = project_request(self.request.bind(py), arguments)?;
let (request, handles) =
project_request(self.request.bind(py), arguments, self.helpers.bind(py))?;
let caller_token = handles.azure_ad_token_provider.is_some();
self.data = OcrHostData::Projected(Box::new(handles));
Ok(OcrOpResult::Request {
@ -84,7 +88,8 @@ impl RouteHost for OcrRouteHost {
}
fn complete(&mut self, py: Python<'_>, response: LiteLLMOcrResponse) -> PyResult<Py<PyAny>> {
py.import("litellm.rust_bridge.ocr.route_host")?
self.helpers
.bind(py)
.getattr("response")?
.call1((to_py(py, &response)?,))
.map(Bound::unbind)
@ -103,8 +108,9 @@ impl RouteHost for OcrRouteHost {
OcrHostData::Projected(handles) => handles.provider,
_ => "",
};
let mapped: Py<PyBaseException> = py
.import("litellm.rust_bridge.ocr.route_host")?
let mapped: Py<PyBaseException> = self
.helpers
.bind(py)
.getattr("map_failure")?
.call1((error.value(py), self.request.bind(py), provider))?
.extract()?;
@ -117,6 +123,7 @@ impl RouteHost for OcrRouteHost {
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
visit.call(&self.request)?;
visit.call(&self.helpers)?;
if let OcrHostData::Projected(handles) = &self.data {
if let Some(reader) = &handles.reader {
reader.traverse(visit)?;

View file

@ -25,6 +25,7 @@ fn run_ocr(
request: Bound<'_, PyAny>,
args: Bound<'_, PyTuple>,
kwargs: Bound<'_, PyDict>,
helpers: Bound<'_, PyAny>,
asynchronous: bool,
) -> PyResult<Py<PyAny>> {
let client = OcrClient::shared().map_err(ocr_error_to_pyerr)?;
@ -33,7 +34,7 @@ fn run_ocr(
if asynchronous { ASYNC_SURFACE } else { SURFACE },
PublicCall::capture(&request, &args, &kwargs)?,
ocr_machine(client),
OcrRouteHost::new(request.unbind()),
OcrRouteHost::new(request.unbind(), helpers.unbind()),
asynchronous,
)
}
@ -44,8 +45,9 @@ pub(crate) fn ocr(
request: Bound<'_, PyAny>,
args: Bound<'_, PyTuple>,
kwargs: Bound<'_, PyDict>,
helpers: Bound<'_, PyAny>,
) -> PyResult<Py<PyAny>> {
run_ocr(py, request, args, kwargs, false)
run_ocr(py, request, args, kwargs, helpers, false)
}
#[pyfunction]
@ -54,6 +56,7 @@ pub(crate) fn aocr(
request: Bound<'_, PyAny>,
args: Bound<'_, PyTuple>,
kwargs: Bound<'_, PyDict>,
helpers: Bound<'_, PyAny>,
) -> PyResult<Py<PyAny>> {
run_ocr(py, request, args, kwargs, true)
run_ocr(py, request, args, kwargs, helpers, true)
}

View file

@ -11,7 +11,7 @@ use serde_json::{Map, Value};
use super::document::{FileDocumentInput, PythonFileReader};
use crate::credentials::{self, CallerTokenProvider};
use crate::errors::ocr_error_to_pyerr;
use crate::marshal::{project_optional_fields, python_timeout_seconds, request_input_sources};
use crate::marshal::{project_optional_fields, request_input_sources};
/// What the host keeps after projection: the caller's callables that answer the document
/// read and token operations, and the provider name the failure mapping reports.
@ -24,6 +24,7 @@ pub(super) struct OcrHostHandles {
struct OcrArguments<'a, 'py> {
request: &'a Bound<'py, PyAny>,
kwargs: &'a Bound<'py, PyDict>,
helpers: &'a Bound<'py, PyAny>,
}
impl<'py> OcrArguments<'_, 'py> {
@ -63,7 +64,12 @@ impl<'py> OcrArguments<'_, 'py> {
Ok(self
.lookup("timeout")?
.extract::<Option<Py<PyAny>>>()?
.map(|value| python_timeout_seconds(self.request.py(), value))
.map(|value| {
self.helpers
.getattr("timeout_to_seconds")?
.call1((value,))?
.extract::<Option<f64>>()
})
.transpose()?
.flatten())
}
@ -111,8 +117,13 @@ impl ProjectedDocument {
pub(super) fn project_request(
request: &Bound<'_, PyAny>,
kwargs: &Bound<'_, PyDict>,
helpers: &Bound<'_, PyAny>,
) -> PyResult<(LiteLLMOcrRequest<OcrDocumentInput>, OcrHostHandles)> {
let arguments = OcrArguments { request, kwargs };
let arguments = OcrArguments {
request,
kwargs,
helpers,
};
let model = arguments.model()?;
let custom_llm_provider = arguments.custom_llm_provider()?;
let document = ProjectedDocument::project(&arguments.document()?)?;
@ -168,8 +179,22 @@ mod tests {
fn arguments<'a, 'py>(
request: &'a Bound<'py, PyAny>,
kwargs: &'a Bound<'py, PyDict>,
helpers: &'a Bound<'py, PyAny>,
) -> OcrArguments<'a, 'py> {
OcrArguments { request, kwargs }
OcrArguments {
request,
kwargs,
helpers,
}
}
fn helpers(py: Python<'_>) -> Bound<'_, PyAny> {
py.eval(
c"__import__('types').SimpleNamespace(timeout_to_seconds=lambda timeout: None if timeout is None else float(timeout))",
None,
None,
)
.unwrap()
}
fn project_document(
@ -186,21 +211,6 @@ mod tests {
.into()
}
fn stub_timeout_conversion(py: Python<'_>) {
eval(
py,
c"
import sys
import types
timeouts = types.ModuleType('litellm.rust_bridge.timeouts')
timeouts.timeout_to_seconds = lambda timeout: None if timeout is None else float(timeout)
sys.modules.setdefault('litellm', types.ModuleType('litellm'))
sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bridge'))
sys.modules['litellm.rust_bridge.timeouts'] = timeouts
",
);
}
#[test]
fn kwargs_override_request_attributes_including_explicit_none() {
Python::initialize();
@ -228,7 +238,8 @@ kwargs = {'model': 'from-kwargs', 'custom_llm_provider': None}
.unwrap()
.cast_into::<PyDict>()
.unwrap();
let arguments = arguments(&request, &kwargs);
let helpers = helpers(py);
let arguments = arguments(&request, &kwargs, &helpers);
assert_eq!(arguments.model().unwrap(), "from-kwargs");
assert_eq!(arguments.custom_llm_provider().unwrap(), None);
let accesses: Vec<String> = request.getattr("accesses").unwrap().extract().unwrap();
@ -262,7 +273,7 @@ kwargs = {}
.cast_into::<PyDict>()
.unwrap();
assert_eq!(
arguments(&request, &kwargs).model().unwrap(),
arguments(&request, &kwargs, &helpers(py)).model().unwrap(),
"mistral-ocr-latest"
);
assert_eq!(
@ -295,7 +306,9 @@ kwargs = {}
.unwrap()
.cast_into::<PyDict>()
.unwrap();
let error = arguments(&request, &kwargs).model().unwrap_err();
let error = arguments(&request, &kwargs, &helpers(py))
.model()
.unwrap_err();
assert!(
error
.value(py)
@ -328,7 +341,8 @@ kwargs = {}
.unwrap()
.cast_into::<PyDict>()
.unwrap();
let arguments = arguments(&request, &kwargs);
let helpers = helpers(py);
let arguments = arguments(&request, &kwargs, &helpers);
assert_eq!(arguments.model().unwrap(), "mistral-ocr-latest");
assert_eq!(arguments.custom_llm_provider().unwrap(), None);
});
@ -338,7 +352,6 @@ kwargs = {}
fn document_readers_are_not_consumed_during_projection() {
Python::initialize();
Python::attach(|py| {
stub_timeout_conversion(py);
let locals = eval(
py,
c"
@ -365,7 +378,8 @@ kwargs = {}
.unwrap()
.cast_into::<PyDict>()
.unwrap();
let arguments = arguments(&request, &kwargs);
let helpers = helpers(py);
let arguments = arguments(&request, &kwargs, &helpers);
let document = arguments.document().unwrap();
let (input, reader) = project_document(&document).unwrap();
assert_eq!(input, OcrDocumentInput::HostReader { mime_type: None });

View file

@ -7,8 +7,9 @@ from litellm.llms.base_llm.ocr.transformation import OCRResponse
from litellm.ocr import main
from litellm.ocr.main import convert_file_document_to_url_document, get_mime_type
from litellm.rust_bridge.catalog import Context, Route
from litellm.rust_bridge.dispatch import PublicDispatch, call_hook
from litellm.rust_bridge.ocr.entrypoints import NATIVE_AOCR, NATIVE_OCR, LiteLLMOcrRequest
from litellm.rust_bridge.dispatch import PublicDispatch
from litellm.rust_bridge.ocr.entrypoints import NATIVE_AOCR, NATIVE_OCR, LiteLLMOcrRequest, NativeAocr, NativeOcr
from litellm.rust_bridge.ocr.route_host import HELPERS
__all__ = ("aocr", "convert_file_document_to_url_document", "get_mime_type", "ocr")
@ -52,6 +53,18 @@ _PYTHON_AOCR: Final = cast( # cast-ok: forward the original call shape through
)
def _call_native(
hook: NativeOcr, request: LiteLLMOcrRequest, args: tuple[object, ...], kwargs: Mapping[str, object]
) -> OCRResponse:
return hook(request, args, kwargs, HELPERS)
def _call_native_async(
hook: NativeAocr, request: LiteLLMOcrRequest, args: tuple[object, ...], kwargs: Mapping[str, object]
) -> Awaitable[OCRResponse]:
return hook(request, args, kwargs, HELPERS)
def _context(request: LiteLLMOcrRequest) -> Context:
return Context(Route.OCR, provider=request.custom_llm_provider, model=request.model)
@ -79,7 +92,7 @@ def ocr(
kwargs,
python=_PYTHON_OCR,
binding=NATIVE_OCR,
native=call_hook,
native=_call_native,
)
@ -89,5 +102,5 @@ async def aocr(*args: object, **kwargs: object) -> OCRResponse: # kwargs-ok: pr
kwargs,
python=_PYTHON_AOCR,
binding=NATIVE_AOCR,
native=call_hook,
native=_call_native_async,
)

View file

@ -1,9 +1,9 @@
from asyncio import Future
from collections.abc import Coroutine, Mapping, Sequence
from collections.abc import Coroutine, Mapping
from typing import Never, final
from litellm.llms.base_llm.ocr.transformation import OCRResponse
from litellm.rust_bridge.ocr.entrypoints import LiteLLMOcrRequest
from litellm.rust_bridge.ocr.entrypoints import LiteLLMOcrRequest, OcrHostHelpers
class RustBridgeDeclined(Exception):
"""Rejected before any provider I/O. Args are (message, rejection)."""
@ -15,11 +15,13 @@ def ocr(
request: LiteLLMOcrRequest,
args: tuple[object, ...],
kwargs: dict[str, object],
helpers: OcrHostHelpers,
) -> OCRResponse: ...
def aocr(
request: LiteLLMOcrRequest,
args: tuple[object, ...],
kwargs: dict[str, object],
helpers: OcrHostHelpers,
) -> Coroutine[object, object, OCRResponse]: ...
def transcription(
model: str,

View file

@ -1,6 +1,6 @@
from __future__ import annotations
from collections.abc import Awaitable, Mapping
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass
from typing import Final, Protocol, cast # noqa: TID251 # validates dynamically loaded native callables
@ -23,12 +23,23 @@ class LiteLLMOcrRequest:
input_sources: Mapping[str, str] | None = None
@dataclass(frozen=True, slots=True)
class OcrHostHelpers:
"""The Python callables the native OCR route answers with: the public response and
exception constructors, and the timeout conversion for the caller's raw keyword."""
response: Callable[[Mapping[str, object]], OCRResponse]
map_failure: Callable[[Exception, LiteLLMOcrRequest, str], Exception]
timeout_to_seconds: Callable[[float | httpx.Timeout | None], float | None]
class NativeOcr(Protocol):
def __call__(
self,
request: LiteLLMOcrRequest,
args: tuple[object, ...],
kwargs: Mapping[str, object],
helpers: OcrHostHelpers,
) -> OCRResponse: ...
@ -38,6 +49,7 @@ class NativeAocr(Protocol):
request: LiteLLMOcrRequest,
args: tuple[object, ...],
kwargs: Mapping[str, object],
helpers: OcrHostHelpers,
) -> Awaitable[OCRResponse]: ...

View file

@ -11,7 +11,8 @@ from pydantic import TypeAdapter
import litellm
from litellm.llms.base_llm.ocr.transformation import PROVIDER_NATIVE_RESPONSE_KEY, OCRResponse
from litellm.rust_bridge import failures
from litellm.rust_bridge.ocr.entrypoints import LiteLLMOcrRequest
from litellm.rust_bridge.ocr.entrypoints import LiteLLMOcrRequest, OcrHostHelpers
from litellm.rust_bridge.timeouts import timeout_to_seconds
_RESPONSE_ADAPTER: Final = TypeAdapter(dict[str, object])
@ -90,3 +91,6 @@ def map_failure(error: Exception, request: LiteLLMOcrRequest, request_provider:
public_error.response = original.response
public_error.status_code = original.status_code
return public_error
HELPERS: Final = OcrHostHelpers(response=response, map_failure=map_failure, timeout_to_seconds=timeout_to_seconds)

View file

@ -20,7 +20,9 @@ from litellm.rust_bridge.ocr.entrypoints import (
LiteLLMOcrRequest,
NativeAocr,
NativeOcr,
OcrHostHelpers,
)
from litellm.rust_bridge.ocr.route_host import HELPERS
PYTHON_RULES: Final[Rules] = (Rule(Route.OCR, Rollout.PYTHON_ONLY),)
RUST_RULES: Final[Rules] = (Rule(Route.OCR, Rollout.RUST_REQUIRED),)
@ -346,7 +348,9 @@ def test_public_ocr_routes_through_dispatch(monkeypatch: pytest.MonkeyPatch) ->
request: LiteLLMOcrRequest,
args: tuple[object, ...],
kwargs: Mapping[str, object],
helpers: OcrHostHelpers,
) -> OCRResponse:
assert helpers is HELPERS
captured.append(request)
return expected
@ -374,7 +378,9 @@ async def test_public_aocr_routes_through_dispatch(monkeypatch: pytest.MonkeyPat
request: LiteLLMOcrRequest,
args: tuple[object, ...],
kwargs: Mapping[str, object],
helpers: OcrHostHelpers,
) -> OCRResponse:
assert helpers is HELPERS
captured.append(request)
return expected

View file

@ -16,6 +16,7 @@ from litellm._logging import trace_id_var
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.rust_bridge.ocr.route_host import HELPERS
from tests.test_litellm_rust.support.callback_recorder import RecordingLogger, drain_logging
from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec
from tests.test_litellm_rust.support.requests import OCR_RESPONSE, call_aocr, call_ocr
@ -594,7 +595,7 @@ def test_unstarted_native_coroutine_releases_input_without_reading_file(ocr_serv
def create():
file: Final = File()
kwargs: Final = {"model": "mistral/mistral-ocr-latest", "document": {"type": "file", "file": file}}
coroutine: Final = _native.aocr(_public_request("aocr", (), kwargs), (), kwargs)
coroutine: Final = _native.aocr(_public_request("aocr", (), kwargs), (), kwargs, HELPERS)
file.owner = coroutine
coroutine.close()
return weakref.ref(file)