diff --git a/litellm-rust/crates/python-bridge/src/marshal.rs b/litellm-rust/crates/python-bridge/src/marshal.rs index 227fe56e721..e337d8b6844 100644 --- a/litellm-rust/crates/python-bridge/src/marshal.rs +++ b/litellm-rust/crates/python-bridge/src/marshal.rs @@ -37,13 +37,6 @@ fn optional_object( required_object(name, from_py_argument(value)?).map(Some) } -pub(crate) fn python_timeout_seconds(py: Python<'_>, timeout: Py) -> PyResult> { - py.import("litellm.rust_bridge.timeouts")? - .getattr("timeout_to_seconds")? - .call1((timeout,))? - .extract() -} - pub(crate) fn project_optional_fields( kwargs: &Bound<'_, PyDict>, names: &[&str], diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs index 37b90fe7756..6865e39caa6 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs @@ -16,16 +16,19 @@ enum OcrHostData { } /// The Python side of the OCR route: projects the prepared arguments, reads file-like -/// documents, acquires Azure AD tokens, and builds the public response and exception. +/// documents, acquires Azure AD tokens, and builds the public response and exception +/// through the helpers the caller handed in. pub(super) struct OcrRouteHost { request: Py, + helpers: Py, data: OcrHostData, } impl OcrRouteHost { - pub(super) fn new(request: Py) -> Self { + pub(super) fn new(request: Py, helpers: Py) -> Self { Self { request, + helpers, data: OcrHostData::Unprojected, } } @@ -68,7 +71,8 @@ impl RouteHost for OcrRouteHost { let OcrHostData::Unprojected = self.data else { return Err(missing_state()); }; - let (request, handles) = project_request(self.request.bind(py), arguments)?; + let (request, handles) = + project_request(self.request.bind(py), arguments, self.helpers.bind(py))?; let caller_token = handles.azure_ad_token_provider.is_some(); self.data = OcrHostData::Projected(Box::new(handles)); Ok(OcrOpResult::Request { @@ -84,7 +88,8 @@ impl RouteHost for OcrRouteHost { } fn complete(&mut self, py: Python<'_>, response: LiteLLMOcrResponse) -> PyResult> { - py.import("litellm.rust_bridge.ocr.route_host")? + self.helpers + .bind(py) .getattr("response")? .call1((to_py(py, &response)?,)) .map(Bound::unbind) @@ -103,8 +108,9 @@ impl RouteHost for OcrRouteHost { OcrHostData::Projected(handles) => handles.provider, _ => "", }; - let mapped: Py = py - .import("litellm.rust_bridge.ocr.route_host")? + let mapped: Py = self + .helpers + .bind(py) .getattr("map_failure")? .call1((error.value(py), self.request.bind(py), provider))? .extract()?; @@ -117,6 +123,7 @@ impl RouteHost for OcrRouteHost { fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { visit.call(&self.request)?; + visit.call(&self.helpers)?; if let OcrHostData::Projected(handles) = &self.data { if let Some(reader) = &handles.reader { reader.traverse(visit)?; diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs index 7640f6e5448..56501fab8d8 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs @@ -25,6 +25,7 @@ fn run_ocr( request: Bound<'_, PyAny>, args: Bound<'_, PyTuple>, kwargs: Bound<'_, PyDict>, + helpers: Bound<'_, PyAny>, asynchronous: bool, ) -> PyResult> { let client = OcrClient::shared().map_err(ocr_error_to_pyerr)?; @@ -33,7 +34,7 @@ fn run_ocr( if asynchronous { ASYNC_SURFACE } else { SURFACE }, PublicCall::capture(&request, &args, &kwargs)?, ocr_machine(client), - OcrRouteHost::new(request.unbind()), + OcrRouteHost::new(request.unbind(), helpers.unbind()), asynchronous, ) } @@ -44,8 +45,9 @@ pub(crate) fn ocr( request: Bound<'_, PyAny>, args: Bound<'_, PyTuple>, kwargs: Bound<'_, PyDict>, + helpers: Bound<'_, PyAny>, ) -> PyResult> { - run_ocr(py, request, args, kwargs, false) + run_ocr(py, request, args, kwargs, helpers, false) } #[pyfunction] @@ -54,6 +56,7 @@ pub(crate) fn aocr( request: Bound<'_, PyAny>, args: Bound<'_, PyTuple>, kwargs: Bound<'_, PyDict>, + helpers: Bound<'_, PyAny>, ) -> PyResult> { - run_ocr(py, request, args, kwargs, true) + run_ocr(py, request, args, kwargs, helpers, true) } diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs index 84656ad90e5..7f88574b458 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs @@ -11,7 +11,7 @@ use serde_json::{Map, Value}; use super::document::{FileDocumentInput, PythonFileReader}; use crate::credentials::{self, CallerTokenProvider}; use crate::errors::ocr_error_to_pyerr; -use crate::marshal::{project_optional_fields, python_timeout_seconds, request_input_sources}; +use crate::marshal::{project_optional_fields, request_input_sources}; /// What the host keeps after projection: the caller's callables that answer the document /// read and token operations, and the provider name the failure mapping reports. @@ -24,6 +24,7 @@ pub(super) struct OcrHostHandles { struct OcrArguments<'a, 'py> { request: &'a Bound<'py, PyAny>, kwargs: &'a Bound<'py, PyDict>, + helpers: &'a Bound<'py, PyAny>, } impl<'py> OcrArguments<'_, 'py> { @@ -63,7 +64,12 @@ impl<'py> OcrArguments<'_, 'py> { Ok(self .lookup("timeout")? .extract::>>()? - .map(|value| python_timeout_seconds(self.request.py(), value)) + .map(|value| { + self.helpers + .getattr("timeout_to_seconds")? + .call1((value,))? + .extract::>() + }) .transpose()? .flatten()) } @@ -111,8 +117,13 @@ impl ProjectedDocument { pub(super) fn project_request( request: &Bound<'_, PyAny>, kwargs: &Bound<'_, PyDict>, + helpers: &Bound<'_, PyAny>, ) -> PyResult<(LiteLLMOcrRequest, OcrHostHandles)> { - let arguments = OcrArguments { request, kwargs }; + let arguments = OcrArguments { + request, + kwargs, + helpers, + }; let model = arguments.model()?; let custom_llm_provider = arguments.custom_llm_provider()?; let document = ProjectedDocument::project(&arguments.document()?)?; @@ -168,8 +179,22 @@ mod tests { fn arguments<'a, 'py>( request: &'a Bound<'py, PyAny>, kwargs: &'a Bound<'py, PyDict>, + helpers: &'a Bound<'py, PyAny>, ) -> OcrArguments<'a, 'py> { - OcrArguments { request, kwargs } + OcrArguments { + request, + kwargs, + helpers, + } + } + + fn helpers(py: Python<'_>) -> Bound<'_, PyAny> { + py.eval( + c"__import__('types').SimpleNamespace(timeout_to_seconds=lambda timeout: None if timeout is None else float(timeout))", + None, + None, + ) + .unwrap() } fn project_document( @@ -186,21 +211,6 @@ mod tests { .into() } - fn stub_timeout_conversion(py: Python<'_>) { - eval( - py, - c" -import sys -import types -timeouts = types.ModuleType('litellm.rust_bridge.timeouts') -timeouts.timeout_to_seconds = lambda timeout: None if timeout is None else float(timeout) -sys.modules.setdefault('litellm', types.ModuleType('litellm')) -sys.modules.setdefault('litellm.rust_bridge', types.ModuleType('litellm.rust_bridge')) -sys.modules['litellm.rust_bridge.timeouts'] = timeouts -", - ); - } - #[test] fn kwargs_override_request_attributes_including_explicit_none() { Python::initialize(); @@ -228,7 +238,8 @@ kwargs = {'model': 'from-kwargs', 'custom_llm_provider': None} .unwrap() .cast_into::() .unwrap(); - let arguments = arguments(&request, &kwargs); + let helpers = helpers(py); + let arguments = arguments(&request, &kwargs, &helpers); assert_eq!(arguments.model().unwrap(), "from-kwargs"); assert_eq!(arguments.custom_llm_provider().unwrap(), None); let accesses: Vec = request.getattr("accesses").unwrap().extract().unwrap(); @@ -262,7 +273,7 @@ kwargs = {} .cast_into::() .unwrap(); assert_eq!( - arguments(&request, &kwargs).model().unwrap(), + arguments(&request, &kwargs, &helpers(py)).model().unwrap(), "mistral-ocr-latest" ); assert_eq!( @@ -295,7 +306,9 @@ kwargs = {} .unwrap() .cast_into::() .unwrap(); - let error = arguments(&request, &kwargs).model().unwrap_err(); + let error = arguments(&request, &kwargs, &helpers(py)) + .model() + .unwrap_err(); assert!( error .value(py) @@ -328,7 +341,8 @@ kwargs = {} .unwrap() .cast_into::() .unwrap(); - let arguments = arguments(&request, &kwargs); + let helpers = helpers(py); + let arguments = arguments(&request, &kwargs, &helpers); assert_eq!(arguments.model().unwrap(), "mistral-ocr-latest"); assert_eq!(arguments.custom_llm_provider().unwrap(), None); }); @@ -338,7 +352,6 @@ kwargs = {} fn document_readers_are_not_consumed_during_projection() { Python::initialize(); Python::attach(|py| { - stub_timeout_conversion(py); let locals = eval( py, c" @@ -365,7 +378,8 @@ kwargs = {} .unwrap() .cast_into::() .unwrap(); - let arguments = arguments(&request, &kwargs); + let helpers = helpers(py); + let arguments = arguments(&request, &kwargs, &helpers); let document = arguments.document().unwrap(); let (input, reader) = project_document(&document).unwrap(); assert_eq!(input, OcrDocumentInput::HostReader { mime_type: None }); diff --git a/litellm/ocr/dispatch.py b/litellm/ocr/dispatch.py index 80c93273d1e..c9b3886354f 100644 --- a/litellm/ocr/dispatch.py +++ b/litellm/ocr/dispatch.py @@ -7,8 +7,9 @@ from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.ocr import main from litellm.ocr.main import convert_file_document_to_url_document, get_mime_type from litellm.rust_bridge.catalog import Context, Route -from litellm.rust_bridge.dispatch import PublicDispatch, call_hook -from litellm.rust_bridge.ocr.entrypoints import NATIVE_AOCR, NATIVE_OCR, LiteLLMOcrRequest +from litellm.rust_bridge.dispatch import PublicDispatch +from litellm.rust_bridge.ocr.entrypoints import NATIVE_AOCR, NATIVE_OCR, LiteLLMOcrRequest, NativeAocr, NativeOcr +from litellm.rust_bridge.ocr.route_host import HELPERS __all__ = ("aocr", "convert_file_document_to_url_document", "get_mime_type", "ocr") @@ -52,6 +53,18 @@ _PYTHON_AOCR: Final = cast( # cast-ok: forward the original call shape through ) +def _call_native( + hook: NativeOcr, request: LiteLLMOcrRequest, args: tuple[object, ...], kwargs: Mapping[str, object] +) -> OCRResponse: + return hook(request, args, kwargs, HELPERS) + + +def _call_native_async( + hook: NativeAocr, request: LiteLLMOcrRequest, args: tuple[object, ...], kwargs: Mapping[str, object] +) -> Awaitable[OCRResponse]: + return hook(request, args, kwargs, HELPERS) + + def _context(request: LiteLLMOcrRequest) -> Context: return Context(Route.OCR, provider=request.custom_llm_provider, model=request.model) @@ -79,7 +92,7 @@ def ocr( kwargs, python=_PYTHON_OCR, binding=NATIVE_OCR, - native=call_hook, + native=_call_native, ) @@ -89,5 +102,5 @@ async def aocr(*args: object, **kwargs: object) -> OCRResponse: # kwargs-ok: pr kwargs, python=_PYTHON_AOCR, binding=NATIVE_AOCR, - native=call_hook, + native=_call_native_async, ) diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index c8ece61aef7..b29a338bc92 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -1,9 +1,9 @@ from asyncio import Future -from collections.abc import Coroutine, Mapping, Sequence +from collections.abc import Coroutine, Mapping from typing import Never, final from litellm.llms.base_llm.ocr.transformation import OCRResponse -from litellm.rust_bridge.ocr.entrypoints import LiteLLMOcrRequest +from litellm.rust_bridge.ocr.entrypoints import LiteLLMOcrRequest, OcrHostHelpers class RustBridgeDeclined(Exception): """Rejected before any provider I/O. Args are (message, rejection).""" @@ -15,11 +15,13 @@ def ocr( request: LiteLLMOcrRequest, args: tuple[object, ...], kwargs: dict[str, object], + helpers: OcrHostHelpers, ) -> OCRResponse: ... def aocr( request: LiteLLMOcrRequest, args: tuple[object, ...], kwargs: dict[str, object], + helpers: OcrHostHelpers, ) -> Coroutine[object, object, OCRResponse]: ... def transcription( model: str, diff --git a/litellm/rust_bridge/ocr/entrypoints.py b/litellm/rust_bridge/ocr/entrypoints.py index 5b87634ec16..00f11c5276b 100644 --- a/litellm/rust_bridge/ocr/entrypoints.py +++ b/litellm/rust_bridge/ocr/entrypoints.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Awaitable, Mapping +from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass from typing import Final, Protocol, cast # noqa: TID251 # validates dynamically loaded native callables @@ -23,12 +23,23 @@ class LiteLLMOcrRequest: input_sources: Mapping[str, str] | None = None +@dataclass(frozen=True, slots=True) +class OcrHostHelpers: + """The Python callables the native OCR route answers with: the public response and + exception constructors, and the timeout conversion for the caller's raw keyword.""" + + response: Callable[[Mapping[str, object]], OCRResponse] + map_failure: Callable[[Exception, LiteLLMOcrRequest, str], Exception] + timeout_to_seconds: Callable[[float | httpx.Timeout | None], float | None] + + class NativeOcr(Protocol): def __call__( self, request: LiteLLMOcrRequest, args: tuple[object, ...], kwargs: Mapping[str, object], + helpers: OcrHostHelpers, ) -> OCRResponse: ... @@ -38,6 +49,7 @@ class NativeAocr(Protocol): request: LiteLLMOcrRequest, args: tuple[object, ...], kwargs: Mapping[str, object], + helpers: OcrHostHelpers, ) -> Awaitable[OCRResponse]: ... diff --git a/litellm/rust_bridge/ocr/route_host.py b/litellm/rust_bridge/ocr/route_host.py index 1ffb9254b1a..d70f800c6ac 100644 --- a/litellm/rust_bridge/ocr/route_host.py +++ b/litellm/rust_bridge/ocr/route_host.py @@ -11,7 +11,8 @@ from pydantic import TypeAdapter import litellm from litellm.llms.base_llm.ocr.transformation import PROVIDER_NATIVE_RESPONSE_KEY, OCRResponse from litellm.rust_bridge import failures -from litellm.rust_bridge.ocr.entrypoints import LiteLLMOcrRequest +from litellm.rust_bridge.ocr.entrypoints import LiteLLMOcrRequest, OcrHostHelpers +from litellm.rust_bridge.timeouts import timeout_to_seconds _RESPONSE_ADAPTER: Final = TypeAdapter(dict[str, object]) @@ -90,3 +91,6 @@ def map_failure(error: Exception, request: LiteLLMOcrRequest, request_provider: public_error.response = original.response public_error.status_code = original.status_code return public_error + + +HELPERS: Final = OcrHostHelpers(response=response, map_failure=map_failure, timeout_to_seconds=timeout_to_seconds) diff --git a/tests/test_litellm/ocr/test_dispatch.py b/tests/test_litellm/ocr/test_dispatch.py index 14d3368f869..cda8c335d20 100644 --- a/tests/test_litellm/ocr/test_dispatch.py +++ b/tests/test_litellm/ocr/test_dispatch.py @@ -20,7 +20,9 @@ from litellm.rust_bridge.ocr.entrypoints import ( LiteLLMOcrRequest, NativeAocr, NativeOcr, + OcrHostHelpers, ) +from litellm.rust_bridge.ocr.route_host import HELPERS PYTHON_RULES: Final[Rules] = (Rule(Route.OCR, Rollout.PYTHON_ONLY),) RUST_RULES: Final[Rules] = (Rule(Route.OCR, Rollout.RUST_REQUIRED),) @@ -346,7 +348,9 @@ def test_public_ocr_routes_through_dispatch(monkeypatch: pytest.MonkeyPatch) -> request: LiteLLMOcrRequest, args: tuple[object, ...], kwargs: Mapping[str, object], + helpers: OcrHostHelpers, ) -> OCRResponse: + assert helpers is HELPERS captured.append(request) return expected @@ -374,7 +378,9 @@ async def test_public_aocr_routes_through_dispatch(monkeypatch: pytest.MonkeyPat request: LiteLLMOcrRequest, args: tuple[object, ...], kwargs: Mapping[str, object], + helpers: OcrHostHelpers, ) -> OCRResponse: + assert helpers is HELPERS captured.append(request) return expected diff --git a/tests/test_litellm_rust/ocr/test_lifecycle.py b/tests/test_litellm_rust/ocr/test_lifecycle.py index aa9794a73a6..25f0b58cb7a 100644 --- a/tests/test_litellm_rust/ocr/test_lifecycle.py +++ b/tests/test_litellm_rust/ocr/test_lifecycle.py @@ -16,6 +16,7 @@ from litellm._logging import trace_id_var from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.rust_bridge.ocr.route_host import HELPERS from tests.test_litellm_rust.support.callback_recorder import RecordingLogger, drain_logging from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec from tests.test_litellm_rust.support.requests import OCR_RESPONSE, call_aocr, call_ocr @@ -594,7 +595,7 @@ def test_unstarted_native_coroutine_releases_input_without_reading_file(ocr_serv def create(): file: Final = File() kwargs: Final = {"model": "mistral/mistral-ocr-latest", "document": {"type": "file", "file": file}} - coroutine: Final = _native.aocr(_public_request("aocr", (), kwargs), (), kwargs) + coroutine: Final = _native.aocr(_public_request("aocr", (), kwargs), (), kwargs, HELPERS) file.owner = coroutine coroutine.close() return weakref.ref(file)