diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index f9e75f45f75..da075cb21f0 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -60,6 +60,9 @@ fn chat_completions_response_to_py( fn core_error_to_pyerr(err: CoreError) -> PyErr { match err { + CoreError::MissingField("document_url") => { + PyValueError::new_err("Document URL is required") + } CoreError::Auth(message) => PyValueError::new_err(message), CoreError::InvalidProvider(_) | CoreError::InvalidRequest(_) diff --git a/litellm/llms/vertex_ai/ocr/deepseek_transformation.py b/litellm/llms/vertex_ai/ocr/deepseek_transformation.py index 51d22283464..1db454ad1f6 100644 --- a/litellm/llms/vertex_ai/ocr/deepseek_transformation.py +++ b/litellm/llms/vertex_ai/ocr/deepseek_transformation.py @@ -50,9 +50,7 @@ class VertexAIDeepSeekOCRConfig(BaseOCRConfig): return { # mutable-ok: base contract **optional_params, **{ # mutable-ok: base contract - key: value - for key, value in non_default_params.items() - if key in self.get_supported_ocr_params(model) + key: value for key, value in non_default_params.items() if key in self.get_supported_ocr_params(model) }, } diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index b918f013700..f3ecb545f66 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -191,6 +191,10 @@ def _prepare_ocr_request( def _rust_ocr_supported(prepared_request: _PreparedOCRRequest) -> bool: if prepared_request.optional_params.get(OCR_REQUEST_FORMAT_PARAM) == "native": return False + if prepared_request.extra_headers is not None and any( + not isinstance(value, str) for value in prepared_request.extra_headers.values() + ): + return False return prepared_request.custom_llm_provider in _RUST_OCR_PROVIDERS diff --git a/tests/route_parity/README.md b/tests/route_parity/README.md index 59b81b35b46..812f71d3b95 100644 --- a/tests/route_parity/README.md +++ b/tests/route_parity/README.md @@ -1,4 +1,4 @@ -# Python/Python parity testing in Python SDK interface +# Python/Rust parity testing in the Python SDK interface > Given the same SDK call and identical provider behavior, does the PyO3 implementation behave same as Python? @@ -11,6 +11,8 @@ - The harness compares the values returned through the Python SDK interface - Non-streaming responses are compared directly, including their concrete return type and public model fields - Streaming responses are consumed and compared chunk by chunk, including wrapper type, chunk type and order, termination, and public exception behavior +- Failed SDK calls are compared by exception class, stable message, status, code, model, provider, and parameter fields +- Traceback paths and line numbers are excluded because they are runtime-specific - Route-specific comparators and chunk normalizers handle differences in each public SDK contract ## Process isolation @@ -48,6 +50,12 @@ a machine records the boundaries it has configured and skips the rest. Reducto f responses, but Reducto remains outside Python/Rust parity until the Rust OCR bridge supports it. Azure-hosted Mistral discovery also requires `AZURE_AI_OCR_MODEL`, the deployment name or full `azure_ai/...` model. +Invalid OCR inputs do not use recorded provider responses. The parity suite checks unsupported providers and models, +malformed documents, invalid request formats, invalid Azure Document Intelligence parameters, and invalid headers in +both sync and async SDK calls. These cases must return the same public exception fields without sending a provider +request. A malformed Azure Document Intelligence document reaches the Rust bridge so its native validation error is +also compared against Python + ## References - [Hypothesis documentation](https://hypothesis.readthedocs.io/en/latest/) diff --git a/tests/route_parity/compare.py b/tests/route_parity/compare.py index 609298496c3..0e469e6eb6d 100644 --- a/tests/route_parity/compare.py +++ b/tests/route_parity/compare.py @@ -25,9 +25,7 @@ def _request_after_transformation(request: CapturedRequest) -> CapturedRequest: return request.model_copy(update={"user_agent": None}) -def assert_request_parity( - python: tuple[CapturedRequest, ...], accelerated: tuple[CapturedRequest, ...] -) -> None: +def assert_request_parity(python: tuple[CapturedRequest, ...], accelerated: tuple[CapturedRequest, ...]) -> None: python_requests: Final = tuple(_request_after_transformation(request) for request in python) accelerated_requests: Final = tuple(_request_after_transformation(request) for request in accelerated) assert python_requests == accelerated_requests @@ -47,4 +45,4 @@ def assert_model_parity(python: BaseModel, accelerated: BaseModel) -> None: def assert_parity(python: Execution, accelerated: Execution, python_user_agent: str) -> None: validate_harness(python, accelerated, python_user_agent) assert_request_parity(python.requests, accelerated.requests) - assert python.report.response == accelerated.report.response + assert python.report == accelerated.report diff --git a/tests/route_parity/models.py b/tests/route_parity/models.py index 41f1da4188d..1b96cfbd355 100644 --- a/tests/route_parity/models.py +++ b/tests/route_parity/models.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Annotated, Literal +from typing import Annotated, Final, Literal, cast from pydantic import BaseModel, ConfigDict, Field, JsonValue @@ -15,12 +15,51 @@ class CapturedRequest(BaseModel): user_agent: str | None -class SDKReport(BaseModel): +class SDKSuccess(BaseModel): model_config = ConfigDict(frozen=True) + status: Literal["ok"] = "ok" response: JsonValue +class SDKError(BaseModel): + model_config = ConfigDict(frozen=True) + + status: Literal["error"] = "error" + exception_type: str + message: str + status_code: int | None + code: str | None + error_type: str | None + param: str | None + model: str | None + llm_provider: str | None + + +SDKReport = Annotated[SDKSuccess | SDKError, Field(discriminator="status")] + + +def _string_attribute(error: Exception, name: str) -> str | None: + value: Final = cast(object | None, getattr(error, name, None)) + return None if value is None else str(value) + + +def sdk_error_report(error: Exception) -> SDKError: + message, _, _ = str(error).partition("\nTraceback (most recent call last):") + raw_status_code: Final = cast(object | None, getattr(error, "status_code", None)) + status_code: Final = raw_status_code if isinstance(raw_status_code, int) else None + return SDKError( + exception_type=f"{type(error).__module__}.{type(error).__qualname__}", + message=message.rstrip(), + status_code=status_code, + code=_string_attribute(error, "code"), + error_type=_string_attribute(error, "type"), + param=_string_attribute(error, "param"), + model=_string_attribute(error, "model"), + llm_provider=_string_attribute(error, "llm_provider"), + ) + + class Execution(BaseModel): model_config = ConfigDict(frozen=True) diff --git a/tests/route_parity/test_parity.py b/tests/route_parity/test_parity.py index 78277469424..6aa362fe21f 100644 --- a/tests/route_parity/test_parity.py +++ b/tests/route_parity/test_parity.py @@ -6,7 +6,7 @@ import pytest from pydantic import BaseModel, JsonValue, PrivateAttr from tests.route_parity.compare import assert_model_parity, assert_parity -from tests.route_parity.models import CapturedRequest, Execution, SDKReport +from tests.route_parity.models import CapturedRequest, Execution, SDKError, SDKSuccess, sdk_error_report SENTINEL: Final = "python-parity-fallback" @@ -23,6 +23,10 @@ class _DifferentResponse(BaseModel): value: str +class _PublicError(ValueError): + status_code: Final = 400 + + def _execution(*, body: JsonValue = None, markdown: str = "same", user_agent: str | None = None) -> Execution: return Execution( requests=( @@ -34,7 +38,7 @@ def _execution(*, body: JsonValue = None, markdown: str = "same", user_agent: st user_agent=user_agent, ), ), - report=SDKReport(response={"items": [{"text": markdown}], "model": "test-model"}), + report=SDKSuccess(response={"items": [{"text": markdown}], "model": "test-model"}), ) @@ -54,6 +58,36 @@ def test_parity_rejects_response_difference() -> None: assert_parity(python, rust, SENTINEL) +def test_parity_rejects_error_difference() -> None: + python: Final = Execution( + requests=(), + report=SDKError( + exception_type="litellm.exceptions.BadRequestError", + message="bad request", + status_code=400, + code=None, + error_type=None, + param=None, + model="test-model", + llm_provider="mistral", + ), + ) + rust: Final = python.model_copy(update={"report": python.report.model_copy(update={"status_code": 500})}) + + with pytest.raises(AssertionError): + assert_parity(python, rust, SENTINEL) + + +def test_sdk_error_report_removes_traceback_but_keeps_public_fields() -> None: + error: Final = _PublicError("invalid input\nTraceback (most recent call last):\n unstable") + + report: Final = sdk_error_report(error) + + assert report.exception_type.endswith("._PublicError") + assert report.message == "invalid input" + assert report.status_code == 400 + + def test_parity_rejects_rust_fallback() -> None: python: Final = _execution(user_agent=SENTINEL) rust: Final = _execution(user_agent=SENTINEL) diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py index d2f02dc9786..1887bff1f97 100644 --- a/tests/test_litellm/ocr/test_rust_bridge.py +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -816,6 +816,27 @@ def test_ocr_falls_back_to_python_when_bridge_unavailable(monkeypatch): assert isinstance(response, OCRResponse) +def test_ocr_non_string_header_uses_python_path(monkeypatch): + bridge = RecordingBridge() + litellm.use_litellm_rust(True, ocr=bridge) + + def fake_handler_ocr(**kwargs): + assert kwargs["headers"] == {"x-invalid": 1} + return OCRResponse(pages=[], model="mistral-ocr-latest", object="ocr") + + monkeypatch.setattr(ocr_main.base_llm_http_handler, "ocr", fake_handler_ocr) + + response = litellm.ocr( + model=MODEL, + document=DOCUMENT, + api_key="sk-test", + extra_headers={"x-invalid": 1}, + ) + + assert isinstance(response, OCRResponse) + assert bridge.calls == [] + + def test_ocr_provider_configs_expose_api_key_env_vars(): from litellm.llms.azure_ai.ocr.document_intelligence.transformation import ( AzureDocumentIntelligenceOCRConfig, diff --git a/tests/test_litellm/ocr/test_sdk_parity.py b/tests/test_litellm/ocr/test_sdk_parity.py index 2f4d16460a8..2fb8ca1ad43 100644 --- a/tests/test_litellm/ocr/test_sdk_parity.py +++ b/tests/test_litellm/ocr/test_sdk_parity.py @@ -6,6 +6,7 @@ import sys import traceback from collections.abc import Awaitable, Callable, Coroutine, Generator from contextlib import contextmanager +from dataclasses import dataclass from enum import Enum from pathlib import Path from typing import Final, cast @@ -19,7 +20,16 @@ from litellm.rust_bridge.ocr import RustAocr, RustOcr from tests.route_parity.compare import assert_model_parity, assert_parity, assert_request_parity from tests.route_parity.fixture_recorder import recorded_fixtures from tests.route_parity.inprocess import run_in_process -from tests.route_parity.models import SDKCommand, SDKReport, WorkerFailure, WorkerResult, WorkerSuccess +from tests.route_parity.models import ( + SDKCommand, + SDKError, + SDKReport, + SDKSuccess, + WorkerFailure, + WorkerResult, + WorkerSuccess, + sdk_error_report, +) from tests.route_parity.replay import replay_server from tests.route_parity.runner import ( PythonScriptRunner, @@ -40,6 +50,115 @@ class SDKRoute(str, Enum): AOCR = "aocr" +@dataclass(frozen=True, slots=True) +class InvalidOcrCase: + name: str + model: str + document: object + expected_exception_type: str + expected_status_code: int + expected_message: str + extra_kwargs: tuple[tuple[str, object], ...] = () + expected_rust_calls: int = 0 + + +INVALID_OCR_CASES: Final = ( + InvalidOcrCase( + name="unsupported_provider", + model="openai/gpt-4o", + document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="OCR is not supported for provider: openai", + ), + InvalidOcrCase( + name="unsupported_reducto_model", + model="reducto/parse-v4", + document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="OCR is not supported for provider: reducto", + ), + InvalidOcrCase( + name="unknown_provider_prefix", + model="not_a_provider/model", + document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, + expected_exception_type="litellm.exceptions.BadRequestError", + expected_status_code=400, + expected_message="LLM Provider NOT provided", + ), + InvalidOcrCase( + name="non_object_document", + model="mistral/mistral-ocr-latest", + document=[], + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="document must be a dict", + ), + InvalidOcrCase( + name="missing_document_type", + model="mistral/mistral-ocr-latest", + document={}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="Invalid document type: None", + ), + InvalidOcrCase( + name="unsupported_document_type", + model="mistral/mistral-ocr-latest", + document={"type": "text", "text": "not a document"}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="Invalid document type: text", + ), + InvalidOcrCase( + name="missing_document_url", + model="azure_ai/doc-intelligence/prebuilt-read", + document={"type": "document_url"}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="Document URL is required", + expected_rust_calls=1, + ), + InvalidOcrCase( + name="invalid_request_format", + model="mistral/mistral-ocr-latest", + document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, + expected_exception_type="litellm.exceptions.UnsupportedParamsError", + expected_status_code=400, + expected_message="Invalid `req_format`: 'bogus'", + extra_kwargs=(("req_format", "bogus"),), + ), + InvalidOcrCase( + name="invalid_document_intelligence_pages", + model="azure_ai/doc-intelligence/prebuilt-read", + document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="`pages` integers must be >= 0", + extra_kwargs=(("pages", [-1]),), + ), + InvalidOcrCase( + name="invalid_document_intelligence_features", + model="azure_ai/doc-intelligence/prebuilt-read", + document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="Invalid `features` for Azure Document Intelligence", + extra_kwargs=(("features", [1]),), + ), + InvalidOcrCase( + name="invalid_header_value", + model="mistral/mistral-ocr-latest", + document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, + expected_exception_type="litellm.exceptions.InternalServerError", + expected_status_code=500, + expected_message="Header value must be str or bytes", + extra_kwargs=(("extra_headers", {"x-invalid": 1}),), + ), +) + + def _call_kwargs(sdk_input: OcrSdkInput, mock_url: str, route: SDKRoute) -> dict[str, object]: return { **sdk_input.as_sdk_kwargs(), @@ -49,22 +168,50 @@ def _call_kwargs(sdk_input: OcrSdkInput, mock_url: str, route: SDKRoute) -> dict } +def _execute_sdk_call( + call_kwargs: dict[str, object], + route: SDKRoute, + event_loop: asyncio.AbstractEventLoop, +) -> SDKReport: + import litellm + + try: + if route is SDKRoute.OCR: + sync_route: Final = cast(Callable[..., OCRResponse], litellm.ocr) + response: Final = sync_route(**call_kwargs) + return SDKSuccess(response=response.model_dump(mode="json")) + async_route: Final = cast(Callable[..., Coroutine[object, object, OCRResponse]], litellm.aocr) + async_response: Final = event_loop.run_until_complete(async_route(**call_kwargs)) + return SDKSuccess(response=async_response.model_dump(mode="json")) + except Exception as error: + return sdk_error_report(error) + + def _execute_sdk_case( sdk_input: OcrSdkInput, route: SDKRoute, mock_url: str, event_loop: asyncio.AbstractEventLoop, ) -> SDKReport: - import litellm - call_kwargs: Final = _call_kwargs(sdk_input, mock_url, route) - if route is SDKRoute.OCR: - sync_route: Final = cast(Callable[..., OCRResponse], litellm.ocr) - response: Final = sync_route(**call_kwargs) - return SDKReport(response=response.model_dump(mode="json")) - async_route: Final = cast(Callable[..., Coroutine[object, object, OCRResponse]], litellm.aocr) - async_response: Final = event_loop.run_until_complete(async_route(**call_kwargs)) - return SDKReport(response=async_response.model_dump(mode="json")) + return _execute_sdk_call(call_kwargs, route, event_loop) + + +def _execute_invalid_sdk_case( + case: InvalidOcrCase, + route: SDKRoute, + mock_url: str, + event_loop: asyncio.AbstractEventLoop, +) -> SDKReport: + call_kwargs: Final = { + "model": case.model, + "document": case.document, + "api_base": mock_url, + "api_key": API_KEY, + "extra_headers": {"x-litellm-parity-route": route.value}, + **dict(case.extra_kwargs), + } + return _execute_sdk_call(call_kwargs, route, event_loop) def _call_sdk_case(sdk_input: OcrSdkInput, route: SDKRoute, mock_url: str) -> OCRResponse: @@ -211,6 +358,42 @@ def test_recorded_ocr_sdk_parity( assert_model_parity(python.response, rust.response) +@pytest.mark.parametrize("case", INVALID_OCR_CASES, ids=tuple(case.name for case in INVALID_OCR_CASES)) +@pytest.mark.parametrize("route", tuple(SDKRoute), ids=tuple(route.value for route in SDKRoute)) +def test_invalid_ocr_sdk_parity(case: InvalidOcrCase, route: SDKRoute) -> None: + sync_spy, async_spy = _native_spies() + event_loop: Final = asyncio.new_event_loop() + try: + with _restore_rust_ocr_state(), replay_server() as provider: + rust_ocr_bridge.use_litellm_rust(False, ocr=sync_spy, aocr=async_spy) + python: Final = run_in_process( + provider, + (), + lambda mock_url: _execute_invalid_sdk_case(case, route, mock_url, event_loop), + ) + assert sync_spy.calls == 0 + assert async_spy.calls == 0 + + rust_ocr_bridge.use_litellm_rust(True, ocr=sync_spy, aocr=async_spy) + rust: Final = run_in_process( + provider, + (), + lambda mock_url: _execute_invalid_sdk_case(case, route, mock_url, event_loop), + ) + finally: + event_loop.close() + + assert sync_spy.calls == (case.expected_rust_calls if route is SDKRoute.OCR else 0) + assert async_spy.calls == (case.expected_rust_calls if route is SDKRoute.AOCR else 0) + assert python.requests == () + assert rust.requests == () + assert python.response == rust.response + assert isinstance(python.response, SDKError) + assert python.response.exception_type == case.expected_exception_type + assert python.response.status_code == case.expected_status_code + assert case.expected_message in python.response.message + + def test_ocr_subprocess_startup_smoke( startup_ocr_fixture: OcrParityCase, tmp_path: Path,