test(ocr): cover invalid input parity

litellm-rust/crates/python-bridge/src/lib.rs

@@ -60,6 +60,9 @@ fn chat_completions_response_to_py(
 
 fn core_error_to_pyerr(err: CoreError) -> PyErr {
     match err {
+        CoreError::MissingField("document_url") => {
+            PyValueError::new_err("Document URL is required")
+        }
         CoreError::Auth(message) => PyValueError::new_err(message),
         CoreError::InvalidProvider(_)
         | CoreError::InvalidRequest(_)

litellm/llms/vertex_ai/ocr/deepseek_transformation.py

@@ -50,9 +50,7 @@ class VertexAIDeepSeekOCRConfig(BaseOCRConfig):
         return {  # mutable-ok: base contract
             **optional_params,
             **{  # mutable-ok: base contract
-                key: value
-                for key, value in non_default_params.items()
-                if key in self.get_supported_ocr_params(model)
+                key: value for key, value in non_default_params.items() if key in self.get_supported_ocr_params(model)
             },
         }
 

litellm/ocr/main.py

@@ -191,6 +191,10 @@ def _prepare_ocr_request(
 def _rust_ocr_supported(prepared_request: _PreparedOCRRequest) -> bool:
     if prepared_request.optional_params.get(OCR_REQUEST_FORMAT_PARAM) == "native":
         return False
+    if prepared_request.extra_headers is not None and any(
+        not isinstance(value, str) for value in prepared_request.extra_headers.values()
+    ):
+        return False
     return prepared_request.custom_llm_provider in _RUST_OCR_PROVIDERS
 
 

tests/route_parity/README.md

@@ -1,4 +1,4 @@
-# Python/Python parity testing in Python SDK interface
+# Python/Rust parity testing in the Python SDK interface
 
 > Given the same SDK call and identical provider behavior, does the PyO3 implementation behave same as Python?
 
@@ -11,6 +11,8 @@
 - The harness compares the values returned through the Python SDK interface
 - Non-streaming responses are compared directly, including their concrete return type and public model fields
 - Streaming responses are consumed and compared chunk by chunk, including wrapper type, chunk type and order, termination, and public exception behavior
+- Failed SDK calls are compared by exception class, stable message, status, code, model, provider, and parameter fields
+- Traceback paths and line numbers are excluded because they are runtime-specific
 - Route-specific comparators and chunk normalizers handle differences in each public SDK contract
 
 ## Process isolation
@@ -48,6 +50,12 @@ a machine records the boundaries it has configured and skips the rest. Reducto f
 responses, but Reducto remains outside Python/Rust parity until the Rust OCR bridge supports it.
 Azure-hosted Mistral discovery also requires `AZURE_AI_OCR_MODEL`, the deployment name or full `azure_ai/...` model.
 
+Invalid OCR inputs do not use recorded provider responses. The parity suite checks unsupported providers and models,
+malformed documents, invalid request formats, invalid Azure Document Intelligence parameters, and invalid headers in
+both sync and async SDK calls. These cases must return the same public exception fields without sending a provider
+request. A malformed Azure Document Intelligence document reaches the Rust bridge so its native validation error is
+also compared against Python
+
 ## References
 
 - [Hypothesis documentation](https://hypothesis.readthedocs.io/en/latest/)

tests/route_parity/compare.py

@@ -25,9 +25,7 @@ def _request_after_transformation(request: CapturedRequest) -> CapturedRequest:
     return request.model_copy(update={"user_agent": None})
 
 
-def assert_request_parity(
-    python: tuple[CapturedRequest, ...], accelerated: tuple[CapturedRequest, ...]
-) -> None:
+def assert_request_parity(python: tuple[CapturedRequest, ...], accelerated: tuple[CapturedRequest, ...]) -> None:
     python_requests: Final = tuple(_request_after_transformation(request) for request in python)
     accelerated_requests: Final = tuple(_request_after_transformation(request) for request in accelerated)
     assert python_requests == accelerated_requests
@@ -47,4 +45,4 @@ def assert_model_parity(python: BaseModel, accelerated: BaseModel) -> None:
 def assert_parity(python: Execution, accelerated: Execution, python_user_agent: str) -> None:
     validate_harness(python, accelerated, python_user_agent)
     assert_request_parity(python.requests, accelerated.requests)
-    assert python.report.response == accelerated.report.response
+    assert python.report == accelerated.report

tests/route_parity/models.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Annotated, Literal
+from typing import Annotated, Final, Literal, cast
 
 from pydantic import BaseModel, ConfigDict, Field, JsonValue
 
@@ -15,12 +15,51 @@ class CapturedRequest(BaseModel):
     user_agent: str | None
 
 
-class SDKReport(BaseModel):
+class SDKSuccess(BaseModel):
     model_config = ConfigDict(frozen=True)
 
+    status: Literal["ok"] = "ok"
     response: JsonValue
 
 
+class SDKError(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    status: Literal["error"] = "error"
+    exception_type: str
+    message: str
+    status_code: int | None
+    code: str | None
+    error_type: str | None
+    param: str | None
+    model: str | None
+    llm_provider: str | None
+
+
+SDKReport = Annotated[SDKSuccess | SDKError, Field(discriminator="status")]
+
+
+def _string_attribute(error: Exception, name: str) -> str | None:
+    value: Final = cast(object | None, getattr(error, name, None))
+    return None if value is None else str(value)
+
+
+def sdk_error_report(error: Exception) -> SDKError:
+    message, _, _ = str(error).partition("\nTraceback (most recent call last):")
+    raw_status_code: Final = cast(object | None, getattr(error, "status_code", None))
+    status_code: Final = raw_status_code if isinstance(raw_status_code, int) else None
+    return SDKError(
+        exception_type=f"{type(error).__module__}.{type(error).__qualname__}",
+        message=message.rstrip(),
+        status_code=status_code,
+        code=_string_attribute(error, "code"),
+        error_type=_string_attribute(error, "type"),
+        param=_string_attribute(error, "param"),
+        model=_string_attribute(error, "model"),
+        llm_provider=_string_attribute(error, "llm_provider"),
+    )
+
+
 class Execution(BaseModel):
     model_config = ConfigDict(frozen=True)
 

tests/route_parity/test_parity.py

@@ -6,7 +6,7 @@ import pytest
 from pydantic import BaseModel, JsonValue, PrivateAttr
 
 from tests.route_parity.compare import assert_model_parity, assert_parity
-from tests.route_parity.models import CapturedRequest, Execution, SDKReport
+from tests.route_parity.models import CapturedRequest, Execution, SDKError, SDKSuccess, sdk_error_report
 
 SENTINEL: Final = "python-parity-fallback"
 
@@ -23,6 +23,10 @@ class _DifferentResponse(BaseModel):
     value: str
 
 
+class _PublicError(ValueError):
+    status_code: Final = 400
+
+
 def _execution(*, body: JsonValue = None, markdown: str = "same", user_agent: str | None = None) -> Execution:
     return Execution(
         requests=(
@@ -34,7 +38,7 @@ def _execution(*, body: JsonValue = None, markdown: str = "same", user_agent: st
                 user_agent=user_agent,
             ),
         ),
-        report=SDKReport(response={"items": [{"text": markdown}], "model": "test-model"}),
+        report=SDKSuccess(response={"items": [{"text": markdown}], "model": "test-model"}),
     )
 
 
@@ -54,6 +58,36 @@ def test_parity_rejects_response_difference() -> None:
         assert_parity(python, rust, SENTINEL)
 
 
+def test_parity_rejects_error_difference() -> None:
+    python: Final = Execution(
+        requests=(),
+        report=SDKError(
+            exception_type="litellm.exceptions.BadRequestError",
+            message="bad request",
+            status_code=400,
+            code=None,
+            error_type=None,
+            param=None,
+            model="test-model",
+            llm_provider="mistral",
+        ),
+    )
+    rust: Final = python.model_copy(update={"report": python.report.model_copy(update={"status_code": 500})})
+
+    with pytest.raises(AssertionError):
+        assert_parity(python, rust, SENTINEL)
+
+
+def test_sdk_error_report_removes_traceback_but_keeps_public_fields() -> None:
+    error: Final = _PublicError("invalid input\nTraceback (most recent call last):\n  unstable")
+
+    report: Final = sdk_error_report(error)
+
+    assert report.exception_type.endswith("._PublicError")
+    assert report.message == "invalid input"
+    assert report.status_code == 400
+
+
 def test_parity_rejects_rust_fallback() -> None:
     python: Final = _execution(user_agent=SENTINEL)
     rust: Final = _execution(user_agent=SENTINEL)

tests/test_litellm/ocr/test_rust_bridge.py

@@ -816,6 +816,27 @@ def test_ocr_falls_back_to_python_when_bridge_unavailable(monkeypatch):
     assert isinstance(response, OCRResponse)
 
 
+def test_ocr_non_string_header_uses_python_path(monkeypatch):
+    bridge = RecordingBridge()
+    litellm.use_litellm_rust(True, ocr=bridge)
+
+    def fake_handler_ocr(**kwargs):
+        assert kwargs["headers"] == {"x-invalid": 1}
+        return OCRResponse(pages=[], model="mistral-ocr-latest", object="ocr")
+
+    monkeypatch.setattr(ocr_main.base_llm_http_handler, "ocr", fake_handler_ocr)
+
+    response = litellm.ocr(
+        model=MODEL,
+        document=DOCUMENT,
+        api_key="sk-test",
+        extra_headers={"x-invalid": 1},
+    )
+
+    assert isinstance(response, OCRResponse)
+    assert bridge.calls == []
+
+
 def test_ocr_provider_configs_expose_api_key_env_vars():
     from litellm.llms.azure_ai.ocr.document_intelligence.transformation import (
         AzureDocumentIntelligenceOCRConfig,

tests/test_litellm/ocr/test_sdk_parity.py

@@ -6,6 +6,7 @@ import sys
 import traceback
 from collections.abc import Awaitable, Callable, Coroutine, Generator
 from contextlib import contextmanager
+from dataclasses import dataclass
 from enum import Enum
 from pathlib import Path
 from typing import Final, cast
@@ -19,7 +20,16 @@ from litellm.rust_bridge.ocr import RustAocr, RustOcr
 from tests.route_parity.compare import assert_model_parity, assert_parity, assert_request_parity
 from tests.route_parity.fixture_recorder import recorded_fixtures
 from tests.route_parity.inprocess import run_in_process
-from tests.route_parity.models import SDKCommand, SDKReport, WorkerFailure, WorkerResult, WorkerSuccess
+from tests.route_parity.models import (
+    SDKCommand,
+    SDKError,
+    SDKReport,
+    SDKSuccess,
+    WorkerFailure,
+    WorkerResult,
+    WorkerSuccess,
+    sdk_error_report,
+)
 from tests.route_parity.replay import replay_server
 from tests.route_parity.runner import (
     PythonScriptRunner,
@@ -40,6 +50,115 @@ class SDKRoute(str, Enum):
     AOCR = "aocr"
 
 
+@dataclass(frozen=True, slots=True)
+class InvalidOcrCase:
+    name: str
+    model: str
+    document: object
+    expected_exception_type: str
+    expected_status_code: int
+    expected_message: str
+    extra_kwargs: tuple[tuple[str, object], ...] = ()
+    expected_rust_calls: int = 0
+
+
+INVALID_OCR_CASES: Final = (
+    InvalidOcrCase(
+        name="unsupported_provider",
+        model="openai/gpt-4o",
+        document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="},
+        expected_exception_type="litellm.exceptions.APIConnectionError",
+        expected_status_code=500,
+        expected_message="OCR is not supported for provider: openai",
+    ),
+    InvalidOcrCase(
+        name="unsupported_reducto_model",
+        model="reducto/parse-v4",
+        document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="},
+        expected_exception_type="litellm.exceptions.APIConnectionError",
+        expected_status_code=500,
+        expected_message="OCR is not supported for provider: reducto",
+    ),
+    InvalidOcrCase(
+        name="unknown_provider_prefix",
+        model="not_a_provider/model",
+        document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="},
+        expected_exception_type="litellm.exceptions.BadRequestError",
+        expected_status_code=400,
+        expected_message="LLM Provider NOT provided",
+    ),
+    InvalidOcrCase(
+        name="non_object_document",
+        model="mistral/mistral-ocr-latest",
+        document=[],
+        expected_exception_type="litellm.exceptions.APIConnectionError",
+        expected_status_code=500,
+        expected_message="document must be a dict",
+    ),
+    InvalidOcrCase(
+        name="missing_document_type",
+        model="mistral/mistral-ocr-latest",
+        document={},
+        expected_exception_type="litellm.exceptions.APIConnectionError",
+        expected_status_code=500,
+        expected_message="Invalid document type: None",
+    ),
+    InvalidOcrCase(
+        name="unsupported_document_type",
+        model="mistral/mistral-ocr-latest",
+        document={"type": "text", "text": "not a document"},
+        expected_exception_type="litellm.exceptions.APIConnectionError",
+        expected_status_code=500,
+        expected_message="Invalid document type: text",
+    ),
+    InvalidOcrCase(
+        name="missing_document_url",
+        model="azure_ai/doc-intelligence/prebuilt-read",
+        document={"type": "document_url"},
+        expected_exception_type="litellm.exceptions.APIConnectionError",
+        expected_status_code=500,
+        expected_message="Document URL is required",
+        expected_rust_calls=1,
+    ),
+    InvalidOcrCase(
+        name="invalid_request_format",
+        model="mistral/mistral-ocr-latest",
+        document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="},
+        expected_exception_type="litellm.exceptions.UnsupportedParamsError",
+        expected_status_code=400,
+        expected_message="Invalid `req_format`: 'bogus'",
+        extra_kwargs=(("req_format", "bogus"),),
+    ),
+    InvalidOcrCase(
+        name="invalid_document_intelligence_pages",
+        model="azure_ai/doc-intelligence/prebuilt-read",
+        document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="},
+        expected_exception_type="litellm.exceptions.APIConnectionError",
+        expected_status_code=500,
+        expected_message="`pages` integers must be >= 0",
+        extra_kwargs=(("pages", [-1]),),
+    ),
+    InvalidOcrCase(
+        name="invalid_document_intelligence_features",
+        model="azure_ai/doc-intelligence/prebuilt-read",
+        document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="},
+        expected_exception_type="litellm.exceptions.APIConnectionError",
+        expected_status_code=500,
+        expected_message="Invalid `features` for Azure Document Intelligence",
+        extra_kwargs=(("features", [1]),),
+    ),
+    InvalidOcrCase(
+        name="invalid_header_value",
+        model="mistral/mistral-ocr-latest",
+        document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="},
+        expected_exception_type="litellm.exceptions.InternalServerError",
+        expected_status_code=500,
+        expected_message="Header value must be str or bytes",
+        extra_kwargs=(("extra_headers", {"x-invalid": 1}),),
+    ),
+)
+
+
 def _call_kwargs(sdk_input: OcrSdkInput, mock_url: str, route: SDKRoute) -> dict[str, object]:
     return {
         **sdk_input.as_sdk_kwargs(),
@@ -49,22 +168,50 @@ def _call_kwargs(sdk_input: OcrSdkInput, mock_url: str, route: SDKRoute) -> dict
     }
 
 
+def _execute_sdk_call(
+    call_kwargs: dict[str, object],
+    route: SDKRoute,
+    event_loop: asyncio.AbstractEventLoop,
+) -> SDKReport:
+    import litellm
+
+    try:
+        if route is SDKRoute.OCR:
+            sync_route: Final = cast(Callable[..., OCRResponse], litellm.ocr)
+            response: Final = sync_route(**call_kwargs)
+            return SDKSuccess(response=response.model_dump(mode="json"))
+        async_route: Final = cast(Callable[..., Coroutine[object, object, OCRResponse]], litellm.aocr)
+        async_response: Final = event_loop.run_until_complete(async_route(**call_kwargs))
+        return SDKSuccess(response=async_response.model_dump(mode="json"))
+    except Exception as error:
+        return sdk_error_report(error)
+
+
 def _execute_sdk_case(
     sdk_input: OcrSdkInput,
     route: SDKRoute,
     mock_url: str,
     event_loop: asyncio.AbstractEventLoop,
 ) -> SDKReport:
-    import litellm
-
     call_kwargs: Final = _call_kwargs(sdk_input, mock_url, route)
-    if route is SDKRoute.OCR:
-        sync_route: Final = cast(Callable[..., OCRResponse], litellm.ocr)
-        response: Final = sync_route(**call_kwargs)
-        return SDKReport(response=response.model_dump(mode="json"))
-    async_route: Final = cast(Callable[..., Coroutine[object, object, OCRResponse]], litellm.aocr)
-    async_response: Final = event_loop.run_until_complete(async_route(**call_kwargs))
-    return SDKReport(response=async_response.model_dump(mode="json"))
+    return _execute_sdk_call(call_kwargs, route, event_loop)
+
+
+def _execute_invalid_sdk_case(
+    case: InvalidOcrCase,
+    route: SDKRoute,
+    mock_url: str,
+    event_loop: asyncio.AbstractEventLoop,
+) -> SDKReport:
+    call_kwargs: Final = {
+        "model": case.model,
+        "document": case.document,
+        "api_base": mock_url,
+        "api_key": API_KEY,
+        "extra_headers": {"x-litellm-parity-route": route.value},
+        **dict(case.extra_kwargs),
+    }
+    return _execute_sdk_call(call_kwargs, route, event_loop)
 
 
 def _call_sdk_case(sdk_input: OcrSdkInput, route: SDKRoute, mock_url: str) -> OCRResponse:
@@ -211,6 +358,42 @@ def test_recorded_ocr_sdk_parity(
     assert_model_parity(python.response, rust.response)
 
 
+@pytest.mark.parametrize("case", INVALID_OCR_CASES, ids=tuple(case.name for case in INVALID_OCR_CASES))
+@pytest.mark.parametrize("route", tuple(SDKRoute), ids=tuple(route.value for route in SDKRoute))
+def test_invalid_ocr_sdk_parity(case: InvalidOcrCase, route: SDKRoute) -> None:
+    sync_spy, async_spy = _native_spies()
+    event_loop: Final = asyncio.new_event_loop()
+    try:
+        with _restore_rust_ocr_state(), replay_server() as provider:
+            rust_ocr_bridge.use_litellm_rust(False, ocr=sync_spy, aocr=async_spy)
+            python: Final = run_in_process(
+                provider,
+                (),
+                lambda mock_url: _execute_invalid_sdk_case(case, route, mock_url, event_loop),
+            )
+            assert sync_spy.calls == 0
+            assert async_spy.calls == 0
+
+            rust_ocr_bridge.use_litellm_rust(True, ocr=sync_spy, aocr=async_spy)
+            rust: Final = run_in_process(
+                provider,
+                (),
+                lambda mock_url: _execute_invalid_sdk_case(case, route, mock_url, event_loop),
+            )
+    finally:
+        event_loop.close()
+
+    assert sync_spy.calls == (case.expected_rust_calls if route is SDKRoute.OCR else 0)
+    assert async_spy.calls == (case.expected_rust_calls if route is SDKRoute.AOCR else 0)
+    assert python.requests == ()
+    assert rust.requests == ()
+    assert python.response == rust.response
+    assert isinstance(python.response, SDKError)
+    assert python.response.exception_type == case.expected_exception_type
+    assert python.response.status_code == case.expected_status_code
+    assert case.expected_message in python.response.message
+
+
 def test_ocr_subprocess_startup_smoke(
     startup_ocr_fixture: OcrParityCase,
     tmp_path: Path,