diff --git a/litellm/llms/vertex_ai/ocr/deepseek_transformation.py b/litellm/llms/vertex_ai/ocr/deepseek_transformation.py index 2603552152d..51d22283464 100644 --- a/litellm/llms/vertex_ai/ocr/deepseek_transformation.py +++ b/litellm/llms/vertex_ai/ocr/deepseek_transformation.py @@ -38,6 +38,24 @@ class VertexAIDeepSeekOCRConfig(BaseOCRConfig): super().__init__() self.vertex_base = VertexBase() + def get_supported_ocr_params(self, model: str) -> list[str]: # mutable-ok: base contract + return ["stream", "temperature", "max_tokens", "top_p", "n", "stop"] # mutable-ok: base contract + + def map_ocr_params( + self, + non_default_params: dict[str, object], # mutable-ok: base contract + optional_params: dict[str, object], # mutable-ok: base contract + model: str, + ) -> dict[str, object]: # mutable-ok: base contract + return { # mutable-ok: base contract + **optional_params, + **{ # mutable-ok: base contract + key: value + for key, value in non_default_params.items() + if key in self.get_supported_ocr_params(model) + }, + } + def get_api_key_env_var(self) -> str | None: return VERTEX_AI_DEEPSEEK_OCR_API_KEY_ENV_VAR @@ -177,8 +195,9 @@ class VertexAIDeepSeekOCRConfig(BaseOCRConfig): content_item = {"type": "image_url", "image_url": document_url} # Build DeepSeek OCR request + upstream_model: Final = model if model.startswith("deepseek-ai/") else f"deepseek-ai/{model}" data: Final = { - "model": "deepseek-ai/" + model, + "model": upstream_model, "messages": [{"role": "user", "content": [content_item]}], } diff --git a/tests/route_parity/README.md b/tests/route_parity/README.md index c063491bd78..59b81b35b46 100644 --- a/tests/route_parity/README.md +++ b/tests/route_parity/README.md @@ -32,6 +32,22 @@ - Provider responses are replayed unchanged, so the parity test does not fuzz or validate provider behavior - Because Hypothesis does not run the parity assertion directly, parity failures are not automatically shrunk +## OCR input boundaries + +OCR strategies generate only public `litellm.ocr()` and `litellm.aocr()` inputs. Every case contains the normalized +`model`, Mistral-shaped `document`, optional `custom_llm_provider`, and LiteLLM keyword arguments. The fixture-only +`boundary` tag selects the valid keyword-argument set and is removed before calling the SDK. Strategies never build +provider wire payloads. + +Each boundary has a required corpus containing a baseline and one case for every supported top-level LiteLLM OCR +parameter. `--examples` controls additional Hypothesis-generated cases; it does not replace the required corpus. + +The explicit boundaries are Mistral, Azure-hosted Mistral, Vertex-hosted Mistral, Azure Document Intelligence, +Vertex DeepSeek, Reducto v3, and Reducto legacy. Provider credentials and endpoints only control target discovery, so +a machine records the boundaries it has configured and skips the rest. Reducto fixtures record both upload and parse +responses, but Reducto remains outside Python/Rust parity until the Rust OCR bridge supports it. +Azure-hosted Mistral discovery also requires `AZURE_AI_OCR_MODEL`, the deployment name or full `azure_ai/...` model. + ## References - [Hypothesis documentation](https://hypothesis.readthedocs.io/en/latest/) diff --git a/tests/route_parity/compare.py b/tests/route_parity/compare.py index 4fe9fdea9b4..609298496c3 100644 --- a/tests/route_parity/compare.py +++ b/tests/route_parity/compare.py @@ -10,23 +10,27 @@ ModelT = TypeVar("ModelT", bound=BaseModel) def validate_harness(python: Execution, accelerated: Execution, python_user_agent: str) -> None: - if python.request.user_agent != python_user_agent: - raise AssertionError( - f"Python provider request did not carry fallback sentinel user-agent {python_user_agent!r}: " - f"{python.request.user_agent!r}" - ) - if accelerated.request.user_agent == python_user_agent: - raise AssertionError("accelerated route fell back to the Python HTTP implementation") + for request in python.requests: + if request.user_agent != python_user_agent: + raise AssertionError( + f"Python provider request did not carry fallback sentinel user-agent {python_user_agent!r}: " + f"{request.user_agent!r}" + ) + for request in accelerated.requests: + if request.user_agent == python_user_agent: + raise AssertionError("accelerated route fell back to the Python HTTP implementation") def _request_after_transformation(request: CapturedRequest) -> CapturedRequest: return request.model_copy(update={"user_agent": None}) -def assert_request_parity(python: CapturedRequest, accelerated: CapturedRequest) -> None: - python_request: Final = _request_after_transformation(python) - accelerated_request: Final = _request_after_transformation(accelerated) - assert python_request == accelerated_request +def assert_request_parity( + python: tuple[CapturedRequest, ...], accelerated: tuple[CapturedRequest, ...] +) -> None: + python_requests: Final = tuple(_request_after_transformation(request) for request in python) + accelerated_requests: Final = tuple(_request_after_transformation(request) for request in accelerated) + assert python_requests == accelerated_requests def public_model_copy(model: ModelT) -> ModelT: @@ -42,5 +46,5 @@ def assert_model_parity(python: BaseModel, accelerated: BaseModel) -> None: def assert_parity(python: Execution, accelerated: Execution, python_user_agent: str) -> None: validate_harness(python, accelerated, python_user_agent) - assert_request_parity(python.request, accelerated.request) + assert_request_parity(python.requests, accelerated.requests) assert python.report.response == accelerated.report.response diff --git a/tests/route_parity/fixture_generator.py b/tests/route_parity/fixture_generator.py index eeef76eb264..0b4eae55c99 100644 --- a/tests/route_parity/fixture_generator.py +++ b/tests/route_parity/fixture_generator.py @@ -31,6 +31,7 @@ class FixtureTarget(Generic[InputT]): provider_spec: ProviderSpec strategy: SearchStrategy[InputT] invoke: Callable[[str, InputT], object] + required_inputs: tuple[InputT, ...] = () def generate_target_fixtures( @@ -40,10 +41,11 @@ def generate_target_fixtures( concurrency: int, case_type: type[CaseT], ) -> None: - case_inputs: Final = generate_case_inputs(target.strategy, examples) + generated_inputs: Final = generate_case_inputs(target.strategy, examples) + case_inputs: Final = (*target.required_inputs, *generated_inputs) results: Final = record_cases( target.provider_spec, - root, + root / target.name, case_inputs, target.invoke, case_type, diff --git a/tests/route_parity/fixture_models.py b/tests/route_parity/fixture_models.py index 799cb472da4..61ec0484611 100644 --- a/tests/route_parity/fixture_models.py +++ b/tests/route_parity/fixture_models.py @@ -1,8 +1,9 @@ from __future__ import annotations -from typing import Generic, Literal, TypeVar, cast +from collections.abc import Mapping +from typing import Final, Generic, Literal, TypeVar, cast -from pydantic import BaseModel, ConfigDict, Field, JsonValue +from pydantic import BaseModel, ConfigDict, Field, JsonValue, model_validator from tests.route_parity.recorded_http import RecordedResponse @@ -14,11 +15,15 @@ class FixtureModel(BaseModel): class SdkInputBase(FixtureModel): + boundary: str = "default" + def as_sdk_kwargs(self) -> dict[str, object]: - return cast(dict[str, object], self.model_dump(mode="python", exclude_unset=True)) + dumped: Final = cast(dict[str, object], self.model_dump(mode="python", exclude_unset=True)) + return {key: value for key, value in dumped.items() if key != "boundary"} def canonical_input(self) -> dict[str, object]: - return cast(dict[str, object], self.model_dump(mode="json", exclude_unset=True)) + dumped: Final = cast(dict[str, object], self.model_dump(mode="json", exclude_unset=True)) + return {"boundary": self.boundary, **{key: value for key, value in dumped.items() if key != "boundary"}} class JsonSchemaDefinition(FixtureModel): @@ -38,4 +43,36 @@ InputT = TypeVar("InputT", bound=SdkInputBase) class ParityCase(FixtureModel, Generic[InputT]): litellm_input: InputT - provider_response: RecordedResponse + provider_responses: tuple[RecordedResponse, ...] + + @model_validator(mode="before") + @classmethod + def load_legacy_single_response(cls, value: object) -> object: + if not isinstance(value, Mapping): + return value + migrated: Final = dict(value) + provider_response: Final = migrated.pop("provider_response", None) + if "provider_responses" not in migrated and provider_response is not None: + migrated["provider_responses"] = (provider_response,) + litellm_input: Final = migrated.get("litellm_input") + if isinstance(litellm_input, Mapping) and "boundary" not in litellm_input: + model: Final = litellm_input.get("model") + if isinstance(model, str): + migrated["litellm_input"] = {"boundary": _legacy_boundary(model), **litellm_input} + return migrated + + +def _legacy_boundary(model: str) -> str: + if model.startswith("azure_ai/doc-intelligence/"): + return "azure_document_intelligence" + if model.startswith("azure_ai/"): + return "azure_mistral" + if model.startswith("vertex_ai/deepseek"): + return "vertex_deepseek" + if model.startswith("vertex_ai/"): + return "vertex_mistral" + if model.endswith("parse-v3"): + return "reducto_v3" + if model.endswith("parse-legacy"): + return "reducto_legacy" + return "mistral" diff --git a/tests/route_parity/fixture_recorder.py b/tests/route_parity/fixture_recorder.py index 6cb9141f9b5..4ee15ac02eb 100644 --- a/tests/route_parity/fixture_recorder.py +++ b/tests/route_parity/fixture_recorder.py @@ -12,6 +12,7 @@ from datetime import datetime, timezone from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from typing import Final, Generic, Protocol, TypeVar, cast +from urllib.parse import urlsplit, urlunsplit import httpx import pytest @@ -29,6 +30,7 @@ from tests.route_parity.recorded_http import ( ) FIXTURE_SCHEMA_VERSION: Final = 1 +_PARITY_PROVIDER_HOST: Final = "parity-provider.invalid" _HOP_BY_HOP_HEADERS: Final = frozenset( { @@ -82,7 +84,29 @@ def _excluded_headers(headers: tuple[tuple[str, str], ...]) -> frozenset[str]: def _end_to_end_headers(headers: httpx.Headers) -> tuple[HttpHeader, ...]: decoded: Final = tuple((name.decode("ascii"), value.decode("latin-1")) for name, value in headers.raw) excluded: Final = _excluded_headers(decoded) | {"content-encoding", "content-length"} - return tuple(HttpHeader(name=name, value=value) for name, value in decoded if name.lower() not in excluded) + return tuple( + HttpHeader(name=name, value=_normalized_response_header(name, value)) + for name, value in decoded + if name.lower() not in excluded + ) + + +def _normalized_response_header(name: str, value: str) -> str: + if name.lower() not in {"location", "operation-location"}: + return value + parsed: Final = urlsplit(value) + if not parsed.netloc: + return value + return urlunsplit(("http", _PARITY_PROVIDER_HOST, parsed.path, parsed.query, parsed.fragment)) + + +def _local_response_header(name: str, value: str, provider_url: str) -> str: + if name.lower() not in {"location", "operation-location"}: + return value + parsed: Final = urlsplit(value) + if parsed.hostname != _PARITY_PROVIDER_HOST: + return value + return f"{provider_url}{parsed.path}{'?' + parsed.query if parsed.query else ''}" class _RecordingProvider(ThreadingHTTPServer): @@ -97,21 +121,29 @@ class _RecordingProvider(ThreadingHTTPServer): def url(self) -> str: return f"http://127.0.0.1:{self.server_address[1]}" - def take_response(self) -> RecordedResponse: + def take_responses(self) -> tuple[RecordedResponse, ...]: try: - return self.responses.get(timeout=5) + first: Final = self.responses.get(timeout=5) except queue.Empty as error: raise RuntimeError("successful SDK call did not produce a recorded response") from error + remaining: Final = tuple(self.responses.get_nowait() for _ in range(self.responses.qsize())) + return (first, *remaining) class _RecordingHandler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" def do_POST(self) -> None: + self._forward() + + def do_GET(self) -> None: + self._forward() + + def _forward(self) -> None: provider: Final = self.server assert isinstance(provider, _RecordingProvider) length: Final = int(self.headers.get("content-length") or "0") - request_body: Final = self.rfile.read(length) + request_body: Final = self.rfile.read(length) if length else b"" raw_headers: Final = tuple(self.headers.raw_items()) excluded: Final = _excluded_headers(raw_headers) | {"host", "content-length"} forwarded_headers: Final = tuple((name, value) for name, value in raw_headers if name.lower() not in excluded) @@ -160,7 +192,9 @@ class _RecordingHandler(BaseHTTPRequestHandler): ) -> RecordedHttpStreamResponse: self.send_response_only(upstream.status_code) for header in headers: - self.send_header(header.name, header.value) + provider: Final = self.server + assert isinstance(provider, _RecordingProvider) + self.send_header(header.name, _local_response_header(header.name, header.value, provider.url)) self.send_header("transfer-encoding", "chunked") self.end_headers() chunks: Final = tuple(self._relay_chunks(upstream.iter_bytes())) @@ -184,7 +218,9 @@ class _RecordingHandler(BaseHTTPRequestHandler): def _send_response(self, status_code: int, headers: tuple[HttpHeader, ...], body: bytes) -> None: self.send_response_only(status_code) for header in headers: - self.send_header(header.name, header.value) + provider: Final = self.server + assert isinstance(provider, _RecordingProvider) + self.send_header(header.name, _local_response_header(header.name, header.value, provider.url)) self.send_header("content-length", str(len(body))) self.end_headers() self.wfile.write(body) @@ -252,9 +288,9 @@ def record_case( with _recording_provider(spec) as recorder: sdk_call(recorder.url, case_input) - upstream_response: Final = recorder.take_response() + upstream_responses: Final = recorder.take_responses() - case: Final = case_type.model_validate({"litellm_input": case_input, "provider_response": upstream_response}) + case: Final = case_type.model_validate({"litellm_input": case_input, "provider_responses": upstream_responses}) envelope: Final = FixtureEnvelope( schema_version=FIXTURE_SCHEMA_VERSION, recorded_at=datetime.now(timezone.utc), diff --git a/tests/route_parity/inprocess.py b/tests/route_parity/inprocess.py index 74abbae9b35..51744ae4e94 100644 --- a/tests/route_parity/inprocess.py +++ b/tests/route_parity/inprocess.py @@ -13,19 +13,20 @@ ResponseT = TypeVar("ResponseT") @dataclass(frozen=True, slots=True) class InProcessExecution(Generic[ResponseT]): - request: CapturedRequest + requests: tuple[CapturedRequest, ...] response: ResponseT def run_in_process( provider: ReplayServer, - recorded_response: RecordedResponse, + recorded_responses: tuple[RecordedResponse, ...], call: Callable[[str], ResponseT], ) -> InProcessExecution[ResponseT]: - provider.enqueue_response(recorded_response) + for recorded_response in recorded_responses: + provider.enqueue_response(recorded_response) try: response = call(provider.url) - return InProcessExecution(request=provider.take_request(), response=response) + return InProcessExecution(requests=provider.take_requests(len(recorded_responses)), response=response) except Exception: provider.reset() raise diff --git a/tests/route_parity/models.py b/tests/route_parity/models.py index e61cdf1ef84..41f1da4188d 100644 --- a/tests/route_parity/models.py +++ b/tests/route_parity/models.py @@ -24,7 +24,7 @@ class SDKReport(BaseModel): class Execution(BaseModel): model_config = ConfigDict(frozen=True) - request: CapturedRequest + requests: tuple[CapturedRequest, ...] report: SDKReport diff --git a/tests/route_parity/replay.py b/tests/route_parity/replay.py index d9eef1181a7..6cfa7ad76a3 100644 --- a/tests/route_parity/replay.py +++ b/tests/route_parity/replay.py @@ -1,5 +1,6 @@ from __future__ import annotations +import base64 import queue import threading from collections.abc import Generator @@ -9,6 +10,7 @@ from typing import Final from pydantic import JsonValue, TypeAdapter +from tests.route_parity.fixture_recorder import _local_response_header from tests.route_parity.models import CapturedRequest from tests.route_parity.recorded_http import RecordedHttpResponse, RecordedHttpStreamResponse, RecordedResponse @@ -41,11 +43,11 @@ class ReplayServer(ThreadingHTTPServer): def enqueue_response(self, response: RecordedResponse) -> None: self.responses.put(response) - def take_request(self) -> CapturedRequest: + def take_requests(self, expected_count: int) -> tuple[CapturedRequest, ...]: request_count: Final = self.requests.qsize() - if request_count != 1: - raise AssertionError(f"expected exactly one provider request, received {request_count}") - return self.requests.get_nowait() + if request_count != expected_count: + raise AssertionError(f"expected exactly {expected_count} provider requests, received {request_count}") + return tuple(self.requests.get_nowait() for _ in range(request_count)) def reset(self) -> None: while not self.responses.empty(): @@ -58,10 +60,24 @@ class _ReplayHandler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" def do_POST(self) -> None: + self._replay() + + def do_GET(self) -> None: + self._replay() + + def _replay(self) -> None: provider: Final = self.server assert isinstance(provider, ReplayServer) length: Final = int(self.headers.get("content-length") or "0") - body: Final = JSON_VALUE.validate_json(self.rfile.read(length)) + raw_body: Final = self.rfile.read(length) if length else b"" + content_type: Final = self.headers.get("content-type", "") + body: Final = ( + JSON_VALUE.validate_json(raw_body) + if raw_body and content_type.lower().startswith("application/json") + else base64.b64encode(raw_body).decode("ascii") + if raw_body + else None + ) headers: Final = tuple( sorted( (name.lower(), value) @@ -86,7 +102,7 @@ class _ReplayHandler(BaseHTTPRequestHandler): self.send_response_only(response.status_code) for header in response.headers: if header.name.lower() not in EXCLUDED_RESPONSE_HEADERS: - self.send_header(header.name, header.value) + self.send_header(header.name, _local_response_header(header.name, header.value, provider.url)) if isinstance(response, RecordedHttpResponse): response_body: Final = response.body_bytes() self.send_header("content-length", str(len(response_body))) diff --git a/tests/route_parity/runner.py b/tests/route_parity/runner.py index 35fdf1689e0..f5eb5a39eb2 100644 --- a/tests/route_parity/runner.py +++ b/tests/route_parity/runner.py @@ -73,12 +73,13 @@ class PythonScriptWorker: self, case_file: Path, route: str, - response: RecordedResponse, + responses: tuple[RecordedResponse, ...], ) -> Execution: stdin: Final = self.process.stdin if stdin is None or self.process.poll() is not None: raise AssertionError(f"{self.mode} {self.route_label} worker exited before processing {case_file}") - self.provider.enqueue_response(response) + for response in responses: + self.provider.enqueue_response(response) command: Final = SDKCommand(case_file=str(case_file), route=route) try: stdin.write(f"{command.model_dump_json()}\n") @@ -103,7 +104,7 @@ class PythonScriptWorker: ) assert isinstance(result, WorkerSuccess) try: - return Execution(request=self.provider.take_request(), report=result.report) + return Execution(requests=self.provider.take_requests(len(responses)), report=result.report) except AssertionError: self.provider.reset() raise @@ -160,9 +161,9 @@ def run_execution( worker: PythonScriptWorker, case_file: Path, route: str, - response: RecordedResponse, + responses: tuple[RecordedResponse, ...], ) -> Execution: - return worker.execute(case_file, route, response) + return worker.execute(case_file, route, responses) @contextmanager diff --git a/tests/route_parity/test_fixture_recorder.py b/tests/route_parity/test_fixture_recorder.py index 86a95e6a7ae..693592841a6 100644 --- a/tests/route_parity/test_fixture_recorder.py +++ b/tests/route_parity/test_fixture_recorder.py @@ -1,7 +1,7 @@ from __future__ import annotations import threading -from collections.abc import Generator +from collections.abc import Callable, Generator from contextlib import contextmanager from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path @@ -50,7 +50,7 @@ class _ParityCase(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") litellm_input: _FixtureInput - provider_response: RecordedResponse + provider_responses: tuple[RecordedResponse, ...] class _ControlledUpstream(ThreadingHTTPServer): @@ -90,6 +90,20 @@ class _ControlledUpstreamHandler(BaseHTTPRequestHandler): assert isinstance(upstream, _ControlledUpstream) length: Final = int(self.headers.get("content-length") or "0") self.rfile.read(length) + if self.path == "/upload": + self._send_json(200, b'{"file_id":"reducto://fixture.pdf"}') + return + if self.path == "/parse": + self._send_json(200, b'{"result":{"chunks":[]}}') + return + if self.path == "/analyze": + upstream: Final = self.server + assert isinstance(upstream, _ControlledUpstream) + self.send_response(202) + self.send_header("operation-location", f"{upstream.url}/results/1") + self.send_header("content-length", "0") + self.end_headers() + return if self.path == "/v1/chat/completions": with upstream.lock: upstream.request_count += 1 @@ -116,6 +130,19 @@ class _ControlledUpstreamHandler(BaseHTTPRequestHandler): finally: upstream.end_tracked_request() + def do_GET(self) -> None: + if self.path == "/results/1": + self._send_json(200, b'{"status":"succeeded","analyzeResult":{"pages":[]}}') + return + self.send_error(404) + + def _send_json(self, status: int, body: bytes) -> None: + self.send_response(status) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + def log_message(self, format: str, *args: object) -> None: return @@ -145,6 +172,22 @@ def _stream_sdk_call(api_base: str, case_input: _FixtureInput) -> object: return httpx.post(f"{api_base}/v1/chat/completions", content=b"{}", timeout=5) +def _multi_sdk_call(api_base: str, case_input: _FixtureInput) -> object: + upload: Final = httpx.post(f"{api_base}/upload", json={"document": case_input.identifier}, timeout=5) + upload.raise_for_status() + parsed: Final = httpx.post(f"{api_base}/parse", json={"input": upload.json()["file_id"]}, timeout=5) + parsed.raise_for_status() + return parsed + + +def _polling_sdk_call(api_base: str, case_input: _FixtureInput) -> object: + started: Final = httpx.post(f"{api_base}/analyze", json={"document": case_input.identifier}, timeout=5) + operation_location: Final = started.headers["operation-location"] + completed: Final = httpx.get(operation_location, timeout=5) + completed.raise_for_status() + return completed + + def test_generate_case_inputs_is_deterministic() -> None: strategy: Final = st.builds(_FixtureInput, identifier=st.integers().map(str)) @@ -192,7 +235,7 @@ def test_streaming_response_records_and_replays_chunks(tmp_path: Path) -> None: _ParityCase, ) - response: Final = result.case.provider_response + response: Final = result.case.provider_responses[0] assert isinstance(response, RecordedHttpStreamResponse) assert tuple(chunk.data_bytes() for chunk in response.chunks) == _SSE_CHUNKS @@ -200,7 +243,7 @@ def test_streaming_response_records_and_replays_chunks(tmp_path: Path) -> None: provider.enqueue_response(response) with httpx.stream("POST", f"{provider.url}/v1/chat/completions", json={}) as replayed: replayed_chunks: Final = tuple(replayed.iter_raw()) - provider.take_request() + provider.take_requests(1) assert replayed_chunks == _SSE_CHUNKS @@ -216,3 +259,27 @@ def test_stream_response_model_rejects_buffered_body() -> None: "body_b64": "", } ) + + +@pytest.mark.parametrize("sdk_call", (_multi_sdk_call, _polling_sdk_call)) +def test_multiple_provider_calls_record_and_replay_in_order( + tmp_path: Path, + sdk_call: Callable[[str, _FixtureInput], object], +) -> None: + with _controlled_upstream() as upstream: + result: Final = record_case( + ProviderSpec(upstream_base=upstream.url), + tmp_path, + _case(sdk_call.__name__), + sdk_call, + _ParityCase, + ) + + assert len(result.case.provider_responses) == 2 + with replay_server() as provider: + for response in result.case.provider_responses: + provider.enqueue_response(response) + sdk_call(provider.url, _case(sdk_call.__name__)) + requests: Final = provider.take_requests(2) + + assert len(requests) == 2 diff --git a/tests/route_parity/test_parity.py b/tests/route_parity/test_parity.py index c92b7bc0298..78277469424 100644 --- a/tests/route_parity/test_parity.py +++ b/tests/route_parity/test_parity.py @@ -25,12 +25,14 @@ class _DifferentResponse(BaseModel): def _execution(*, body: JsonValue = None, markdown: str = "same", user_agent: str | None = None) -> Execution: return Execution( - request=CapturedRequest( - method="POST", - path="/v1/test-route?mode=test", - headers=(("authorization", "Bearer test-key"), ("content-type", "application/json")), - body={"model": "test-model"} if body is None else body, - user_agent=user_agent, + requests=( + CapturedRequest( + method="POST", + path="/v1/test-route?mode=test", + headers=(("authorization", "Bearer test-key"), ("content-type", "application/json")), + body={"model": "test-model"} if body is None else body, + user_agent=user_agent, + ), ), report=SDKReport(response={"items": [{"text": markdown}], "model": "test-model"}), ) diff --git a/tests/test_litellm/ocr/conftest.py b/tests/test_litellm/ocr/conftest.py index 7a9f93ec72a..e724bdb4e31 100644 --- a/tests/test_litellm/ocr/conftest.py +++ b/tests/test_litellm/ocr/conftest.py @@ -27,7 +27,7 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: env_var=FIXTURE_DIR_ENV, default_directory=default_directory, regeneration_command=( - f"uv run python tests/test_litellm/ocr/generate_fixtures.py --fixture-dir {default_directory}" + f"uv run python -m tests.test_litellm.ocr.generate_fixtures --fixture-dir {default_directory}" ), id_builder=_fixture_id, ) diff --git a/tests/test_litellm/ocr/fixture_models.py b/tests/test_litellm/ocr/fixture_models.py index 4257d517663..95927c91ec0 100644 --- a/tests/test_litellm/ocr/fixture_models.py +++ b/tests/test_litellm/ocr/fixture_models.py @@ -16,7 +16,13 @@ from tests.route_parity.fixture_models import ( SdkInputBase, ) -__all__ = ("JsonSchemaDefinition", "JsonSchemaResponseFormat", "OcrParityCase", "OcrSdkInputBase") +__all__ = ( + "JsonSchemaDefinition", + "JsonSchemaResponseFormat", + "OcrParityCase", + "OcrSdkInput", + "OcrSdkInputBase", +) OcrSdkInputBase = SdkInputBase @@ -57,10 +63,8 @@ MistralModel = Literal[ ] -class MistralOcrSdkInput(OcrSdkInputBase): - model: MistralModel +class MistralCompatibleOcrSdkInput(OcrSdkInputBase): document: MistralDocument - custom_llm_provider: Literal["mistral"] | None = None pages: str | list[int] | None = None include_image_base64: bool | None = None image_limit: int | None = None @@ -76,14 +80,74 @@ class MistralOcrSdkInput(OcrSdkInputBase): id: str | None = None @model_validator(mode="after") - def validate_provider_routing(self) -> Self: - if not self.model.startswith("mistral/") and self.custom_llm_provider != "mistral": - raise ValueError("unqualified Mistral models require custom_llm_provider='mistral'") + def validate_annotation_prompt(self) -> Self: if self.document_annotation_prompt is not None and self.document_annotation_format is None: raise ValueError("document_annotation_prompt requires document_annotation_format") return self +class MistralOcrSdkInput(MistralCompatibleOcrSdkInput): + boundary: Literal["mistral"] = "mistral" + model: MistralModel + custom_llm_provider: Literal["mistral"] | None = None + + @model_validator(mode="after") + def validate_provider_routing(self) -> Self: + if not self.model.startswith("mistral/") and self.custom_llm_provider != "mistral": + raise ValueError("unqualified Mistral models require custom_llm_provider='mistral'") + return self + + +class AzureMistralOcrSdkInput(MistralCompatibleOcrSdkInput): + boundary: Literal["azure_mistral"] = "azure_mistral" + model: str + custom_llm_provider: Literal["azure_ai"] | None = None + + @field_validator("model") + @classmethod + def validate_model_namespace(cls, model: str) -> str: + if not model.startswith("azure_ai/"): + raise ValueError("Azure Mistral models must use the azure_ai/ LiteLLM namespace") + return model + + +class VertexMistralOcrSdkInput(MistralCompatibleOcrSdkInput): + boundary: Literal["vertex_mistral"] = "vertex_mistral" + model: Literal["vertex_ai/mistral-ocr-2505"] = "vertex_ai/mistral-ocr-2505" + custom_llm_provider: Literal["vertex_ai"] | None = None + vertex_project: str + vertex_location: str = "us-central1" + + +class AzureDocumentIntelligenceOcrSdkInput(OcrSdkInputBase): + boundary: Literal["azure_document_intelligence"] = "azure_document_intelligence" + model: Literal[ + "azure_ai/doc-intelligence/prebuilt-read", + "azure_ai/doc-intelligence/prebuilt-layout", + "azure_ai/doc-intelligence/prebuilt-document", + ] + document: MistralDocument + custom_llm_provider: Literal["azure_ai"] | None = None + pages: str | list[int] | None = None + features: str | list[str] | None = None + req_format: Literal["litellm"] = "litellm" + + +class VertexDeepSeekOcrSdkInput(OcrSdkInputBase): + boundary: Literal["vertex_deepseek"] = "vertex_deepseek" + model: Literal["vertex_ai/deepseek-ai/deepseek-ocr-maas"] = "vertex_ai/deepseek-ai/deepseek-ocr-maas" + document: MistralDocument + custom_llm_provider: Literal["vertex_ai"] | None = None + vertex_project: str + vertex_location: str = "us-central1" + stream: bool | None = None + temperature: float | None = None + max_tokens: int | None = None + top_p: float | None = None + n: int | None = None + stop: str | list[str] | None = None + + def _validate_reducto_source(source: str) -> str: if source.startswith("reducto://"): return source @@ -243,6 +307,7 @@ class ReductoSettings(FixtureModel): class ReductoParseV3SdkInput(OcrSdkInputBase): + boundary: Literal["reducto_v3"] = "reducto_v3" model: Literal["reducto/parse-v3", "parse-v3"] document: ReductoDocument custom_llm_provider: Literal["reducto"] | None = None @@ -258,6 +323,7 @@ class ReductoParseV3SdkInput(OcrSdkInputBase): class ReductoParseLegacySdkInput(OcrSdkInputBase): + boundary: Literal["reducto_legacy"] = "reducto_legacy" model: Literal["reducto/parse-legacy", "parse-legacy"] document: ReductoDocument custom_llm_provider: Literal["reducto"] | None = None @@ -270,5 +336,17 @@ class ReductoParseLegacySdkInput(OcrSdkInputBase): return self -class OcrParityCase(ParityCase[MistralOcrSdkInput]): +OcrSdkInput = Annotated[ + MistralOcrSdkInput + | AzureMistralOcrSdkInput + | VertexMistralOcrSdkInput + | AzureDocumentIntelligenceOcrSdkInput + | VertexDeepSeekOcrSdkInput + | ReductoParseV3SdkInput + | ReductoParseLegacySdkInput, + Field(discriminator="boundary"), +] + + +class OcrParityCase(ParityCase[OcrSdkInput]): pass diff --git a/tests/test_litellm/ocr/fixtures/067fe064938269ae58ac14030064a77b92d76edfd00e485fca93a0efbb82bda3.json b/tests/test_litellm/ocr/fixtures/067fe064938269ae58ac14030064a77b92d76edfd00e485fca93a0efbb82bda3.json deleted file mode 100644 index 7c6168d48a7..00000000000 --- a/tests/test_litellm/ocr/fixtures/067fe064938269ae58ac14030064a77b92d76edfd00e485fca93a0efbb82bda3.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "case": { - "litellm_input": { - "document": { - "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24", - "type": "image_url" - }, - "id": "case-1", - "model": "mistral/mistral-ocr-latest" - }, - "provider_response": { - "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsImJsb2NrcyI6W3sidG9wX2xlZnRfeCI6MzM1LCJ0b3BfbGVmdF95IjoxMzgsImJvdHRvbV9yaWdodF94Ijo0NjQsImJvdHRvbV9yaWdodF95IjoxNjIsImNvbnRlbnQiOiJpbnZvaWNlIDEyMyIsImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJ0eXBlIjoidGV4dCJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOm51bGwsInVzYWdlX2luZm8iOnsicGFnZXNfcHJvY2Vzc2VkIjoxLCJkb2Nfc2l6ZV9ieXRlcyI6NDEyNH19", - "headers": [ - { - "name": "Date", - "value": "Sat, 29 Aug 2026 23:18:02 GMT" - }, - { - "name": "Content-Type", - "value": "application/json" - }, - { - "name": "set-cookie", - "value": "__cf_bm=xhZ4oMdiKbThFP71tBxvQtwQRHpnzzS31o6bhmCrX_o-1788045481.8243725-1.0.1.1-ju2kNNBEq.Ff7jRRXRNgRw7jODlxu2bjMlA9ni_JBOIo6Mw.5QKyVVN8wN38J.gLItzZCZAKXFSCWAqVM39oZ9NMBCNTuwXEbf6bM5NIOCn0LgE2DrvxielM2VJkpDan; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Sat, 29 Aug 2026 23:48:02 GMT" - }, - { - "name": "mistral-correlation-id", - "value": "01a04fd0-d7b3-7279-a78d-c48ecdcde419" - }, - { - "name": "x-kong-request-id", - "value": "01a04fd0-d7b3-7279-a78d-c48ecdcde419" - }, - { - "name": "x-ratelimit-limit-ocr-pages-minute", - "value": "60" - }, - { - "name": "x-ratelimit-remaining-ocr-pages-minute", - "value": "59" - }, - { - "name": "x-ratelimit-ocr-pages-query-cost", - "value": "1" - }, - { - "name": "x-envoy-upstream-service-time", - "value": "379" - }, - { - "name": "Server", - "value": "cloudflare" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "x-kong-upstream-latency", - "value": "380" - }, - { - "name": "x-kong-proxy-latency", - "value": "12" - }, - { - "name": "X-Content-Type-Options", - "value": "nosniff" - }, - { - "name": "cf-cache-status", - "value": "DYNAMIC" - }, - { - "name": "Strict-Transport-Security", - "value": "max-age=15552000; includeSubDomains; preload" - }, - { - "name": "CF-RAY", - "value": "a32f45c56be31975-SJC" - }, - { - "name": "alt-svc", - "value": "h3=\":443\"; ma=86400" - } - ], - "kind": "http", - "status_code": 200 - } - }, - "recorded_at": "2026-08-29T23:18:02.736046Z", - "schema_version": 1 -} diff --git a/tests/test_litellm/ocr/fixtures/d7a77e2b36580d60ba22ef0a9cd910adf782ab9194e309d3c62fe0f2fadc326c.json b/tests/test_litellm/ocr/fixtures/d7a77e2b36580d60ba22ef0a9cd910adf782ab9194e309d3c62fe0f2fadc326c.json deleted file mode 100644 index 4abf13f66ce..00000000000 --- a/tests/test_litellm/ocr/fixtures/d7a77e2b36580d60ba22ef0a9cd910adf782ab9194e309d3c62fe0f2fadc326c.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "case": { - "litellm_input": { - "document": { - "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24", - "type": "image_url" - }, - "model": "mistral/mistral-ocr-latest" - }, - "provider_response": { - "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsImJsb2NrcyI6W3sidG9wX2xlZnRfeCI6MzM1LCJ0b3BfbGVmdF95IjoxMzgsImJvdHRvbV9yaWdodF94Ijo0NjQsImJvdHRvbV9yaWdodF95IjoxNjIsImNvbnRlbnQiOiJpbnZvaWNlIDEyMyIsImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJ0eXBlIjoidGV4dCJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOm51bGwsInVzYWdlX2luZm8iOnsicGFnZXNfcHJvY2Vzc2VkIjoxLCJkb2Nfc2l6ZV9ieXRlcyI6NDEyNH19", - "headers": [ - { - "name": "Date", - "value": "Sat, 29 Aug 2026 23:18:02 GMT" - }, - { - "name": "Content-Type", - "value": "application/json" - }, - { - "name": "set-cookie", - "value": "__cf_bm=4Y4BNG5seFc4759BsUBjy4Z2odUy3Rgf0SOYkyESdPA-1788045481.8373353-1.0.1.1-eirQC2neaOT8r6_9qxcWeUjOMH7L1NmsXj2YyMbcVmTO62TWrDCjf7VqxKigiJ1lViQQkJQwVpMsSHTfUOaziX4zDTloRi4b.oaA37lqE.gBbH9FylA_nMTl2mYg65X9; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Sat, 29 Aug 2026 23:48:02 GMT" - }, - { - "name": "mistral-correlation-id", - "value": "01a04fd0-d7b8-799c-9ac2-48ab880605d6" - }, - { - "name": "x-kong-request-id", - "value": "01a04fd0-d7b8-799c-9ac2-48ab880605d6" - }, - { - "name": "x-ratelimit-limit-ocr-pages-minute", - "value": "60" - }, - { - "name": "x-ratelimit-remaining-ocr-pages-minute", - "value": "59" - }, - { - "name": "x-ratelimit-ocr-pages-query-cost", - "value": "1" - }, - { - "name": "x-envoy-upstream-service-time", - "value": "355" - }, - { - "name": "Server", - "value": "cloudflare" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "x-kong-upstream-latency", - "value": "356" - }, - { - "name": "x-kong-proxy-latency", - "value": "17" - }, - { - "name": "X-Content-Type-Options", - "value": "nosniff" - }, - { - "name": "cf-cache-status", - "value": "DYNAMIC" - }, - { - "name": "Strict-Transport-Security", - "value": "max-age=15552000; includeSubDomains; preload" - }, - { - "name": "CF-RAY", - "value": "a32f45c57f8e7834-SJC" - }, - { - "name": "alt-svc", - "value": "h3=\":443\"; ma=86400" - } - ], - "kind": "http", - "status_code": 200 - } - }, - "recorded_at": "2026-08-29T23:18:02.735198Z", - "schema_version": 1 -} diff --git a/tests/test_litellm/ocr/fixtures/mistral-ocr/0ab748e6b7315f0c7a36195cc4564eff8b03523dfa908ea59ff1bb7bd185ede8.json b/tests/test_litellm/ocr/fixtures/mistral-ocr/0ab748e6b7315f0c7a36195cc4564eff8b03523dfa908ea59ff1bb7bd185ede8.json new file mode 100644 index 00000000000..ee8b1734500 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/mistral-ocr/0ab748e6b7315f0c7a36195cc4564eff8b03523dfa908ea59ff1bb7bd185ede8.json @@ -0,0 +1,142 @@ +{ + "case": { + "litellm_input": { + "bbox_annotation_format": { + "json_schema": { + "description": "Extract the visible document fields", + "name": "bounding_boxes", + "schema": { + "additionalProperties": false, + "properties": { + "title": { + "type": "string" + } + }, + "required": [ + "title" + ], + "type": "object" + }, + "strict": true + }, + "type": "json_schema" + }, + "document": { + "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24", + "type": "image_url" + }, + "document_annotation_format": { + "json_schema": { + "description": "Extract the visible document fields", + "name": "document_title", + "schema": { + "additionalProperties": false, + "properties": { + "title": { + "type": "string" + } + }, + "required": [ + "title" + ], + "type": "object" + }, + "strict": true + }, + "type": "json_schema" + }, + "extract_header": true, + "image_min_size": 300, + "include_blocks": true, + "include_image_base64": false, + "model": "mistral/mistral-ocr-latest", + "pages": [ + 0 + ], + "table_format": "markdown" + }, + "provider_responses": [ + { + "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsImJsb2NrcyI6W3sidG9wX2xlZnRfeCI6MzM1LCJ0b3BfbGVmdF95IjoxMzgsImJvdHRvbV9yaWdodF94Ijo0NjQsImJvdHRvbV9yaWdodF95IjoxNjIsImNvbnRlbnQiOiJpbnZvaWNlIDEyMyIsImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJ0eXBlIjoidGV4dCJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOiJ7XCJ0aXRsZVwiOiBcIkludm9pY2VfMTIzXCJ9IiwidXNhZ2VfaW5mbyI6eyJwYWdlc19wcm9jZXNzZWQiOjEsImRvY19zaXplX2J5dGVzIjo0MTI0fX0=", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:26 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "set-cookie", + "value": "__cf_bm=c5QAleICyoEptgycuGlRuItpEqcNe2MRZASOE9DeW80-1788292466.0205042-1.0.1.1-_PjnHuYd9Ba1XjxpUtrCWo_gUI5uKAvA73jcZAj6t_RARG9jiuX8N3t6bXu7K15RP3WG6AYFN4hbSKwlEpfksa9krmM7JYpf_JEpS5GDbst5vpFFLNtv.ymdX8lDSb5a; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:26 GMT" + }, + { + "name": "mistral-correlation-id", + "value": "01a05e89-85bd-7075-af53-8e341b35a63e" + }, + { + "name": "x-kong-request-id", + "value": "01a05e89-85bd-7075-af53-8e341b35a63e" + }, + { + "name": "x-ratelimit-limit-ocr-pages-minute", + "value": "60" + }, + { + "name": "x-ratelimit-remaining-ocr-pages-minute", + "value": "43" + }, + { + "name": "x-ratelimit-ocr-pages-query-cost", + "value": "1" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "432" + }, + { + "name": "Server", + "value": "cloudflare" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "x-kong-upstream-latency", + "value": "433" + }, + { + "name": "x-kong-proxy-latency", + "value": "13" + }, + { + "name": "X-Content-Type-Options", + "value": "nosniff" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "name": "Strict-Transport-Security", + "value": "max-age=15552000; includeSubDomains; preload" + }, + { + "name": "CF-RAY", + "value": "a346d3a89a85f953-SJC" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:26.965078Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/mistral-ocr/0c9204f5ed668c628835b9d60806a6cea47e1f67c0f48c9804cb09d1c4695f42.json b/tests/test_litellm/ocr/fixtures/mistral-ocr/0c9204f5ed668c628835b9d60806a6cea47e1f67c0f48c9804cb09d1c4695f42.json new file mode 100644 index 00000000000..ab71aaa4507 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/mistral-ocr/0c9204f5ed668c628835b9d60806a6cea47e1f67c0f48c9804cb09d1c4695f42.json @@ -0,0 +1,95 @@ +{ + "case": { + "litellm_input": { + "document": { + "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24", + "type": "image_url" + }, + "extract_footer": true, + "model": "mistral/mistral-ocr-latest" + }, + "provider_responses": [ + { + "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsImJsb2NrcyI6W3sidG9wX2xlZnRfeCI6MzM1LCJ0b3BfbGVmdF95IjoxMzgsImJvdHRvbV9yaWdodF94Ijo0NjQsImJvdHRvbV9yaWdodF95IjoxNjIsImNvbnRlbnQiOiJpbnZvaWNlIDEyMyIsImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJ0eXBlIjoidGV4dCJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOm51bGwsInVzYWdlX2luZm8iOnsicGFnZXNfcHJvY2Vzc2VkIjoxLCJkb2Nfc2l6ZV9ieXRlcyI6NDEyNH19", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:20 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "mistral-correlation-id", + "value": "01a05e89-6c42-7ea1-b08b-07cae50734e0" + }, + { + "name": "x-kong-request-id", + "value": "01a05e89-6c42-7ea1-b08b-07cae50734e0" + }, + { + "name": "x-ratelimit-limit-ocr-pages-minute", + "value": "60" + }, + { + "name": "x-ratelimit-remaining-ocr-pages-minute", + "value": "50" + }, + { + "name": "x-ratelimit-ocr-pages-query-cost", + "value": "1" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "523" + }, + { + "name": "Server", + "value": "cloudflare" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "x-kong-upstream-latency", + "value": "525" + }, + { + "name": "x-kong-proxy-latency", + "value": "13" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "name": "set-cookie", + "value": "__cf_bm=guezPmBNpI0djDo3Kvw0miZywJszLF5AN8loY9Uma0E-1788292459.497073-1.0.1.1-meRuKWGaBsHhkiA58SnRIUHMOaMgSPVzPrmAuu4_xq4UqYETqJn06.dJRx2xrVcoseTuUZNKdyRxTbkz2pdMRMe2AMQWaAMXXW27bG1VkapsA2hW1KtB48F3ubpGbp0v; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:20 GMT" + }, + { + "name": "Strict-Transport-Security", + "value": "max-age=15552000; includeSubDomains; preload" + }, + { + "name": "X-Content-Type-Options", + "value": "nosniff" + }, + { + "name": "CF-RAY", + "value": "a346d37fde0f1703-SJC" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:20.425444Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/mistral-ocr/216aacc1aa47a6060445f2ea49a2e0a8f4a8bde179faaaa28f39b049e7886f07.json b/tests/test_litellm/ocr/fixtures/mistral-ocr/216aacc1aa47a6060445f2ea49a2e0a8f4a8bde179faaaa28f39b049e7886f07.json new file mode 100644 index 00000000000..ef889c69c0b --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/mistral-ocr/216aacc1aa47a6060445f2ea49a2e0a8f4a8bde179faaaa28f39b049e7886f07.json @@ -0,0 +1,95 @@ +{ + "case": { + "litellm_input": { + "document": { + "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24", + "type": "image_url" + }, + "model": "mistral/mistral-ocr-latest", + "table_format": "markdown" + }, + "provider_responses": [ + { + "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsImJsb2NrcyI6W3sidG9wX2xlZnRfeCI6MzM1LCJ0b3BfbGVmdF95IjoxMzgsImJvdHRvbV9yaWdodF94Ijo0NjQsImJvdHRvbV9yaWdodF95IjoxNjIsImNvbnRlbnQiOiJpbnZvaWNlIDEyMyIsImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJ0eXBlIjoidGV4dCJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOm51bGwsInVzYWdlX2luZm8iOnsicGFnZXNfcHJvY2Vzc2VkIjoxLCJkb2Nfc2l6ZV9ieXRlcyI6NDEyNH19", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:21 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "set-cookie", + "value": "__cf_bm=XcXH8M77UDLCt6rebhVaPfmyZAa4b4OQ6qSuzN8zcxQ-1788292460.477192-1.0.1.1-jIqoIYlVPq1KWC06tEhpWZ2IGqr1TQU1jPokZk1wkK2UVb0L0LVboaLWA9UnCvYLvy5JzgUf1srFJQlzr7edR8HiWMaxsHfRsCVCPVlCr3H6S_Pikt3AqQmlW8xWM9vh; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:21 GMT" + }, + { + "name": "mistral-correlation-id", + "value": "01a05e89-7015-7a8d-a18c-37b9ceacbe77" + }, + { + "name": "x-kong-request-id", + "value": "01a05e89-7015-7a8d-a18c-37b9ceacbe77" + }, + { + "name": "x-ratelimit-limit-ocr-pages-minute", + "value": "60" + }, + { + "name": "x-ratelimit-remaining-ocr-pages-minute", + "value": "49" + }, + { + "name": "x-ratelimit-ocr-pages-query-cost", + "value": "1" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "337" + }, + { + "name": "Server", + "value": "cloudflare" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "x-kong-upstream-latency", + "value": "338" + }, + { + "name": "x-kong-proxy-latency", + "value": "21" + }, + { + "name": "X-Content-Type-Options", + "value": "nosniff" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "name": "Strict-Transport-Security", + "value": "max-age=15552000; includeSubDomains; preload" + }, + { + "name": "CF-RAY", + "value": "a346d385ffbda0f2-SJC" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:21.430472Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/mistral-ocr/2c893c555fe38cdfa0f03b43b5e678b8acd5c2dc460cd06f5302a0bf05f3089c.json b/tests/test_litellm/ocr/fixtures/mistral-ocr/2c893c555fe38cdfa0f03b43b5e678b8acd5c2dc460cd06f5302a0bf05f3089c.json new file mode 100644 index 00000000000..9078a82ffcb --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/mistral-ocr/2c893c555fe38cdfa0f03b43b5e678b8acd5c2dc460cd06f5302a0bf05f3089c.json @@ -0,0 +1,95 @@ +{ + "case": { + "litellm_input": { + "document": { + "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24", + "type": "image_url" + }, + "image_limit": 1, + "model": "mistral/mistral-ocr-latest" + }, + "provider_responses": [ + { + "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsImJsb2NrcyI6W3sidG9wX2xlZnRfeCI6MzM1LCJ0b3BfbGVmdF95IjoxMzgsImJvdHRvbV9yaWdodF94Ijo0NjQsImJvdHRvbV9yaWdodF95IjoxNjIsImNvbnRlbnQiOiJpbnZvaWNlIDEyMyIsImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJ0eXBlIjoidGV4dCJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOm51bGwsInVzYWdlX2luZm8iOnsicGFnZXNfcHJvY2Vzc2VkIjoxLCJkb2Nfc2l6ZV9ieXRlcyI6NDEyNH19", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:15 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "set-cookie", + "value": "__cf_bm=zdJ57Tl12b3GlmPJbN9IJjqlrvpX6J37Pmt6.Jj4vEs-1788292454.9590237-1.0.1.1-KS5i0m8mBhan9C3JxsJ488KsWHHpjYMz7ASc2SsaJ9MdFLx6JQWyViWZcw3ohhK1y_8a7DwExnKFQEIkFvF8J6HRVhoD2qaYjFW7DHz3t.2DBZunmBgvOouYMcjJyi7X; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:15 GMT" + }, + { + "name": "mistral-correlation-id", + "value": "01a05e89-5a87-7148-8ae4-4e14967bebf3" + }, + { + "name": "x-kong-request-id", + "value": "01a05e89-5a87-7148-8ae4-4e14967bebf3" + }, + { + "name": "x-ratelimit-limit-ocr-pages-minute", + "value": "60" + }, + { + "name": "x-ratelimit-remaining-ocr-pages-minute", + "value": "56" + }, + { + "name": "x-ratelimit-ocr-pages-query-cost", + "value": "1" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "226" + }, + { + "name": "Server", + "value": "cloudflare" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "x-kong-upstream-latency", + "value": "227" + }, + { + "name": "x-kong-proxy-latency", + "value": "19" + }, + { + "name": "X-Content-Type-Options", + "value": "nosniff" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "name": "Strict-Transport-Security", + "value": "max-age=15552000; includeSubDomains; preload" + }, + { + "name": "CF-RAY", + "value": "a346d3637dc2c090-SJC" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:15.394028Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/mistral-ocr/3e4f8a006942b85eb954833d42cc19740883cf42bb87221eafd16df3e597ceda.json b/tests/test_litellm/ocr/fixtures/mistral-ocr/3e4f8a006942b85eb954833d42cc19740883cf42bb87221eafd16df3e597ceda.json new file mode 100644 index 00000000000..54cf3a8e443 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/mistral-ocr/3e4f8a006942b85eb954833d42cc19740883cf42bb87221eafd16df3e597ceda.json @@ -0,0 +1,114 @@ +{ + "case": { + "litellm_input": { + "bbox_annotation_format": { + "json_schema": { + "description": "Extract the visible document fields", + "name": "bounding_boxes", + "schema": { + "additionalProperties": false, + "properties": { + "title": { + "type": "string" + } + }, + "required": [ + "title" + ], + "type": "object" + }, + "strict": true + }, + "type": "json_schema" + }, + "document": { + "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24", + "type": "image_url" + }, + "model": "mistral/mistral-ocr-latest" + }, + "provider_responses": [ + { + "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsImJsb2NrcyI6W3sidG9wX2xlZnRfeCI6MzM1LCJ0b3BfbGVmdF95IjoxMzgsImJvdHRvbV9yaWdodF94Ijo0NjQsImJvdHRvbV9yaWdodF95IjoxNjIsImNvbnRlbnQiOiJpbnZvaWNlIDEyMyIsImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJ0eXBlIjoidGV4dCJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOm51bGwsInVzYWdlX2luZm8iOnsicGFnZXNfcHJvY2Vzc2VkIjoxLCJkb2Nfc2l6ZV9ieXRlcyI6NDEyNH19", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:16 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "set-cookie", + "value": "__cf_bm=VjsfRKiwC7YjbVgfHbbpQfMBCW9MC1ehUXU3LsRZeUY-1788292455.9575524-1.0.1.1-KL3St9l4QfN1eWLUV07XDYmwE0xr2.26Z85yKf1yBMTDFCHBnz_3_RP.oNw6p.GfJsd2mnKb2lOvgPQcBDZygv7yJhN41wN87pIkAvurdr1z_snzwBpofXTvcjibOV1K; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:16 GMT" + }, + { + "name": "mistral-correlation-id", + "value": "01a05e89-5e6e-70ac-890d-c0110a5e81af" + }, + { + "name": "x-kong-request-id", + "value": "01a05e89-5e6e-70ac-890d-c0110a5e81af" + }, + { + "name": "x-ratelimit-limit-ocr-pages-minute", + "value": "60" + }, + { + "name": "x-ratelimit-remaining-ocr-pages-minute", + "value": "54" + }, + { + "name": "x-ratelimit-ocr-pages-query-cost", + "value": "1" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "210" + }, + { + "name": "Server", + "value": "cloudflare" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "x-kong-upstream-latency", + "value": "212" + }, + { + "name": "x-kong-proxy-latency", + "value": "17" + }, + { + "name": "X-Content-Type-Options", + "value": "nosniff" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "name": "Strict-Transport-Security", + "value": "max-age=15552000; includeSubDomains; preload" + }, + { + "name": "CF-RAY", + "value": "a346d369be838fc5-SJC" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:16.402593Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/mistral-ocr/49328aed7d9a4833427d80c7ac4f54fc03ab0db9076004caf5262ce3fed4f06b.json b/tests/test_litellm/ocr/fixtures/mistral-ocr/49328aed7d9a4833427d80c7ac4f54fc03ab0db9076004caf5262ce3fed4f06b.json new file mode 100644 index 00000000000..edf15aa1839 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/mistral-ocr/49328aed7d9a4833427d80c7ac4f54fc03ab0db9076004caf5262ce3fed4f06b.json @@ -0,0 +1,94 @@ +{ + "case": { + "litellm_input": { + "document": { + "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24", + "type": "image_url" + }, + "model": "mistral/mistral-ocr-latest" + }, + "provider_responses": [ + { + "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsImJsb2NrcyI6W3sidG9wX2xlZnRfeCI6MzM1LCJ0b3BfbGVmdF95IjoxMzgsImJvdHRvbV9yaWdodF94Ijo0NjQsImJvdHRvbV9yaWdodF95IjoxNjIsImNvbnRlbnQiOiJpbnZvaWNlIDEyMyIsImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJ0eXBlIjoidGV4dCJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOm51bGwsInVzYWdlX2luZm8iOnsicGFnZXNfcHJvY2Vzc2VkIjoxLCJkb2Nfc2l6ZV9ieXRlcyI6NDEyNH19", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:13 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "set-cookie", + "value": "__cf_bm=0BJujMXEX2uflOAjZGqeTJi9lXNgGJmX61YbHtpbRKI-1788292452.9600976-1.0.1.1-7idLuOz.nd3kpO10lpC8IkPMArZ8M9iDs13Z.kTwKLTBHVm4.lyUeSqedg4EQV_Mo3D6JLf6TJgvdhWHsemZe6NyIzTBQwA3cwyM0cMQ5Aw68sVgtDOuGBaqhtelZ0RW; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:13 GMT" + }, + { + "name": "mistral-correlation-id", + "value": "01a05e89-52b4-7e7b-983e-5bc4e5469b39" + }, + { + "name": "x-kong-request-id", + "value": "01a05e89-52b4-7e7b-983e-5bc4e5469b39" + }, + { + "name": "x-ratelimit-limit-ocr-pages-minute", + "value": "60" + }, + { + "name": "x-ratelimit-remaining-ocr-pages-minute", + "value": "59" + }, + { + "name": "x-ratelimit-ocr-pages-query-cost", + "value": "1" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "554" + }, + { + "name": "Server", + "value": "cloudflare" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "x-kong-upstream-latency", + "value": "557" + }, + { + "name": "x-kong-proxy-latency", + "value": "13" + }, + { + "name": "X-Content-Type-Options", + "value": "nosniff" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "name": "Strict-Transport-Security", + "value": "max-age=15552000; includeSubDomains; preload" + }, + { + "name": "CF-RAY", + "value": "a346d356fb3698ce-SJC" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:13.880852Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/mistral-ocr/67a074c43483176b6a8da74f21869eb5c13a7327ed36f77a40c0b192c3a1bf4f.json b/tests/test_litellm/ocr/fixtures/mistral-ocr/67a074c43483176b6a8da74f21869eb5c13a7327ed36f77a40c0b192c3a1bf4f.json new file mode 100644 index 00000000000..9c7162f0ff6 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/mistral-ocr/67a074c43483176b6a8da74f21869eb5c13a7327ed36f77a40c0b192c3a1bf4f.json @@ -0,0 +1,126 @@ +{ + "case": { + "litellm_input": { + "bbox_annotation_format": { + "json_schema": { + "description": "Extract the visible document fields", + "name": "bounding_boxes", + "schema": { + "additionalProperties": false, + "properties": { + "title": { + "type": "string" + } + }, + "required": [ + "title" + ], + "type": "object" + }, + "strict": true + }, + "type": "json_schema" + }, + "confidence_scores_granularity": "page", + "document": { + "document_url": "data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg==", + "type": "document_url" + }, + "extract_footer": false, + "extract_header": true, + "id": "case-1", + "image_limit": 1, + "image_min_size": 300, + "include_blocks": false, + "include_image_base64": true, + "model": "mistral/mistral-ocr-latest", + "pages": [ + 0 + ], + "table_format": "markdown" + }, + "provider_responses": [ + { + "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiVGVzdCBQREYgRmlsZSIsImltYWdlcyI6W10sInRhYmxlcyI6W10sImh5cGVybGlua3MiOltdLCJoZWFkZXIiOm51bGwsImZvb3RlciI6bnVsbCwiZGltZW5zaW9ucyI6eyJkcGkiOjkzLCJoZWlnaHQiOjEwMjMsIndpZHRoIjo3OTF9LCJjb25maWRlbmNlX3Njb3JlcyI6eyJ3b3JkX2NvbmZpZGVuY2Vfc2NvcmVzIjpbXSwiYXZlcmFnZV9wYWdlX2NvbmZpZGVuY2Vfc2NvcmUiOjAuOTM3NjIyOTc0NDMyMjkzNiwibWluaW11bV9wYWdlX2NvbmZpZGVuY2Vfc2NvcmUiOjAuMjI1OTA1NTA3OTYwMzY4MzV9LCJibG9ja3MiOm51bGx9XSwibW9kZWwiOiJtaXN0cmFsLW9jci1sYXRlc3QiLCJkb2N1bWVudF9hbm5vdGF0aW9uIjpudWxsLCJ1c2FnZV9pbmZvIjp7InBhZ2VzX3Byb2Nlc3NlZCI6MSwiZG9jX3NpemVfYnl0ZXMiOjU4OX19", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:24 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "set-cookie", + "value": "__cf_bm=sPN0c81343UATX4OL40rLiXwZzjj6.f9wiR.sUEwidI-1788292464.0167594-1.0.1.1-1I1p0tYuzSZefNGWXeuwPPCH2oi83ZL7wHVVU_skvYhidQGDYesdwH32zscGrLGoxF1LPkg9h4yruaTw9Pai6ahKkyPofBTXU4pjatWxIYJOV8ILJQRwjJAY9r5x01g5; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:24 GMT" + }, + { + "name": "mistral-correlation-id", + "value": "01a05e89-7deb-72b4-976d-4c244b056782" + }, + { + "name": "x-kong-request-id", + "value": "01a05e89-7deb-72b4-976d-4c244b056782" + }, + { + "name": "x-ratelimit-limit-ocr-pages-minute", + "value": "60" + }, + { + "name": "x-ratelimit-remaining-ocr-pages-minute", + "value": "45" + }, + { + "name": "x-ratelimit-ocr-pages-query-cost", + "value": "1" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "373" + }, + { + "name": "Server", + "value": "cloudflare" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "x-kong-upstream-latency", + "value": "373" + }, + { + "name": "x-kong-proxy-latency", + "value": "17" + }, + { + "name": "X-Content-Type-Options", + "value": "nosniff" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "name": "Strict-Transport-Security", + "value": "max-age=15552000; includeSubDomains; preload" + }, + { + "name": "CF-RAY", + "value": "a346d39c1fe5cf12-SJC" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:24.951956Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/mistral-ocr/9f220bfe8a2042fa74e1dada33facf8c80e20574f6ef9782997eaef9adc10e0b.json b/tests/test_litellm/ocr/fixtures/mistral-ocr/9f220bfe8a2042fa74e1dada33facf8c80e20574f6ef9782997eaef9adc10e0b.json new file mode 100644 index 00000000000..82a0edac1d9 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/mistral-ocr/9f220bfe8a2042fa74e1dada33facf8c80e20574f6ef9782997eaef9adc10e0b.json @@ -0,0 +1,95 @@ +{ + "case": { + "litellm_input": { + "document": { + "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24", + "type": "image_url" + }, + "image_min_size": 300, + "model": "mistral/mistral-ocr-latest" + }, + "provider_responses": [ + { + "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsImJsb2NrcyI6W3sidG9wX2xlZnRfeCI6MzM1LCJ0b3BfbGVmdF95IjoxMzgsImJvdHRvbV9yaWdodF94Ijo0NjQsImJvdHRvbV9yaWdodF95IjoxNjIsImNvbnRlbnQiOiJpbnZvaWNlIDEyMyIsImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJ0eXBlIjoidGV4dCJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOm51bGwsInVzYWdlX2luZm8iOnsicGFnZXNfcHJvY2Vzc2VkIjoxLCJkb2Nfc2l6ZV9ieXRlcyI6NDEyNH19", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:15 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "mistral-correlation-id", + "value": "01a05e89-5c60-769b-abf3-44dcefdf5282" + }, + { + "name": "x-kong-request-id", + "value": "01a05e89-5c60-769b-abf3-44dcefdf5282" + }, + { + "name": "x-ratelimit-limit-ocr-pages-minute", + "value": "60" + }, + { + "name": "x-ratelimit-remaining-ocr-pages-minute", + "value": "55" + }, + { + "name": "x-ratelimit-ocr-pages-query-cost", + "value": "1" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "272" + }, + { + "name": "Server", + "value": "cloudflare" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "x-kong-upstream-latency", + "value": "273" + }, + { + "name": "x-kong-proxy-latency", + "value": "17" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "name": "set-cookie", + "value": "__cf_bm=Ld7W50uqSPv414iFBL3xHBo1US6FgEGi176rm3Ir7jU-1788292455.4420552-1.0.1.1-U1t1LZ3WH0stmljgl3EZ7Q.vhOhxNGrc3Zj75kgYaDNZCxuwaMc55Khvf.CeGempT.Y5ajCYMNsBwGv0h3rJ4PtpKYxp6Jt16enjPTx6jUkPdvUVHJwVTRMSTNSJxf5S; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:15 GMT" + }, + { + "name": "Strict-Transport-Security", + "value": "max-age=15552000; includeSubDomains; preload" + }, + { + "name": "X-Content-Type-Options", + "value": "nosniff" + }, + { + "name": "CF-RAY", + "value": "a346d36688d203c2-SJC" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:15.898087Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/mistral-ocr/afc42ea89caa932d7ea33666d91e4f1a486bbe3165f5be777e67376075fd89a8.json b/tests/test_litellm/ocr/fixtures/mistral-ocr/afc42ea89caa932d7ea33666d91e4f1a486bbe3165f5be777e67376075fd89a8.json new file mode 100644 index 00000000000..728068b2e1e --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/mistral-ocr/afc42ea89caa932d7ea33666d91e4f1a486bbe3165f5be777e67376075fd89a8.json @@ -0,0 +1,115 @@ +{ + "case": { + "litellm_input": { + "document": { + "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24", + "type": "image_url" + }, + "document_annotation_format": { + "json_schema": { + "description": "Extract the visible document fields", + "name": "document_title", + "schema": { + "additionalProperties": false, + "properties": { + "title": { + "type": "string" + } + }, + "required": [ + "title" + ], + "type": "object" + }, + "strict": true + }, + "type": "json_schema" + }, + "document_annotation_prompt": "Extract the visible title", + "model": "mistral/mistral-ocr-latest" + }, + "provider_responses": [ + { + "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsImJsb2NrcyI6W3sidG9wX2xlZnRfeCI6MzM1LCJ0b3BfbGVmdF95IjoxMzgsImJvdHRvbV9yaWdodF94Ijo0NjQsImJvdHRvbV9yaWdodF95IjoxNjIsImNvbnRlbnQiOiJpbnZvaWNlIDEyMyIsImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJ0eXBlIjoidGV4dCJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOiJ7XCJ0aXRsZVwiOiBcImludm9pY2UgMTIzXCJ9IiwidXNhZ2VfaW5mbyI6eyJwYWdlc19wcm9jZXNzZWQiOjEsImRvY19zaXplX2J5dGVzIjo0MTI0fX0=", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:18 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "set-cookie", + "value": "__cf_bm=kVVa_K4blNT7UfkHNHEr2jJrVZoVVcKoCLeEqC9ojSY-1788292457.9748948-1.0.1.1-4aZN8zUH7fXP1NZK_aaS15fCbNd8A8ZJbBeCartOJWpgZMx2hW3klcPhDDvC0KSV6e.K1Ahz9wRggeTiUZQjsVuKYbR_vJ8dGfKXVNP2KyYcOfgaHyr0Kb1JOZSDGGnS; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:18 GMT" + }, + { + "name": "mistral-correlation-id", + "value": "01a05e89-664a-740d-a112-0a1183c302b3" + }, + { + "name": "x-kong-request-id", + "value": "01a05e89-664a-740d-a112-0a1183c302b3" + }, + { + "name": "x-ratelimit-limit-ocr-pages-minute", + "value": "60" + }, + { + "name": "x-ratelimit-remaining-ocr-pages-minute", + "value": "52" + }, + { + "name": "x-ratelimit-ocr-pages-query-cost", + "value": "1" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "402" + }, + { + "name": "Server", + "value": "cloudflare" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "x-kong-upstream-latency", + "value": "403" + }, + { + "name": "x-kong-proxy-latency", + "value": "20" + }, + { + "name": "X-Content-Type-Options", + "value": "nosniff" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "name": "Strict-Transport-Security", + "value": "max-age=15552000; includeSubDomains; preload" + }, + { + "name": "CF-RAY", + "value": "a346d376581f74f9-SJC" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:18.915814Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/mistral-ocr/c74f9b5ad64bd97d8b4c48772c9ee17598a3a1265a630f9a6e42a2df352906bf.json b/tests/test_litellm/ocr/fixtures/mistral-ocr/c74f9b5ad64bd97d8b4c48772c9ee17598a3a1265a630f9a6e42a2df352906bf.json new file mode 100644 index 00000000000..1653d03634a --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/mistral-ocr/c74f9b5ad64bd97d8b4c48772c9ee17598a3a1265a630f9a6e42a2df352906bf.json @@ -0,0 +1,95 @@ +{ + "case": { + "litellm_input": { + "confidence_scores_granularity": "page", + "document": { + "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24", + "type": "image_url" + }, + "model": "mistral/mistral-ocr-latest" + }, + "provider_responses": [ + { + "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOnsid29yZF9jb25maWRlbmNlX3Njb3JlcyI6W10sImF2ZXJhZ2VfcGFnZV9jb25maWRlbmNlX3Njb3JlIjowLjkwODQ1NTY0NTU0ODk3LCJtaW5pbXVtX3BhZ2VfY29uZmlkZW5jZV9zY29yZSI6MC4xNjE2ODIwODgzOTgyMzQ3NX0sImJsb2NrcyI6W3sidG9wX2xlZnRfeCI6MzM1LCJ0b3BfbGVmdF95IjoxMzgsImJvdHRvbV9yaWdodF94Ijo0NjQsImJvdHRvbV9yaWdodF95IjoxNjIsImNvbnRlbnQiOiJpbnZvaWNlIDEyMyIsImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJ0eXBlIjoidGV4dCJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOm51bGwsInVzYWdlX2luZm8iOnsicGFnZXNfcHJvY2Vzc2VkIjoxLCJkb2Nfc2l6ZV9ieXRlcyI6NDEyNH19", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:22 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "mistral-correlation-id", + "value": "01a05e89-7411-7d2c-9e43-01320e4cad1c" + }, + { + "name": "x-kong-request-id", + "value": "01a05e89-7411-7d2c-9e43-01320e4cad1c" + }, + { + "name": "x-ratelimit-limit-ocr-pages-minute", + "value": "60" + }, + { + "name": "x-ratelimit-remaining-ocr-pages-minute", + "value": "48" + }, + { + "name": "x-ratelimit-ocr-pages-query-cost", + "value": "1" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "329" + }, + { + "name": "Server", + "value": "cloudflare" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "x-kong-upstream-latency", + "value": "330" + }, + { + "name": "x-kong-proxy-latency", + "value": "14" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "name": "set-cookie", + "value": "__cf_bm=vGgqF1m3.RSN4S6h3XP_Kk8ojWPq3vypiJ2NZgCazl0-1788292461.4966974-1.0.1.1-OYrG7cwlLUmuwb_TTkXh4Te8fKHOoi5hlP5Mok.vCVqZD0lY1bv2Efm83lRx69__NMYwiYqJwrzDNChyYI5Vl9HpHmBybXfpbZ6WpjVEIscEyaLBDMJ_0UI5whsB2.xf; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:22 GMT" + }, + { + "name": "Strict-Transport-Security", + "value": "max-age=15552000; includeSubDomains; preload" + }, + { + "name": "X-Content-Type-Options", + "value": "nosniff" + }, + { + "name": "CF-RAY", + "value": "a346d38c59681749-SJC" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:22.437385Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/mistral-ocr/cd2461bbef1fc222e80b3397231c79a7f6895b1d6ef5c8f544bae77854bfd951.json b/tests/test_litellm/ocr/fixtures/mistral-ocr/cd2461bbef1fc222e80b3397231c79a7f6895b1d6ef5c8f544bae77854bfd951.json new file mode 100644 index 00000000000..120e5a43007 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/mistral-ocr/cd2461bbef1fc222e80b3397231c79a7f6895b1d6ef5c8f544bae77854bfd951.json @@ -0,0 +1,97 @@ +{ + "case": { + "litellm_input": { + "document": { + "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24", + "type": "image_url" + }, + "model": "mistral/mistral-ocr-latest", + "pages": [ + 0 + ] + }, + "provider_responses": [ + { + "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsImJsb2NrcyI6W3sidG9wX2xlZnRfeCI6MzM1LCJ0b3BfbGVmdF95IjoxMzgsImJvdHRvbV9yaWdodF94Ijo0NjQsImJvdHRvbV9yaWdodF95IjoxNjIsImNvbnRlbnQiOiJpbnZvaWNlIDEyMyIsImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJ0eXBlIjoidGV4dCJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOm51bGwsInVzYWdlX2luZm8iOnsicGFnZXNfcHJvY2Vzc2VkIjoxLCJkb2Nfc2l6ZV9ieXRlcyI6NDEyNH19", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:14 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "set-cookie", + "value": "__cf_bm=CJBaWJ.SsBbSNRd1h3ENyFHO8QA2S5qq0irYhSrOSQY-1788292453.9335105-1.0.1.1-IIqKfxCHaT.6ATkFZNdqQ87W8B2Lnv_N5RSB8sU.dZa5E0gROwfWTZcIsI3aQm_r6aqQT3o.c86aG9ltQq25grYhzFc1LXxWUsmCyMEW7WunMgbzYacSazYXeAxE0yPi; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:14 GMT" + }, + { + "name": "mistral-correlation-id", + "value": "01a05e89-5685-747f-b76d-dab242ea7512" + }, + { + "name": "x-kong-request-id", + "value": "01a05e89-5685-747f-b76d-dab242ea7512" + }, + { + "name": "x-ratelimit-limit-ocr-pages-minute", + "value": "60" + }, + { + "name": "x-ratelimit-remaining-ocr-pages-minute", + "value": "58" + }, + { + "name": "x-ratelimit-ocr-pages-query-cost", + "value": "1" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "179" + }, + { + "name": "Server", + "value": "cloudflare" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "x-kong-upstream-latency", + "value": "180" + }, + { + "name": "x-kong-proxy-latency", + "value": "12" + }, + { + "name": "X-Content-Type-Options", + "value": "nosniff" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "name": "Strict-Transport-Security", + "value": "max-age=15552000; includeSubDomains; preload" + }, + { + "name": "CF-RAY", + "value": "a346d35d1fbbebe5-SJC" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:14.385374Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/mistral-ocr/d1aa25a0353a8ff15c49088247a6d73ea777503f904a6bff9960a25046085d0d.json b/tests/test_litellm/ocr/fixtures/mistral-ocr/d1aa25a0353a8ff15c49088247a6d73ea777503f904a6bff9960a25046085d0d.json new file mode 100644 index 00000000000..50d23042db5 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/mistral-ocr/d1aa25a0353a8ff15c49088247a6d73ea777503f904a6bff9960a25046085d0d.json @@ -0,0 +1,95 @@ +{ + "case": { + "litellm_input": { + "document": { + "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24", + "type": "image_url" + }, + "include_image_base64": true, + "model": "mistral/mistral-ocr-latest" + }, + "provider_responses": [ + { + "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsImJsb2NrcyI6W3sidG9wX2xlZnRfeCI6MzM1LCJ0b3BfbGVmdF95IjoxMzgsImJvdHRvbV9yaWdodF94Ijo0NjQsImJvdHRvbV9yaWdodF95IjoxNjIsImNvbnRlbnQiOiJpbnZvaWNlIDEyMyIsImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJ0eXBlIjoidGV4dCJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOm51bGwsInVzYWdlX2luZm8iOnsicGFnZXNfcHJvY2Vzc2VkIjoxLCJkb2Nfc2l6ZV9ieXRlcyI6NDEyNH19", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:14 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "set-cookie", + "value": "__cf_bm=RwMrJUmbvjo3BcW48Leja7wReidkx5xcJUtnE.9z924-1788292454.453593-1.0.1.1-oxMCFin79BJrtZjXcFewsGcQA0dzFQlC6NHkBN9AuMlUSYoT8EZty3nieW5ORoOnygaYSrpckLffOUJddtxb4TgxSNpkJCBFZOzs3QLLlGpkZrsJx8Rw9ype09j_OKZx; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:14 GMT" + }, + { + "name": "mistral-correlation-id", + "value": "01a05e89-588e-7e32-a54b-f816bc6c1dc4" + }, + { + "name": "x-kong-request-id", + "value": "01a05e89-588e-7e32-a54b-f816bc6c1dc4" + }, + { + "name": "x-ratelimit-limit-ocr-pages-minute", + "value": "60" + }, + { + "name": "x-ratelimit-remaining-ocr-pages-minute", + "value": "57" + }, + { + "name": "x-ratelimit-ocr-pages-query-cost", + "value": "1" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "242" + }, + { + "name": "Server", + "value": "cloudflare" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "x-kong-upstream-latency", + "value": "242" + }, + { + "name": "x-kong-proxy-latency", + "value": "16" + }, + { + "name": "X-Content-Type-Options", + "value": "nosniff" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "name": "Strict-Transport-Security", + "value": "max-age=15552000; includeSubDomains; preload" + }, + { + "name": "CF-RAY", + "value": "a346d360593c138a-SJC" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:14.889667Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/mistral-ocr/e362d4eb7aa5f50706b54d0bbb777eebedf342e1844f9114d8db8b9de8a8b08b.json b/tests/test_litellm/ocr/fixtures/mistral-ocr/e362d4eb7aa5f50706b54d0bbb777eebedf342e1844f9114d8db8b9de8a8b08b.json new file mode 100644 index 00000000000..e4417561fbf --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/mistral-ocr/e362d4eb7aa5f50706b54d0bbb777eebedf342e1844f9114d8db8b9de8a8b08b.json @@ -0,0 +1,114 @@ +{ + "case": { + "litellm_input": { + "document": { + "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24", + "type": "image_url" + }, + "document_annotation_format": { + "json_schema": { + "description": "Extract the visible document fields", + "name": "document_title", + "schema": { + "additionalProperties": false, + "properties": { + "title": { + "type": "string" + } + }, + "required": [ + "title" + ], + "type": "object" + }, + "strict": true + }, + "type": "json_schema" + }, + "model": "mistral/mistral-ocr-latest" + }, + "provider_responses": [ + { + "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsImJsb2NrcyI6W3sidG9wX2xlZnRfeCI6MzM1LCJ0b3BfbGVmdF95IjoxMzgsImJvdHRvbV9yaWdodF94Ijo0NjQsImJvdHRvbV9yaWdodF95IjoxNjIsImNvbnRlbnQiOiJpbnZvaWNlIDEyMyIsImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJ0eXBlIjoidGV4dCJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOiJ7XCJ0aXRsZVwiOiBcIkludm9pY2VfMTIzXCJ9IiwidXNhZ2VfaW5mbyI6eyJwYWdlc19wcm9jZXNzZWQiOjEsImRvY19zaXplX2J5dGVzIjo0MTI0fX0=", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:17 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "mistral-correlation-id", + "value": "01a05e89-6060-71cb-9e35-66ab873b99f3" + }, + { + "name": "x-kong-request-id", + "value": "01a05e89-6060-71cb-9e35-66ab873b99f3" + }, + { + "name": "x-ratelimit-limit-ocr-pages-minute", + "value": "60" + }, + { + "name": "x-ratelimit-remaining-ocr-pages-minute", + "value": "53" + }, + { + "name": "x-ratelimit-ocr-pages-query-cost", + "value": "1" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "888" + }, + { + "name": "Server", + "value": "cloudflare" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "x-kong-upstream-latency", + "value": "889" + }, + { + "name": "x-kong-proxy-latency", + "value": "12" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "name": "set-cookie", + "value": "__cf_bm=xoadxKi40.IFnjiaqOmRs2WH6OMEVyT3As6eFw.vBvY-1788292456.4529853-1.0.1.1-jbao7uSuL.oLriQMYYO4SXltYHP3IPGIYg7yiRKk3Xqlwsc3xaQu0ES59EpMR3XVjJADyl9rHQfnKY3HvKRxXdGIA8Jerdfcu68wRUZNHcXd_epa4pcO5nnks7YPh93z; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:17 GMT" + }, + { + "name": "Strict-Transport-Security", + "value": "max-age=15552000; includeSubDomains; preload" + }, + { + "name": "X-Content-Type-Options", + "value": "nosniff" + }, + { + "name": "CF-RAY", + "value": "a346d36cd82f15ba-SJC" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:17.908846Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/mistral-ocr/e9217f2a27a061f2ed26a567677d176db3731e07107f45318fb681740ee49a74.json b/tests/test_litellm/ocr/fixtures/mistral-ocr/e9217f2a27a061f2ed26a567677d176db3731e07107f45318fb681740ee49a74.json new file mode 100644 index 00000000000..56de121e6ce --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/mistral-ocr/e9217f2a27a061f2ed26a567677d176db3731e07107f45318fb681740ee49a74.json @@ -0,0 +1,95 @@ +{ + "case": { + "litellm_input": { + "document": { + "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24", + "type": "image_url" + }, + "extract_header": true, + "model": "mistral/mistral-ocr-latest" + }, + "provider_responses": [ + { + "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsImJsb2NrcyI6W3sidG9wX2xlZnRfeCI6MzM1LCJ0b3BfbGVmdF95IjoxMzgsImJvdHRvbV9yaWdodF94Ijo0NjQsImJvdHRvbV9yaWdodF95IjoxNjIsImNvbnRlbnQiOiJpbnZvaWNlIDEyMyIsImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJ0eXBlIjoidGV4dCJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOm51bGwsInVzYWdlX2luZm8iOnsicGFnZXNfcHJvY2Vzc2VkIjoxLCJkb2Nfc2l6ZV9ieXRlcyI6NDEyNH19", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:19 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "mistral-correlation-id", + "value": "01a05e89-6a3a-76c7-85b8-d40a7156155e" + }, + { + "name": "x-kong-request-id", + "value": "01a05e89-6a3a-76c7-85b8-d40a7156155e" + }, + { + "name": "x-ratelimit-limit-ocr-pages-minute", + "value": "60" + }, + { + "name": "x-ratelimit-remaining-ocr-pages-minute", + "value": "51" + }, + { + "name": "x-ratelimit-ocr-pages-query-cost", + "value": "1" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "230" + }, + { + "name": "Server", + "value": "cloudflare" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "x-kong-upstream-latency", + "value": "230" + }, + { + "name": "x-kong-proxy-latency", + "value": "16" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "name": "set-cookie", + "value": "__cf_bm=O0x.ne9EKgb7nXAnOFoTqrbi69sVwHkdvll_fTS6PYE-1788292458.978935-1.0.1.1-2MYJEhW_I9Kf6XeYxBAqVLCXAnN8HABHbK351INmQpOQo036RmkEgf2..oRaM4BDqtR_F7Xrf72ekGCSm4C1h2tQaM6qXV0eCbt78fDRKI3i4N_cWLzghtZs0RJRw7Ir; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:19 GMT" + }, + { + "name": "Strict-Transport-Security", + "value": "max-age=15552000; includeSubDomains; preload" + }, + { + "name": "X-Content-Type-Options", + "value": "nosniff" + }, + { + "name": "CF-RAY", + "value": "a346d37c9944d8a7-SJC" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:19.420285Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/mistral-ocr/ea3c537a691b95003afc21b351549d5a62e67fa9e6c51ba30de4eaf4bba5a73f.json b/tests/test_litellm/ocr/fixtures/mistral-ocr/ea3c537a691b95003afc21b351549d5a62e67fa9e6c51ba30de4eaf4bba5a73f.json new file mode 100644 index 00000000000..348c234273d --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/mistral-ocr/ea3c537a691b95003afc21b351549d5a62e67fa9e6c51ba30de4eaf4bba5a73f.json @@ -0,0 +1,95 @@ +{ + "case": { + "litellm_input": { + "document": { + "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24", + "type": "image_url" + }, + "id": "case-1", + "model": "mistral/mistral-ocr-latest" + }, + "provider_responses": [ + { + "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsImJsb2NrcyI6W3sidG9wX2xlZnRfeCI6MzM1LCJ0b3BfbGVmdF95IjoxMzgsImJvdHRvbV9yaWdodF94Ijo0NjQsImJvdHRvbV9yaWdodF95IjoxNjIsImNvbnRlbnQiOiJpbnZvaWNlIDEyMyIsImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJ0eXBlIjoidGV4dCJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOm51bGwsInVzYWdlX2luZm8iOnsicGFnZXNfcHJvY2Vzc2VkIjoxLCJkb2Nfc2l6ZV9ieXRlcyI6NDEyNH19", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:23 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "mistral-correlation-id", + "value": "01a05e89-7bdc-74c5-b8eb-15fbd25b6cfd" + }, + { + "name": "x-kong-request-id", + "value": "01a05e89-7bdc-74c5-b8eb-15fbd25b6cfd" + }, + { + "name": "x-ratelimit-limit-ocr-pages-minute", + "value": "60" + }, + { + "name": "x-ratelimit-remaining-ocr-pages-minute", + "value": "46" + }, + { + "name": "x-ratelimit-ocr-pages-query-cost", + "value": "1" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "182" + }, + { + "name": "Server", + "value": "cloudflare" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "x-kong-upstream-latency", + "value": "183" + }, + { + "name": "x-kong-proxy-latency", + "value": "22" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "name": "set-cookie", + "value": "__cf_bm=0rjHVJtpp3aXFzTWuyNNxtBvLU2V3RPEgQGcO8MQIJo-1788292463.4929135-1.0.1.1-39O3ne30GQFGDB22kGltMQVL6E9Hk2pTTGEyKXMcYXy8Dzs.BogwfvcphEP473aOQPn_RYs3f5Jz_fhcd4BzKbs1YHm0AQbi84qoKxg.2BQ7kpkarYT1GHgFIfOs_Pxw; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:23 GMT" + }, + { + "name": "Strict-Transport-Security", + "value": "max-age=15552000; includeSubDomains; preload" + }, + { + "name": "X-Content-Type-Options", + "value": "nosniff" + }, + { + "name": "CF-RAY", + "value": "a346d398da783ad4-SJC" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:23.946966Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/mistral-ocr/effe3c112538d2aa6ac98e8f47033086cb79e87d2bdb99bdbb3b7e11751c43d7.json b/tests/test_litellm/ocr/fixtures/mistral-ocr/effe3c112538d2aa6ac98e8f47033086cb79e87d2bdb99bdbb3b7e11751c43d7.json new file mode 100644 index 00000000000..e9e3bdf836d --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/mistral-ocr/effe3c112538d2aa6ac98e8f47033086cb79e87d2bdb99bdbb3b7e11751c43d7.json @@ -0,0 +1,114 @@ +{ + "case": { + "litellm_input": { + "document": { + "document_url": "data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg==", + "type": "document_url" + }, + "document_annotation_format": { + "json_schema": { + "description": "Extract the visible document fields", + "name": "document_title", + "schema": { + "additionalProperties": false, + "properties": { + "title": { + "type": "string" + } + }, + "required": [ + "title" + ], + "type": "object" + }, + "strict": true + }, + "type": "json_schema" + }, + "model": "mistral/mistral-ocr-latest" + }, + "provider_responses": [ + { + "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiVGVzdCBQREYgRmlsZSIsImltYWdlcyI6W10sInRhYmxlcyI6W10sImh5cGVybGlua3MiOltdLCJoZWFkZXIiOm51bGwsImZvb3RlciI6bnVsbCwiZGltZW5zaW9ucyI6eyJkcGkiOjkzLCJoZWlnaHQiOjEwMjMsIndpZHRoIjo3OTF9LCJjb25maWRlbmNlX3Njb3JlcyI6bnVsbCwiYmxvY2tzIjpbeyJ0b3BfbGVmdF94IjoxMjYsInRvcF9sZWZ0X3kiOjEwNCwiYm90dG9tX3JpZ2h0X3giOjIyOSwiYm90dG9tX3JpZ2h0X3kiOjEyNywiY29udGVudCI6IlRlc3QgUERGIEZpbGUiLCJjb25maWRlbmNlX3Njb3JlcyI6bnVsbCwidHlwZSI6InRleHQifV19XSwibW9kZWwiOiJtaXN0cmFsLW9jci1sYXRlc3QiLCJkb2N1bWVudF9hbm5vdGF0aW9uIjoie1widGl0bGVcIjogXCJUZXN0X1BERl9GaWxlXCJ9IiwidXNhZ2VfaW5mbyI6eyJwYWdlc19wcm9jZXNzZWQiOjEsImRvY19zaXplX2J5dGVzIjo1ODl9fQ==", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:25 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "mistral-correlation-id", + "value": "01a05e89-81d5-79f8-914d-e3863af2d24b" + }, + { + "name": "x-kong-request-id", + "value": "01a05e89-81d5-79f8-914d-e3863af2d24b" + }, + { + "name": "x-ratelimit-limit-ocr-pages-minute", + "value": "60" + }, + { + "name": "x-ratelimit-remaining-ocr-pages-minute", + "value": "44" + }, + { + "name": "x-ratelimit-ocr-pages-query-cost", + "value": "1" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "448" + }, + { + "name": "Server", + "value": "cloudflare" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "x-kong-upstream-latency", + "value": "449" + }, + { + "name": "x-kong-proxy-latency", + "value": "15" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "name": "set-cookie", + "value": "__cf_bm=dDZPnK7j_xGIDo5mFF6PHBwL.E.iE3aIhXdIZ6xZIPg-1788292465.0206954-1.0.1.1-XVajgKnQ8WEsB75xPcp1hdJXhMUfK.oz.Apkd1nRlGofoxlEWK4F7M2bC8p67VFjDyEZK3O6mCzpiePraLUty31M1H92EBrZBT9wRwuaet2H7P73KO8QHZmO2Jm5tgyC; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:25 GMT" + }, + { + "name": "Strict-Transport-Security", + "value": "max-age=15552000; includeSubDomains; preload" + }, + { + "name": "X-Content-Type-Options", + "value": "nosniff" + }, + { + "name": "CF-RAY", + "value": "a346d3a25d0cccb8-SJC" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:25.959391Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/mistral-ocr/f430ede1f4069abab794731b9c233573014c6353f8d2486cc34753ceab6c7c1a.json b/tests/test_litellm/ocr/fixtures/mistral-ocr/f430ede1f4069abab794731b9c233573014c6353f8d2486cc34753ceab6c7c1a.json new file mode 100644 index 00000000000..1b79b9c7f81 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/mistral-ocr/f430ede1f4069abab794731b9c233573014c6353f8d2486cc34753ceab6c7c1a.json @@ -0,0 +1,95 @@ +{ + "case": { + "litellm_input": { + "document": { + "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24", + "type": "image_url" + }, + "include_blocks": false, + "model": "mistral/mistral-ocr-latest" + }, + "provider_responses": [ + { + "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsImJsb2NrcyI6bnVsbH1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOm51bGwsInVzYWdlX2luZm8iOnsicGFnZXNfcHJvY2Vzc2VkIjoxLCJkb2Nfc2l6ZV9ieXRlcyI6NDEyNH19", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:22 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "set-cookie", + "value": "__cf_bm=h45pW85qKj22bgXtkfz55PHEqryQZKiE7tImTTQx2u4-1788292462.5055265-1.0.1.1-kea4VbQ7Y.FRb9t5HgroFMzOCmbkU.byJfToALDpbgme.ScLO1wX13v5qEdll59C0DGKoXk2YYLAXvz0D6QfKN9mwTDRu23O72VJGgXtGu80xcMz16ZSRJfPn7hqnVFI; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:22 GMT" + }, + { + "name": "mistral-correlation-id", + "value": "01a05e89-7802-768c-bf57-9021a209ab48" + }, + { + "name": "x-kong-request-id", + "value": "01a05e89-7802-768c-bf57-9021a209ab48" + }, + { + "name": "x-ratelimit-limit-ocr-pages-minute", + "value": "60" + }, + { + "name": "x-ratelimit-remaining-ocr-pages-minute", + "value": "47" + }, + { + "name": "x-ratelimit-ocr-pages-query-cost", + "value": "1" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "213" + }, + { + "name": "Server", + "value": "cloudflare" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "x-kong-upstream-latency", + "value": "214" + }, + { + "name": "x-kong-proxy-latency", + "value": "17" + }, + { + "name": "X-Content-Type-Options", + "value": "nosniff" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "name": "Strict-Transport-Security", + "value": "max-age=15552000; includeSubDomains; preload" + }, + { + "name": "CF-RAY", + "value": "a346d392a8432af7-SJC" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:23.442341Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/reducto-legacy/5671d085483943d56d3e16d95a0a8cf08e7dc5bbe1f7286d7c04a0454baf5c66.json b/tests/test_litellm/ocr/fixtures/reducto-legacy/5671d085483943d56d3e16d95a0a8cf08e7dc5bbe1f7286d7c04a0454baf5c66.json new file mode 100644 index 00000000000..bee1e77d15a --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/reducto-legacy/5671d085483943d56d3e16d95a0a8cf08e7dc5bbe1f7286d7c04a0454baf5c66.json @@ -0,0 +1,46 @@ +{ + "case": { + "litellm_input": { + "custom_llm_provider": "reducto", + "document": { + "document_url": "data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg==", + "type": "document_url" + }, + "model": "parse-legacy" + }, + "provider_responses": [ + { + "body_b64": "eyJmaWxlX2lkIjoicmVkdWN0bzovLzQ1ZDNjYmJjLTRkNzctNDk2Ny04OTQxLTkzNWIwYzRhMDQ5My5wZGYiLCJwcmVzaWduZWRfdXJsIjpudWxsfQ==", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:58 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ], + "kind": "http", + "status_code": 200 + }, + { + "body_b64": "eyJyZXNwb25zZV90eXBlIjoicGFyc2UiLCJqb2JfaWQiOiIyNjJlZDY4My1lY2Q4LTQwYjktYTU2OC1mN2I3MDE0ODU0YzkiLCJkdXJhdGlvbiI6Mi40MTQ2NTk3Mzg1NDA2NDk0LCJwZGZfdXJsIjoiaHR0cHM6Ly9wcm9kLXN0b3JhZ2UyMDI0MTAxMDE0NDc0NTE0MDkwMDAwMDAwMS5zMy5hbWF6b25hd3MuY29tL29yZy84YmQ5ODNkMS00N2RkLTQ0MGItOTNkOC00MzE0MTJkODE5YmYvYTkxZjU2OGItNmU5OC00OTYyLTk5MjQtMDk3MjAyMDkzNzc5LnBkZj9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUEyVU9LNk9WQk9VWUw3V1lBJTJGMjAyNjA5MDElMkZ1cy13ZXN0LTIlMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjYwOTAxVDE5NTQ1NlomWC1BbXotRXhwaXJlcz00MzIwMCZYLUFtei1TaWduZWRIZWFkZXJzPWhvc3QmWC1BbXotU2lnbmF0dXJlPWFhNzdiMzNlMjBmYzE0MjYzMDc2ODhmNTUxNDdkNTdlOGQzYjIyODZhNzg3YzA5NmIyZGEwMTZkN2Q3ZDBjOTMiLCJzdHVkaW9fbGluayI6Imh0dHBzOi8vc3R1ZGlvLnJlZHVjdG8uYWkvam9iL2NkZGQ2MzVlLTVkMTYtNDYyMS1iZjQ4LTAzMWU3NjM3ZGMwYiIsInVzYWdlIjp7Im51bV9wYWdlcyI6MSwiY3JlZGl0cyI6MS4wLCJjcmVkaXRfYnJlYWtkb3duIjp7InBhZ2UiOjEuMH0sInBhZ2VfYmlsbGluZ19icmVha2Rvd24iOnsiMSI6WyJwYWdlIl19LCJub25fZW1wdHlfY2VsbF9jb3VudCI6bnVsbH0sInJlc3VsdCI6eyJ0eXBlIjoiZnVsbCIsImNodW5rcyI6W3siY29udGVudCI6IiMgVGVzdCBQREYgRmlsZSIsImVtYmVkIjoiIyBUZXN0IFBERiBGaWxlIiwiZW5yaWNoZWQiOm51bGwsImVucmljaG1lbnRfc3VjY2VzcyI6ZmFsc2UsImJsb2NrcyI6W3sidHlwZSI6IlRpdGxlIiwiYmJveCI6eyJsZWZ0IjowLjE2MzUyOTY5Mzg3MzU2ODAzLCJ0b3AiOjAuMTA0OTAyMzc2NjM1MDcwNTYsIndpZHRoIjowLjExODU1NjQwMDc3NDA1NTA4LCJoZWlnaHQiOjAuMDExMzU5MDQ1ODgxNDQyMjIxLCJwYWdlIjoxLCJvcmlnaW5hbF9wYWdlIjoxfSwiY29udGVudCI6IlRlc3QgUERGIEZpbGUiLCJpbWFnZV91cmwiOm51bGwsImNoYXJ0X2RhdGEiOm51bGwsImNvbmZpZGVuY2UiOiJoaWdoIiwiZ3JhbnVsYXJfY29uZmlkZW5jZSI6eyJleHRyYWN0X2NvbmZpZGVuY2UiOm51bGwsInBhcnNlX2NvbmZpZGVuY2UiOjAuNzEyNjcxNTMzMjI2OTY2OX0sImV4dHJhIjpudWxsfV19XSwib2NyIjpudWxsLCJjdXN0b20iOm51bGx9LCJwYXJzZV9tb2RlIjpudWxsLCJkb2N1bWVudF9wcm9wZXJ0aWVzIjpudWxsfQ==", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:59 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:59.198829Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/reducto-legacy/ea3262579a0882bcb61972e2a2aa4b0c7ae2e28ce957493e996a006ef2fcc6be.json b/tests/test_litellm/ocr/fixtures/reducto-legacy/ea3262579a0882bcb61972e2a2aa4b0c7ae2e28ce957493e996a006ef2fcc6be.json new file mode 100644 index 00000000000..34d76fe9abf --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/reducto-legacy/ea3262579a0882bcb61972e2a2aa4b0c7ae2e28ce957493e996a006ef2fcc6be.json @@ -0,0 +1,46 @@ +{ + "case": { + "litellm_input": { + "document": { + "document_url": "data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg==", + "type": "document_url" + }, + "enhance": {}, + "model": "reducto/parse-legacy" + }, + "provider_responses": [ + { + "body_b64": "eyJmaWxlX2lkIjoicmVkdWN0bzovL2I5ZjI0MmZkLWZkYjQtNGIwYS05NTM1LWYxMWY0MzJjNzY3OC5wZGYiLCJwcmVzaWduZWRfdXJsIjpudWxsfQ==", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:57 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ], + "kind": "http", + "status_code": 200 + }, + { + "body_b64": "eyJyZXNwb25zZV90eXBlIjoicGFyc2UiLCJqb2JfaWQiOiIyNWI3ODE4OS02NmZlLTQ2ZDktODExNC0xNDM3MmE5ZDQ0MmQiLCJkdXJhdGlvbiI6Mi40MTQ2NTk3Mzg1NDA2NDk0LCJwZGZfdXJsIjoiaHR0cHM6Ly9wcm9kLXN0b3JhZ2UyMDI0MTAxMDE0NDc0NTE0MDkwMDAwMDAwMS5zMy5hbWF6b25hd3MuY29tL29yZy84YmQ5ODNkMS00N2RkLTQ0MGItOTNkOC00MzE0MTJkODE5YmYvYTkxZjU2OGItNmU5OC00OTYyLTk5MjQtMDk3MjAyMDkzNzc5LnBkZj9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUEyVU9LNk9WQk9VWUw3V1lBJTJGMjAyNjA5MDElMkZ1cy13ZXN0LTIlMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjYwOTAxVDE5NTQ1NlomWC1BbXotRXhwaXJlcz00MzIwMCZYLUFtei1TaWduZWRIZWFkZXJzPWhvc3QmWC1BbXotU2lnbmF0dXJlPWFhNzdiMzNlMjBmYzE0MjYzMDc2ODhmNTUxNDdkNTdlOGQzYjIyODZhNzg3YzA5NmIyZGEwMTZkN2Q3ZDBjOTMiLCJzdHVkaW9fbGluayI6Imh0dHBzOi8vc3R1ZGlvLnJlZHVjdG8uYWkvam9iL2NkZGQ2MzVlLTVkMTYtNDYyMS1iZjQ4LTAzMWU3NjM3ZGMwYiIsInVzYWdlIjp7Im51bV9wYWdlcyI6MSwiY3JlZGl0cyI6MS4wLCJjcmVkaXRfYnJlYWtkb3duIjp7InBhZ2UiOjEuMH0sInBhZ2VfYmlsbGluZ19icmVha2Rvd24iOnsiMSI6WyJwYWdlIl19LCJub25fZW1wdHlfY2VsbF9jb3VudCI6bnVsbH0sInJlc3VsdCI6eyJ0eXBlIjoiZnVsbCIsImNodW5rcyI6W3siY29udGVudCI6IiMgVGVzdCBQREYgRmlsZSIsImVtYmVkIjoiIyBUZXN0IFBERiBGaWxlIiwiZW5yaWNoZWQiOm51bGwsImVucmljaG1lbnRfc3VjY2VzcyI6ZmFsc2UsImJsb2NrcyI6W3sidHlwZSI6IlRpdGxlIiwiYmJveCI6eyJsZWZ0IjowLjE2MzUyOTY5Mzg3MzU2ODAzLCJ0b3AiOjAuMTA0OTAyMzc2NjM1MDcwNTYsIndpZHRoIjowLjExODU1NjQwMDc3NDA1NTA4LCJoZWlnaHQiOjAuMDExMzU5MDQ1ODgxNDQyMjIxLCJwYWdlIjoxLCJvcmlnaW5hbF9wYWdlIjoxfSwiY29udGVudCI6IlRlc3QgUERGIEZpbGUiLCJpbWFnZV91cmwiOm51bGwsImNoYXJ0X2RhdGEiOm51bGwsImNvbmZpZGVuY2UiOiJoaWdoIiwiZ3JhbnVsYXJfY29uZmlkZW5jZSI6eyJleHRyYWN0X2NvbmZpZGVuY2UiOm51bGwsInBhcnNlX2NvbmZpZGVuY2UiOjAuNzEyNjcxNTMzMjI2OTY2OX0sImV4dHJhIjpudWxsfV19XSwib2NyIjpudWxsLCJjdXN0b20iOm51bGx9LCJwYXJzZV9tb2RlIjpudWxsLCJkb2N1bWVudF9wcm9wZXJ0aWVzIjpudWxsfQ==", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:57 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:58.471820Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/reducto-legacy/fbf21ae169e9ddbca638f445bad921e3e782e4e9f10be9a4ba000a8caf1e9ab5.json b/tests/test_litellm/ocr/fixtures/reducto-legacy/fbf21ae169e9ddbca638f445bad921e3e782e4e9f10be9a4ba000a8caf1e9ab5.json new file mode 100644 index 00000000000..3761d4e1f45 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/reducto-legacy/fbf21ae169e9ddbca638f445bad921e3e782e4e9f10be9a4ba000a8caf1e9ab5.json @@ -0,0 +1,45 @@ +{ + "case": { + "litellm_input": { + "document": { + "document_url": "data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg==", + "type": "document_url" + }, + "model": "reducto/parse-legacy" + }, + "provider_responses": [ + { + "body_b64": "eyJmaWxlX2lkIjoicmVkdWN0bzovL2E5MWY1NjhiLTZlOTgtNDk2Mi05OTI0LTA5NzIwMjA5Mzc3OS5wZGYiLCJwcmVzaWduZWRfdXJsIjpudWxsfQ==", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:53 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ], + "kind": "http", + "status_code": 200 + }, + { + "body_b64": "eyJyZXNwb25zZV90eXBlIjoicGFyc2UiLCJqb2JfaWQiOiJjZGRkNjM1ZS01ZDE2LTQ2MjEtYmY0OC0wMzFlNzYzN2RjMGIiLCJkdXJhdGlvbiI6Mi40MTQ2NTk3Mzg1NDA2NDk0LCJwZGZfdXJsIjoiaHR0cHM6Ly9wcm9kLXN0b3JhZ2UyMDI0MTAxMDE0NDc0NTE0MDkwMDAwMDAwMS5zMy5hbWF6b25hd3MuY29tL29yZy84YmQ5ODNkMS00N2RkLTQ0MGItOTNkOC00MzE0MTJkODE5YmYvYTkxZjU2OGItNmU5OC00OTYyLTk5MjQtMDk3MjAyMDkzNzc5LnBkZj9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUEyVU9LNk9WQk9VWUw3V1lBJTJGMjAyNjA5MDElMkZ1cy13ZXN0LTIlMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjYwOTAxVDE5NTQ1NlomWC1BbXotRXhwaXJlcz00MzIwMCZYLUFtei1TaWduZWRIZWFkZXJzPWhvc3QmWC1BbXotU2lnbmF0dXJlPWFhNzdiMzNlMjBmYzE0MjYzMDc2ODhmNTUxNDdkNTdlOGQzYjIyODZhNzg3YzA5NmIyZGEwMTZkN2Q3ZDBjOTMiLCJzdHVkaW9fbGluayI6Imh0dHBzOi8vc3R1ZGlvLnJlZHVjdG8uYWkvam9iL2NkZGQ2MzVlLTVkMTYtNDYyMS1iZjQ4LTAzMWU3NjM3ZGMwYiIsInVzYWdlIjp7Im51bV9wYWdlcyI6MSwiY3JlZGl0cyI6MS4wLCJjcmVkaXRfYnJlYWtkb3duIjp7InBhZ2UiOjEuMH0sInBhZ2VfYmlsbGluZ19icmVha2Rvd24iOnsiMSI6WyJwYWdlIl19LCJub25fZW1wdHlfY2VsbF9jb3VudCI6bnVsbH0sInJlc3VsdCI6eyJ0eXBlIjoiZnVsbCIsImNodW5rcyI6W3siY29udGVudCI6IiMgVGVzdCBQREYgRmlsZSIsImVtYmVkIjoiIyBUZXN0IFBERiBGaWxlIiwiZW5yaWNoZWQiOm51bGwsImVucmljaG1lbnRfc3VjY2VzcyI6ZmFsc2UsImJsb2NrcyI6W3sidHlwZSI6IlRpdGxlIiwiYmJveCI6eyJsZWZ0IjowLjE2MzUyOTY5Mzg3MzU2ODAzLCJ0b3AiOjAuMTA0OTAyMzc2NjM1MDcwNTYsIndpZHRoIjowLjExODU1NjQwMDc3NDA1NTA4LCJoZWlnaHQiOjAuMDExMzU5MDQ1ODgxNDQyMjIxLCJwYWdlIjoxLCJvcmlnaW5hbF9wYWdlIjoxfSwiY29udGVudCI6IlRlc3QgUERGIEZpbGUiLCJpbWFnZV91cmwiOm51bGwsImNoYXJ0X2RhdGEiOm51bGwsImNvbmZpZGVuY2UiOiJoaWdoIiwiZ3JhbnVsYXJfY29uZmlkZW5jZSI6eyJleHRyYWN0X2NvbmZpZGVuY2UiOm51bGwsInBhcnNlX2NvbmZpZGVuY2UiOjAuNzEyNjcxNTMzMjI2OTY2OX0sImV4dHJhIjpudWxsfV19XSwib2NyIjpudWxsLCJjdXN0b20iOm51bGx9LCJwYXJzZV9tb2RlIjpudWxsLCJkb2N1bWVudF9wcm9wZXJ0aWVzIjpudWxsfQ==", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:57 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:57.246119Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/reducto-v3/00312c58e0dccd4699086d6c37b45a56b19b4775393c22bf9fa6ead2c3cc323c.json b/tests/test_litellm/ocr/fixtures/reducto-v3/00312c58e0dccd4699086d6c37b45a56b19b4775393c22bf9fa6ead2c3cc323c.json new file mode 100644 index 00000000000..f4343921dd2 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/reducto-v3/00312c58e0dccd4699086d6c37b45a56b19b4775393c22bf9fa6ead2c3cc323c.json @@ -0,0 +1,46 @@ +{ + "case": { + "litellm_input": { + "custom_llm_provider": "reducto", + "document": { + "document_url": "data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg==", + "type": "document_url" + }, + "model": "parse-v3" + }, + "provider_responses": [ + { + "body_b64": "eyJmaWxlX2lkIjoicmVkdWN0bzovL2E2ZDNjM2JiLWE2ZjEtNDYzNi04YjNhLWVlOTIwYjI5ZDM4Ny5wZGYiLCJwcmVzaWduZWRfdXJsIjpudWxsfQ==", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:46 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ], + "kind": "http", + "status_code": 200 + }, + { + "body_b64": "eyJyZXNwb25zZV90eXBlIjoicGFyc2UiLCJqb2JfaWQiOiI0YTc4Yzk3NS0xODY1LTQ2ZmUtOGY4OS1iZWE5MmE0OWUyMTUiLCJkdXJhdGlvbiI6MS4xMTk0NDI3MDEzMzk3MjE3LCJwZGZfdXJsIjoiaHR0cHM6Ly9wcm9kLXN0b3JhZ2UyMDI0MTAxMDE0NDc0NTE0MDkwMDAwMDAwMS5zMy5hbWF6b25hd3MuY29tL29yZy84YmQ5ODNkMS00N2RkLTQ0MGItOTNkOC00MzE0MTJkODE5YmYvNmFiOWJjZjQtMTg5My00YjM3LWJkZGItYWNkYThjZTQ1ZGZiLnBkZj9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUEyVU9LNk9WQk9VWUw3V1lBJTJGMjAyNjA5MDElMkZ1cy13ZXN0LTIlMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjYwOTAxVDE5NTQ0NVomWC1BbXotRXhwaXJlcz00MzIwMCZYLUFtei1TaWduZWRIZWFkZXJzPWhvc3QmWC1BbXotU2lnbmF0dXJlPTFkYjYxZDMzZjYwMmYxM2U4NDQ0ZTg1ODBiNzM3YjMxZjM0NmU0NzQ5M2NjN2VjYTdjNGZjZGRiZGUwZGZiNDMiLCJzdHVkaW9fbGluayI6Imh0dHBzOi8vc3R1ZGlvLnJlZHVjdG8uYWkvam9iL2QzNDU1NDlmLTVkOTgtNGM2Mi1iM2UzLTI0Yzk5MDE0NGRmNCIsInVzYWdlIjp7Im51bV9wYWdlcyI6MSwiY3JlZGl0cyI6MS4wLCJjcmVkaXRfYnJlYWtkb3duIjp7InBhZ2UiOjEuMH0sInBhZ2VfYmlsbGluZ19icmVha2Rvd24iOnsiMSI6WyJwYWdlIl19LCJub25fZW1wdHlfY2VsbF9jb3VudCI6bnVsbH0sInJlc3VsdCI6eyJ0eXBlIjoiZnVsbCIsImNodW5rcyI6W3siY29udGVudCI6IiMgVGVzdCBQREYgRmlsZSIsImVtYmVkIjoiIyBUZXN0IFBERiBGaWxlIiwiZW5yaWNoZWQiOm51bGwsImVucmljaG1lbnRfc3VjY2VzcyI6ZmFsc2UsImJsb2NrcyI6W3sidHlwZSI6IlRpdGxlIiwiYmJveCI6eyJsZWZ0IjowLjE2MTg5NTc0NTI3MTgyNzQ4LCJ0b3AiOjAuMDk0NzkyOTU1NTM4MTQxMjEsIndpZHRoIjowLjExOTQ2MjkwNjUwODIwODc1LCJoZWlnaHQiOjAuMDIxNDY0MTQzMDE3MTA1MDcsInBhZ2UiOjEsIm9yaWdpbmFsX3BhZ2UiOjF9LCJjb250ZW50IjoiVGVzdCBQREYgRmlsZSIsImltYWdlX3VybCI6bnVsbCwiY2hhcnRfZGF0YSI6bnVsbCwiY29uZmlkZW5jZSI6ImhpZ2giLCJncmFudWxhcl9jb25maWRlbmNlIjp7ImV4dHJhY3RfY29uZmlkZW5jZSI6bnVsbCwicGFyc2VfY29uZmlkZW5jZSI6MC43MTg1MzU1NjkzMTAxODgyfSwiZXh0cmEiOm51bGx9XX1dLCJvY3IiOm51bGwsImN1c3RvbSI6bnVsbH0sInBhcnNlX21vZGUiOm51bGwsImRvY3VtZW50X3Byb3BlcnRpZXMiOm51bGx9", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:47 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:47.967951Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/reducto-v3/0b4ca483494e37fce75cc861ba62f06ce970d117de9828d72ed76b71b6f5adc1.json b/tests/test_litellm/ocr/fixtures/reducto-v3/0b4ca483494e37fce75cc861ba62f06ce970d117de9828d72ed76b71b6f5adc1.json new file mode 100644 index 00000000000..61080492535 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/reducto-v3/0b4ca483494e37fce75cc861ba62f06ce970d117de9828d72ed76b71b6f5adc1.json @@ -0,0 +1,56 @@ +{ + "case": { + "litellm_input": { + "custom_llm_provider": null, + "document": { + "document_url": "data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg==", + "type": "document_url" + }, + "formatting": { + "add_page_markers": true, + "include": [ + "change_tracking", + "highlight", + "comments" + ], + "merge_tables": true, + "table_output_format": "json" + }, + "model": "reducto/parse-v3" + }, + "provider_responses": [ + { + "body_b64": "eyJmaWxlX2lkIjoicmVkdWN0bzovL2Y4YmYxN2U4LWNhMGItNGFkZC1iNDg0LTk5ODJkMmU0YWMyYS5wZGYiLCJwcmVzaWduZWRfdXJsIjpudWxsfQ==", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:50 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ], + "kind": "http", + "status_code": 200 + }, + { + "body_b64": "eyJyZXNwb25zZV90eXBlIjoicGFyc2UiLCJqb2JfaWQiOiJlMzc5MjM4Ny04Mzg0LTQ1YWEtYjAyZS1hODQ1MjFkYjExNmEiLCJkdXJhdGlvbiI6MS4wNDk3MzYyNjEzNjc3OTc5LCJwZGZfdXJsIjoiaHR0cHM6Ly9wcm9kLXN0b3JhZ2UyMDI0MTAxMDE0NDc0NTE0MDkwMDAwMDAwMS5zMy5hbWF6b25hd3MuY29tL29yZy84YmQ5ODNkMS00N2RkLTQ0MGItOTNkOC00MzE0MTJkODE5YmYvZjhiZjE3ZTgtY2EwYi00YWRkLWI0ODQtOTk4MmQyZTRhYzJhLnBkZj9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUEyVU9LNk9WQk9VWUw3V1lBJTJGMjAyNjA5MDElMkZ1cy13ZXN0LTIlMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjYwOTAxVDE5NTQ1MlomWC1BbXotRXhwaXJlcz00MzIwMCZYLUFtei1TaWduZWRIZWFkZXJzPWhvc3QmWC1BbXotU2lnbmF0dXJlPTczNjcyNzVhNjlkZmQ1M2Y5ZmNmOWYwMjc3N2YwZmU5NzNlN2U1NDFmYTE4NzFlMDM0N2Y3ODhiOGM1MDc4ZjQiLCJzdHVkaW9fbGluayI6Imh0dHBzOi8vc3R1ZGlvLnJlZHVjdG8uYWkvam9iL2UzNzkyMzg3LTgzODQtNDVhYS1iMDJlLWE4NDUyMWRiMTE2YSIsInVzYWdlIjp7Im51bV9wYWdlcyI6MSwiY3JlZGl0cyI6MS4wLCJjcmVkaXRfYnJlYWtkb3duIjp7InBhZ2UiOjEuMH0sInBhZ2VfYmlsbGluZ19icmVha2Rvd24iOnsiMSI6WyJwYWdlIl19LCJub25fZW1wdHlfY2VsbF9jb3VudCI6bnVsbH0sInJlc3VsdCI6eyJ0eXBlIjoiZnVsbCIsImNodW5rcyI6W3siY29udGVudCI6IltbU1RBUlQgT0YgUEFHRSAxXV1cblxuIyBUZXN0IFBERiBGaWxlXG5cbltbRU5EIE9GIFBBR0UgMV1dIiwiZW1iZWQiOiJbW1NUQVJUIE9GIFBBR0UgMV1dXG5cbiMgVGVzdCBQREYgRmlsZVxuXG5bW0VORCBPRiBQQUdFIDFdXSIsImVucmljaGVkIjpudWxsLCJlbnJpY2htZW50X3N1Y2Nlc3MiOmZhbHNlLCJibG9ja3MiOlt7InR5cGUiOiJQYWdlIE51bWJlciIsImJib3giOnsibGVmdCI6MC4wLCJ0b3AiOjAuMCwid2lkdGgiOjAuMCwiaGVpZ2h0IjowLjAsInBhZ2UiOjEsIm9yaWdpbmFsX3BhZ2UiOjF9LCJjb250ZW50IjoiW1tTVEFSVCBPRiBQQUdFIDFdXSIsImltYWdlX3VybCI6bnVsbCwiY2hhcnRfZGF0YSI6bnVsbCwiY29uZmlkZW5jZSI6ImhpZ2giLCJncmFudWxhcl9jb25maWRlbmNlIjp7ImV4dHJhY3RfY29uZmlkZW5jZSI6bnVsbCwicGFyc2VfY29uZmlkZW5jZSI6bnVsbH0sImV4dHJhIjpudWxsfSx7InR5cGUiOiJUaXRsZSIsImJib3giOnsibGVmdCI6MC4xNjE4OTU3NDUyNzE4Mjc0OCwidG9wIjowLjA5NDc5Mjk1NTUzODE0MTIxLCJ3aWR0aCI6MC4xMTk0NjI5MDY1MDgyMDg3NSwiaGVpZ2h0IjowLjAyMTQ2NDE0MzAxNzEwNTA3LCJwYWdlIjoxLCJvcmlnaW5hbF9wYWdlIjoxfSwiY29udGVudCI6IlRlc3QgUERGIEZpbGUiLCJpbWFnZV91cmwiOm51bGwsImNoYXJ0X2RhdGEiOm51bGwsImNvbmZpZGVuY2UiOiJoaWdoIiwiZ3JhbnVsYXJfY29uZmlkZW5jZSI6eyJleHRyYWN0X2NvbmZpZGVuY2UiOm51bGwsInBhcnNlX2NvbmZpZGVuY2UiOjAuNzE4NTM1NTY5MzEwMTg4Mn0sImV4dHJhIjpudWxsfSx7InR5cGUiOiJQYWdlIE51bWJlciIsImJib3giOnsibGVmdCI6MC4wLCJ0b3AiOjAuMCwid2lkdGgiOjAuMCwiaGVpZ2h0IjowLjAsInBhZ2UiOjEsIm9yaWdpbmFsX3BhZ2UiOjF9LCJjb250ZW50IjoiW1tFTkQgT0YgUEFHRSAxXV0iLCJpbWFnZV91cmwiOm51bGwsImNoYXJ0X2RhdGEiOm51bGwsImNvbmZpZGVuY2UiOiJoaWdoIiwiZ3JhbnVsYXJfY29uZmlkZW5jZSI6eyJleHRyYWN0X2NvbmZpZGVuY2UiOm51bGwsInBhcnNlX2NvbmZpZGVuY2UiOm51bGx9LCJleHRyYSI6bnVsbH1dfV0sIm9jciI6bnVsbCwiY3VzdG9tIjpudWxsfSwicGFyc2VfbW9kZSI6bnVsbCwiZG9jdW1lbnRfcHJvcGVydGllcyI6bnVsbH0=", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:52 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:53.454592Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/reducto-v3/3a512e502dc6310e671e8967a7652520370e0c873939813c2167799cffacba06.json b/tests/test_litellm/ocr/fixtures/reducto-v3/3a512e502dc6310e671e8967a7652520370e0c873939813c2167799cffacba06.json new file mode 100644 index 00000000000..dbdce610f6d --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/reducto-v3/3a512e502dc6310e671e8967a7652520370e0c873939813c2167799cffacba06.json @@ -0,0 +1,48 @@ +{ + "case": { + "litellm_input": { + "document": { + "document_url": "data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg==", + "type": "document_url" + }, + "model": "reducto/parse-v3", + "settings": { + "return_ocr_data": true + } + }, + "provider_responses": [ + { + "body_b64": "eyJmaWxlX2lkIjoicmVkdWN0bzovL2ZjNmViY2IxLTk1ZDQtNDZlYy05MGQwLWVmY2EzM2VmMTk3Zi5wZGYiLCJwcmVzaWduZWRfdXJsIjpudWxsfQ==", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:41 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ], + "kind": "http", + "status_code": 200 + }, + { + "body_b64": "eyJyZXNwb25zZV90eXBlIjoicGFyc2UiLCJqb2JfaWQiOiIzZjZlZjY0ZC1jOTQ5LTRiOGQtOTg5OC1hZmIzYjAxNjY5ZTYiLCJkdXJhdGlvbiI6MS40NDgwNzQzNDA4MjAzMTI1LCJwZGZfdXJsIjoiaHR0cHM6Ly9wcm9kLXN0b3JhZ2UyMDI0MTAxMDE0NDc0NTE0MDkwMDAwMDAwMS5zMy5hbWF6b25hd3MuY29tL29yZy84YmQ5ODNkMS00N2RkLTQ0MGItOTNkOC00MzE0MTJkODE5YmYvZmM2ZWJjYjEtOTVkNC00NmVjLTkwZDAtZWZjYTMzZWYxOTdmLnBkZj9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUEyVU9LNk9WQk9VWUw3V1lBJTJGMjAyNjA5MDElMkZ1cy13ZXN0LTIlMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjYwOTAxVDE5NTQ0M1omWC1BbXotRXhwaXJlcz00MzIwMCZYLUFtei1TaWduZWRIZWFkZXJzPWhvc3QmWC1BbXotU2lnbmF0dXJlPTA5NjVjYTEyOWNlZTQ5YTQxNjUwZGYxNTIwNTdhMTdhNGY5ZTI2NWY5MmYyMjYxYTRmNjU3NjU4NzI3MmViMDMiLCJzdHVkaW9fbGluayI6Imh0dHBzOi8vc3R1ZGlvLnJlZHVjdG8uYWkvam9iLzNmNmVmNjRkLWM5NDktNGI4ZC05ODk4LWFmYjNiMDE2NjllNiIsInVzYWdlIjp7Im51bV9wYWdlcyI6MSwiY3JlZGl0cyI6MS4wLCJjcmVkaXRfYnJlYWtkb3duIjp7InBhZ2UiOjEuMH0sInBhZ2VfYmlsbGluZ19icmVha2Rvd24iOnsiMSI6WyJwYWdlIl19LCJub25fZW1wdHlfY2VsbF9jb3VudCI6bnVsbH0sInJlc3VsdCI6eyJ0eXBlIjoiZnVsbCIsImNodW5rcyI6W3siY29udGVudCI6IiMgVGVzdCBQREYgRmlsZSIsImVtYmVkIjoiIyBUZXN0IFBERiBGaWxlIiwiZW5yaWNoZWQiOm51bGwsImVucmljaG1lbnRfc3VjY2VzcyI6ZmFsc2UsImJsb2NrcyI6W3sidHlwZSI6IlRpdGxlIiwiYmJveCI6eyJsZWZ0IjowLjE2MTg5NTc0NTI3MTgyNzQ4LCJ0b3AiOjAuMDk0NzkyOTU1NTM4MTQxMjEsIndpZHRoIjowLjExOTQ2MjkwNjUwODIwODc1LCJoZWlnaHQiOjAuMDIxNDY0MTQzMDE3MTA1MDcsInBhZ2UiOjEsIm9yaWdpbmFsX3BhZ2UiOjF9LCJjb250ZW50IjoiVGVzdCBQREYgRmlsZSIsImltYWdlX3VybCI6bnVsbCwiY2hhcnRfZGF0YSI6bnVsbCwiY29uZmlkZW5jZSI6ImhpZ2giLCJncmFudWxhcl9jb25maWRlbmNlIjp7ImV4dHJhY3RfY29uZmlkZW5jZSI6bnVsbCwicGFyc2VfY29uZmlkZW5jZSI6MC43MTg1MzU1NjkzMTAxODgyfSwiZXh0cmEiOm51bGx9XX1dLCJvY3IiOnsid29yZHMiOlt7InRleHQiOiJUZXN0IiwiYmJveCI6eyJsZWZ0IjowLjE2MTg5NDc1NDY0NjY1MDMyLCJ0b3AiOjAuMDk0OTE5MjE0MzQ0OTg3OTcsIndpZHRoIjowLjAzODMyMDg5MDQzOTEwMjA3LCJoZWlnaHQiOjAuMDIxMDE5NTMxMDE4ODA2MjI1LCJwYWdlIjoxLCJvcmlnaW5hbF9wYWdlIjoxfSwiY29uZmlkZW5jZSI6MS4wLCJjaHVua19pbmRleCI6bnVsbCwicm90YXRpb24iOjM1OX0seyJ0ZXh0IjoiUERGIiwiYmJveCI6eyJsZWZ0IjowLjIwNTQ4MTk1OTMyNDI1NzA2LCJ0b3AiOjAuMDk1MTQ5OTA3MTk4ODE5MjQsIndpZHRoIjowLjAzOTM5OTI5MDM5NjQ5NzE0LCJoZWlnaHQiOjAuMDIxMDI1MjQzNDMxNjMwNzYsInBhZ2UiOjEsIm9yaWdpbmFsX3BhZ2UiOjF9LCJjb25maWRlbmNlIjoxLjAsImNodW5rX2luZGV4IjpudWxsLCJyb3RhdGlvbiI6MzU5fSx7InRleHQiOiJGaWxlIiwiYmJveCI6eyJsZWZ0IjowLjI1MDE0NzU3NjQyNTU4OTc3LCJ0b3AiOjAuMDk1Mzg2MzAyODMyNDAxMTUsIndpZHRoIjowLjAzMTc3MjAxNTE0MTUwNTgyLCJoZWlnaHQiOjAuMDIwOTg0ODcxMjE4OTAyODk1LCJwYWdlIjoxLCJvcmlnaW5hbF9wYWdlIjoxfSwiY29uZmlkZW5jZSI6MS4wLCJjaHVua19pbmRleCI6bnVsbCwicm90YXRpb24iOjM1OX1dLCJsaW5lcyI6W3sidGV4dCI6IlRlc3QgUERGIEZpbGUiLCJiYm94Ijp7ImxlZnQiOjAuMTYxODk0NzU0NjQ2NjUwMzIsInRvcCI6MC4wOTQ5MTkyMTQzNDQ5ODc5Nywid2lkdGgiOjAuMTIwMDI0ODM2OTIwNDQ1MjYsImhlaWdodCI6MC4wMjE0NTE5NTk3MDYzMTYwOTIsInBhZ2UiOjEsIm9yaWdpbmFsX3BhZ2UiOjF9LCJjb25maWRlbmNlIjoxLjAsImNodW5rX2luZGV4IjpudWxsLCJyb3RhdGlvbiI6MzU5fV19LCJjdXN0b20iOm51bGx9LCJwYXJzZV9tb2RlIjpudWxsLCJkb2N1bWVudF9wcm9wZXJ0aWVzIjpudWxsfQ==", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:43 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:43.968997Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/reducto-v3/5227441615ac16de68a18458e9704be5c772291d89dd5a67480c21e4e2ac3834.json b/tests/test_litellm/ocr/fixtures/reducto-v3/5227441615ac16de68a18458e9704be5c772291d89dd5a67480c21e4e2ac3834.json new file mode 100644 index 00000000000..ca28cecbe63 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/reducto-v3/5227441615ac16de68a18458e9704be5c772291d89dd5a67480c21e4e2ac3834.json @@ -0,0 +1,45 @@ +{ + "case": { + "litellm_input": { + "document": { + "document_url": "data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg==", + "type": "document_url" + }, + "model": "reducto/parse-v3" + }, + "provider_responses": [ + { + "body_b64": "eyJmaWxlX2lkIjoicmVkdWN0bzovLzAxMGIwMWIyLTgzYmQtNDQ2ZC1hZjExLWQxMjBjNWFjMmMwMi5wZGYiLCJwcmVzaWduZWRfdXJsIjpudWxsfQ==", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:27 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ], + "kind": "http", + "status_code": 200 + }, + { + "body_b64": "eyJyZXNwb25zZV90eXBlIjoicGFyc2UiLCJqb2JfaWQiOiI4MTM4YTZjNi1iNzI2LTQzMDAtOTBiNC1lNmQ1ZDQzZjZmNzAiLCJkdXJhdGlvbiI6MS4yNjQ4MjcyNTE0MzQzMjYyLCJwZGZfdXJsIjoiaHR0cHM6Ly9wcm9kLXN0b3JhZ2UyMDI0MTAxMDE0NDc0NTE0MDkwMDAwMDAwMS5zMy5hbWF6b25hd3MuY29tL29yZy84YmQ5ODNkMS00N2RkLTQ0MGItOTNkOC00MzE0MTJkODE5YmYvMDEwYjAxYjItODNiZC00NDZkLWFmMTEtZDEyMGM1YWMyYzAyLnBkZj9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUEyVU9LNk9WQk9VWUw3V1lBJTJGMjAyNjA5MDElMkZ1cy13ZXN0LTIlMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjYwOTAxVDE5NTQyOFomWC1BbXotRXhwaXJlcz00MzIwMCZYLUFtei1TaWduZWRIZWFkZXJzPWhvc3QmWC1BbXotU2lnbmF0dXJlPTNjZGE5MDIwZmVmZmY4NWQ5ZGQxM2YxMjQ5MDhkN2IwMGU1NWI2ZTgyYTcyMDM5NTQ2ZDI2MjA5OWNmY2YzYjQiLCJzdHVkaW9fbGluayI6Imh0dHBzOi8vc3R1ZGlvLnJlZHVjdG8uYWkvam9iLzgxMzhhNmM2LWI3MjYtNDMwMC05MGI0LWU2ZDVkNDNmNmY3MCIsInVzYWdlIjp7Im51bV9wYWdlcyI6MSwiY3JlZGl0cyI6MS4wLCJjcmVkaXRfYnJlYWtkb3duIjp7InBhZ2UiOjEuMH0sInBhZ2VfYmlsbGluZ19icmVha2Rvd24iOnsiMSI6WyJwYWdlIl19LCJub25fZW1wdHlfY2VsbF9jb3VudCI6bnVsbH0sInJlc3VsdCI6eyJ0eXBlIjoiZnVsbCIsImNodW5rcyI6W3siY29udGVudCI6IiMgVGVzdCBQREYgRmlsZSIsImVtYmVkIjoiIyBUZXN0IFBERiBGaWxlIiwiZW5yaWNoZWQiOm51bGwsImVucmljaG1lbnRfc3VjY2VzcyI6ZmFsc2UsImJsb2NrcyI6W3sidHlwZSI6IlRpdGxlIiwiYmJveCI6eyJsZWZ0IjowLjE2MTg5NTc0NTI3MTgyNzQ4LCJ0b3AiOjAuMDk0NzkyOTU1NTM4MTQxMjEsIndpZHRoIjowLjExOTQ2MjkwNjUwODIwODc1LCJoZWlnaHQiOjAuMDIxNDY0MTQzMDE3MTA1MDcsInBhZ2UiOjEsIm9yaWdpbmFsX3BhZ2UiOjF9LCJjb250ZW50IjoiVGVzdCBQREYgRmlsZSIsImltYWdlX3VybCI6bnVsbCwiY2hhcnRfZGF0YSI6bnVsbCwiY29uZmlkZW5jZSI6ImhpZ2giLCJncmFudWxhcl9jb25maWRlbmNlIjp7ImV4dHJhY3RfY29uZmlkZW5jZSI6bnVsbCwicGFyc2VfY29uZmlkZW5jZSI6MC43MTg1MzU1NjkzMTAxODgyfSwiZXh0cmEiOm51bGx9XX1dLCJvY3IiOm51bGwsImN1c3RvbSI6bnVsbH0sInBhcnNlX21vZGUiOm51bGwsImRvY3VtZW50X3Byb3BlcnRpZXMiOm51bGx9", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:29 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:30.255600Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/reducto-v3/75876320a008dc29db48c33e4d4ccb9e95c3aae1d774f03869cf7cacfaa9aa07.json b/tests/test_litellm/ocr/fixtures/reducto-v3/75876320a008dc29db48c33e4d4ccb9e95c3aae1d774f03869cf7cacfaa9aa07.json new file mode 100644 index 00000000000..b28e1a7efee --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/reducto-v3/75876320a008dc29db48c33e4d4ccb9e95c3aae1d774f03869cf7cacfaa9aa07.json @@ -0,0 +1,82 @@ +{ + "case": { + "litellm_input": { + "custom_llm_provider": null, + "document": { + "document_url": "data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg==", + "type": "document_url" + }, + "formatting": { + "add_page_markers": false, + "include": [ + "signatures", + "ignore_watermarks" + ], + "merge_tables": true, + "table_output_format": "json" + }, + "model": "reducto/parse-v3", + "retrieval": { + "chunking": { + "chunk_mode": "variable", + "chunk_overlap": 32, + "chunk_size": 1500 + }, + "embedding_optimized": false, + "filter_blocks": [ + "Figure", + "Table", + "Key Value" + ] + }, + "settings": { + "embed_pdf_metadata": false, + "embed_pdf_metadata_dpi": 100, + "extraction_mode": "hybrid", + "force_url_result": false, + "ocr_system": "legacy", + "page_range": [ + 1 + ], + "persist_results": false, + "return_images": [], + "return_ocr_data": false, + "timeout": 900.0 + } + }, + "provider_responses": [ + { + "body_b64": "eyJmaWxlX2lkIjoicmVkdWN0bzovLzVkZjc0NWY0LTI4NzctNGNmNi05YmRlLWEyZjgyOWM5M2VlYS5wZGYiLCJwcmVzaWduZWRfdXJsIjpudWxsfQ==", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:48 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ], + "kind": "http", + "status_code": 200 + }, + { + "body_b64": "eyJyZXNwb25zZV90eXBlIjoicGFyc2UiLCJqb2JfaWQiOiI3Y2E2NzI0Mi0xNjc3LTQ4NGUtYjJlZi1iMjU2ZGNkZDFlYTQiLCJkdXJhdGlvbiI6MS4zMzU5MzA4MjQyNzk3ODUyLCJwZGZfdXJsIjoiaHR0cHM6Ly9wcm9kLXN0b3JhZ2UyMDI0MTAxMDE0NDc0NTE0MDkwMDAwMDAwMS5zMy5hbWF6b25hd3MuY29tL29yZy84YmQ5ODNkMS00N2RkLTQ0MGItOTNkOC00MzE0MTJkODE5YmYvNWRmNzQ1ZjQtMjg3Ny00Y2Y2LTliZGUtYTJmODI5YzkzZWVhLnBkZj9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUEyVU9LNk9WQk9VWUw3V1lBJTJGMjAyNjA5MDElMkZ1cy13ZXN0LTIlMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjYwOTAxVDE5NTQ0OVomWC1BbXotRXhwaXJlcz00MzIwMCZYLUFtei1TaWduZWRIZWFkZXJzPWhvc3QmWC1BbXotU2lnbmF0dXJlPTA3N2E2NTUxMjAxZDI1NjJmZDE1NDNmZDVjMzkxNTQxY2ZmZTYwNmQ2NDMwN2EzZjZkMTAzY2YxODIxOWY4MGEiLCJzdHVkaW9fbGluayI6Imh0dHBzOi8vc3R1ZGlvLnJlZHVjdG8uYWkvam9iLzdjYTY3MjQyLTE2NzctNDg0ZS1iMmVmLWIyNTZkY2RkMWVhNCIsInVzYWdlIjp7Im51bV9wYWdlcyI6MSwiY3JlZGl0cyI6MS4wLCJjcmVkaXRfYnJlYWtkb3duIjp7InBhZ2UiOjEuMH0sInBhZ2VfYmlsbGluZ19icmVha2Rvd24iOnsiMSI6WyJwYWdlIl19LCJub25fZW1wdHlfY2VsbF9jb3VudCI6bnVsbH0sInJlc3VsdCI6eyJ0eXBlIjoiZnVsbCIsImNodW5rcyI6W3siY29udGVudCI6IiMgVGVzdCBQREYgRmlsZSIsImVtYmVkIjoiIyBUZXN0IFBERiBGaWxlIiwiZW5yaWNoZWQiOm51bGwsImVucmljaG1lbnRfc3VjY2VzcyI6ZmFsc2UsImJsb2NrcyI6W3sidHlwZSI6IlRpdGxlIiwiYmJveCI6eyJsZWZ0IjowLjE2MTg5NTc0NTI3MTgyNzQ4LCJ0b3AiOjAuMDk0NzkyOTU1NTM4MTQxMjEsIndpZHRoIjowLjExOTQ2MjkwNjUwODIwODc1LCJoZWlnaHQiOjAuMDIxNDY0MTQzMDE3MTA1MDcsInBhZ2UiOjEsIm9yaWdpbmFsX3BhZ2UiOjF9LCJjb250ZW50IjoiVGVzdCBQREYgRmlsZSIsImltYWdlX3VybCI6bnVsbCwiY2hhcnRfZGF0YSI6bnVsbCwiY29uZmlkZW5jZSI6ImhpZ2giLCJncmFudWxhcl9jb25maWRlbmNlIjp7ImV4dHJhY3RfY29uZmlkZW5jZSI6bnVsbCwicGFyc2VfY29uZmlkZW5jZSI6MC43MTg1MzU1NjkzMTAxODgyfSwiZXh0cmEiOm51bGx9XX1dLCJvY3IiOm51bGwsImN1c3RvbSI6bnVsbH0sInBhcnNlX21vZGUiOm51bGwsImRvY3VtZW50X3Byb3BlcnRpZXMiOm51bGx9", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:50 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:50.708637Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/reducto-v3/aee30a1d7a51a3fd8837b56d6baa3a43364ea319d905b5030014f4d85e7c94e9.json b/tests/test_litellm/ocr/fixtures/reducto-v3/aee30a1d7a51a3fd8837b56d6baa3a43364ea319d905b5030014f4d85e7c94e9.json new file mode 100644 index 00000000000..ba07f5baa4f --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/reducto-v3/aee30a1d7a51a3fd8837b56d6baa3a43364ea319d905b5030014f4d85e7c94e9.json @@ -0,0 +1,50 @@ +{ + "case": { + "litellm_input": { + "document": { + "document_url": "data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg==", + "type": "document_url" + }, + "model": "reducto/parse-v3", + "retrieval": { + "chunking": { + "chunk_mode": "page" + } + } + }, + "provider_responses": [ + { + "body_b64": "eyJmaWxlX2lkIjoicmVkdWN0bzovLzY3MWQ2ZTAwLTZkZjUtNDkzYS1iZDllLThiYmYzZDc3ZDNhYi5wZGYiLCJwcmVzaWduZWRfdXJsIjpudWxsfQ==", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:36 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ], + "kind": "http", + "status_code": 200 + }, + { + "body_b64": "eyJyZXNwb25zZV90eXBlIjoicGFyc2UiLCJqb2JfaWQiOiI4ZjE1YjZkZC1mMmQyLTQ4ZTItYTRjOS0wMjMwYTlhY2U3MDQiLCJkdXJhdGlvbiI6My43NTEwNjQzMDA1MzcxMDk0LCJwZGZfdXJsIjoiaHR0cHM6Ly9wcm9kLXN0b3JhZ2UyMDI0MTAxMDE0NDc0NTE0MDkwMDAwMDAwMS5zMy5hbWF6b25hd3MuY29tL29yZy84YmQ5ODNkMS00N2RkLTQ0MGItOTNkOC00MzE0MTJkODE5YmYvNjcxZDZlMDAtNmRmNS00OTNhLWJkOWUtOGJiZjNkNzdkM2FiLnBkZj9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUEyVU9LNk9WQk9VWUw3V1lBJTJGMjAyNjA5MDElMkZ1cy13ZXN0LTIlMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjYwOTAxVDE5NTQ0MFomWC1BbXotRXhwaXJlcz00MzIwMCZYLUFtei1TaWduZWRIZWFkZXJzPWhvc3QmWC1BbXotU2lnbmF0dXJlPTk2YTRmOTA0ZThkM2NhNThkMzk5YWIyMWFkZTY5ZWMwZWE2MmZmNWRmZWUzMjBjNWVlYWE2ZWFhMGM0ZTA1NjEiLCJzdHVkaW9fbGluayI6Imh0dHBzOi8vc3R1ZGlvLnJlZHVjdG8uYWkvam9iLzhmMTViNmRkLWYyZDItNDhlMi1hNGM5LTAyMzBhOWFjZTcwNCIsInVzYWdlIjp7Im51bV9wYWdlcyI6MSwiY3JlZGl0cyI6MS4wLCJjcmVkaXRfYnJlYWtkb3duIjp7InBhZ2UiOjEuMH0sInBhZ2VfYmlsbGluZ19icmVha2Rvd24iOnsiMSI6WyJwYWdlIl19LCJub25fZW1wdHlfY2VsbF9jb3VudCI6bnVsbH0sInJlc3VsdCI6eyJ0eXBlIjoiZnVsbCIsImNodW5rcyI6W3siY29udGVudCI6IiMgVGVzdCBQREYgRmlsZSIsImVtYmVkIjoiIyBUZXN0IFBERiBGaWxlIiwiZW5yaWNoZWQiOm51bGwsImVucmljaG1lbnRfc3VjY2VzcyI6ZmFsc2UsImJsb2NrcyI6W3sidHlwZSI6IlRpdGxlIiwiYmJveCI6eyJsZWZ0IjowLjE2MTg5NTc0NTI3MTgyNzQ4LCJ0b3AiOjAuMDk0NzkyOTU1NTM4MTQxMjEsIndpZHRoIjowLjExOTQ2MjkwNjUwODIwODc1LCJoZWlnaHQiOjAuMDIxNDY0MTQzMDE3MTA1MDcsInBhZ2UiOjEsIm9yaWdpbmFsX3BhZ2UiOjF9LCJjb250ZW50IjoiVGVzdCBQREYgRmlsZSIsImltYWdlX3VybCI6bnVsbCwiY2hhcnRfZGF0YSI6bnVsbCwiY29uZmlkZW5jZSI6ImhpZ2giLCJncmFudWxhcl9jb25maWRlbmNlIjp7ImV4dHJhY3RfY29uZmlkZW5jZSI6bnVsbCwicGFyc2VfY29uZmlkZW5jZSI6MC43MTg1MzU1NjkzMTAxODgyfSwiZXh0cmEiOm51bGx9XX1dLCJvY3IiOm51bGwsImN1c3RvbSI6bnVsbH0sInBhcnNlX21vZGUiOm51bGwsImRvY3VtZW50X3Byb3BlcnRpZXMiOm51bGx9", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:40 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:41.233002Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/reducto-v3/cfb4093138a106055f36402de37409b1ca17c23b9e491b4f8b6ac705acd05637.json b/tests/test_litellm/ocr/fixtures/reducto-v3/cfb4093138a106055f36402de37409b1ca17c23b9e491b4f8b6ac705acd05637.json new file mode 100644 index 00000000000..8b7dc56ebc5 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/reducto-v3/cfb4093138a106055f36402de37409b1ca17c23b9e491b4f8b6ac705acd05637.json @@ -0,0 +1,48 @@ +{ + "case": { + "litellm_input": { + "document": { + "document_url": "data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg==", + "type": "document_url" + }, + "formatting": { + "table_output_format": "md" + }, + "model": "reducto/parse-v3" + }, + "provider_responses": [ + { + "body_b64": "eyJmaWxlX2lkIjoicmVkdWN0bzovL2YzY2MzYzcyLTYxNGEtNDEwNC1iYjcwLWE1MTZmMzcxNDhlMC5wZGYiLCJwcmVzaWduZWRfdXJsIjpudWxsfQ==", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:30 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ], + "kind": "http", + "status_code": 200 + }, + { + "body_b64": "eyJyZXNwb25zZV90eXBlIjoicGFyc2UiLCJqb2JfaWQiOiJhYmE5MjRkYy03NzczLTRkZmYtYTRhOC0zNTQ5MzY2ZWE5ZmEiLCJkdXJhdGlvbiI6NC41NzY0MTcyMDc3MTc4OTU1LCJwZGZfdXJsIjoiaHR0cHM6Ly9wcm9kLXN0b3JhZ2UyMDI0MTAxMDE0NDc0NTE0MDkwMDAwMDAwMS5zMy5hbWF6b25hd3MuY29tL29yZy84YmQ5ODNkMS00N2RkLTQ0MGItOTNkOC00MzE0MTJkODE5YmYvZjNjYzNjNzItNjE0YS00MTA0LWJiNzAtYTUxNmYzNzE0OGUwLnBkZj9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUEyVU9LNk9WQk9VWUw3V1lBJTJGMjAyNjA5MDElMkZ1cy13ZXN0LTIlMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjYwOTAxVDE5NTQzNVomWC1BbXotRXhwaXJlcz00MzIwMCZYLUFtei1TaWduZWRIZWFkZXJzPWhvc3QmWC1BbXotU2lnbmF0dXJlPTdkNGM5NDY0YmIzYTFhOWQ1YzdhNWY5NTdiYzEwODEzZmIxNmZjYzZkZTNmZWEyMDU0YTQwYjQ2OWZmYTc5YjMiLCJzdHVkaW9fbGluayI6Imh0dHBzOi8vc3R1ZGlvLnJlZHVjdG8uYWkvam9iL2FiYTkyNGRjLTc3NzMtNGRmZi1hNGE4LTM1NDkzNjZlYTlmYSIsInVzYWdlIjp7Im51bV9wYWdlcyI6MSwiY3JlZGl0cyI6MS4wLCJjcmVkaXRfYnJlYWtkb3duIjp7InBhZ2UiOjEuMH0sInBhZ2VfYmlsbGluZ19icmVha2Rvd24iOnsiMSI6WyJwYWdlIl19LCJub25fZW1wdHlfY2VsbF9jb3VudCI6bnVsbH0sInJlc3VsdCI6eyJ0eXBlIjoiZnVsbCIsImNodW5rcyI6W3siY29udGVudCI6IiMgVGVzdCBQREYgRmlsZSIsImVtYmVkIjoiIyBUZXN0IFBERiBGaWxlIiwiZW5yaWNoZWQiOm51bGwsImVucmljaG1lbnRfc3VjY2VzcyI6ZmFsc2UsImJsb2NrcyI6W3sidHlwZSI6IlRpdGxlIiwiYmJveCI6eyJsZWZ0IjowLjE2MTg5NTc0NTI3MTgyNzQ4LCJ0b3AiOjAuMDk0NzkyOTU1NTM4MTQxMjEsIndpZHRoIjowLjExOTQ2MjkwNjUwODIwODc1LCJoZWlnaHQiOjAuMDIxNDY0MTQzMDE3MTA1MDcsInBhZ2UiOjEsIm9yaWdpbmFsX3BhZ2UiOjF9LCJjb250ZW50IjoiVGVzdCBQREYgRmlsZSIsImltYWdlX3VybCI6bnVsbCwiY2hhcnRfZGF0YSI6bnVsbCwiY29uZmlkZW5jZSI6ImhpZ2giLCJncmFudWxhcl9jb25maWRlbmNlIjp7ImV4dHJhY3RfY29uZmlkZW5jZSI6bnVsbCwicGFyc2VfY29uZmlkZW5jZSI6MC43MTg1MzU1NjkzMTAxODgyfSwiZXh0cmEiOm51bGx9XX1dLCJvY3IiOm51bGwsImN1c3RvbSI6bnVsbH0sInBhcnNlX21vZGUiOm51bGwsImRvY3VtZW50X3Byb3BlcnRpZXMiOm51bGx9", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:35 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:35.993066Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/fixtures/reducto-v3/d5a49e5897e920719ca28697413ee0396ea754023bd5ab34065efcc852914f15.json b/tests/test_litellm/ocr/fixtures/reducto-v3/d5a49e5897e920719ca28697413ee0396ea754023bd5ab34065efcc852914f15.json new file mode 100644 index 00000000000..2253db90a1f --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/reducto-v3/d5a49e5897e920719ca28697413ee0396ea754023bd5ab34065efcc852914f15.json @@ -0,0 +1,46 @@ +{ + "case": { + "litellm_input": { + "custom_llm_provider": null, + "document": { + "document_url": "data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg==", + "type": "document_url" + }, + "model": "reducto/parse-v3" + }, + "provider_responses": [ + { + "body_b64": "eyJmaWxlX2lkIjoicmVkdWN0bzovLzZhYjliY2Y0LTE4OTMtNGIzNy1iZGRiLWFjZGE4Y2U0NWRmYi5wZGYiLCJwcmVzaWduZWRfdXJsIjpudWxsfQ==", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:44 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ], + "kind": "http", + "status_code": 200 + }, + { + "body_b64": "eyJyZXNwb25zZV90eXBlIjoicGFyc2UiLCJqb2JfaWQiOiJkMzQ1NTQ5Zi01ZDk4LTRjNjItYjNlMy0yNGM5OTAxNDRkZjQiLCJkdXJhdGlvbiI6MS4xMTk0NDI3MDEzMzk3MjE3LCJwZGZfdXJsIjoiaHR0cHM6Ly9wcm9kLXN0b3JhZ2UyMDI0MTAxMDE0NDc0NTE0MDkwMDAwMDAwMS5zMy5hbWF6b25hd3MuY29tL29yZy84YmQ5ODNkMS00N2RkLTQ0MGItOTNkOC00MzE0MTJkODE5YmYvNmFiOWJjZjQtMTg5My00YjM3LWJkZGItYWNkYThjZTQ1ZGZiLnBkZj9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUEyVU9LNk9WQk9VWUw3V1lBJTJGMjAyNjA5MDElMkZ1cy13ZXN0LTIlMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjYwOTAxVDE5NTQ0NVomWC1BbXotRXhwaXJlcz00MzIwMCZYLUFtei1TaWduZWRIZWFkZXJzPWhvc3QmWC1BbXotU2lnbmF0dXJlPTFkYjYxZDMzZjYwMmYxM2U4NDQ0ZTg1ODBiNzM3YjMxZjM0NmU0NzQ5M2NjN2VjYTdjNGZjZGRiZGUwZGZiNDMiLCJzdHVkaW9fbGluayI6Imh0dHBzOi8vc3R1ZGlvLnJlZHVjdG8uYWkvam9iL2QzNDU1NDlmLTVkOTgtNGM2Mi1iM2UzLTI0Yzk5MDE0NGRmNCIsInVzYWdlIjp7Im51bV9wYWdlcyI6MSwiY3JlZGl0cyI6MS4wLCJjcmVkaXRfYnJlYWtkb3duIjp7InBhZ2UiOjEuMH0sInBhZ2VfYmlsbGluZ19icmVha2Rvd24iOnsiMSI6WyJwYWdlIl19LCJub25fZW1wdHlfY2VsbF9jb3VudCI6bnVsbH0sInJlc3VsdCI6eyJ0eXBlIjoiZnVsbCIsImNodW5rcyI6W3siY29udGVudCI6IiMgVGVzdCBQREYgRmlsZSIsImVtYmVkIjoiIyBUZXN0IFBERiBGaWxlIiwiZW5yaWNoZWQiOm51bGwsImVucmljaG1lbnRfc3VjY2VzcyI6ZmFsc2UsImJsb2NrcyI6W3sidHlwZSI6IlRpdGxlIiwiYmJveCI6eyJsZWZ0IjowLjE2MTg5NTc0NTI3MTgyNzQ4LCJ0b3AiOjAuMDk0NzkyOTU1NTM4MTQxMjEsIndpZHRoIjowLjExOTQ2MjkwNjUwODIwODc1LCJoZWlnaHQiOjAuMDIxNDY0MTQzMDE3MTA1MDcsInBhZ2UiOjEsIm9yaWdpbmFsX3BhZ2UiOjF9LCJjb250ZW50IjoiVGVzdCBQREYgRmlsZSIsImltYWdlX3VybCI6bnVsbCwiY2hhcnRfZGF0YSI6bnVsbCwiY29uZmlkZW5jZSI6ImhpZ2giLCJncmFudWxhcl9jb25maWRlbmNlIjp7ImV4dHJhY3RfY29uZmlkZW5jZSI6bnVsbCwicGFyc2VfY29uZmlkZW5jZSI6MC43MTg1MzU1NjkzMTAxODgyfSwiZXh0cmEiOm51bGx9XX1dLCJvY3IiOm51bGwsImN1c3RvbSI6bnVsbH0sInBhcnNlX21vZGUiOm51bGwsImRvY3VtZW50X3Byb3BlcnRpZXMiOm51bGx9", + "headers": [ + { + "name": "Date", + "value": "Tue, 01 Sep 2026 19:54:46 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ], + "kind": "http", + "status_code": 200 + } + ] + }, + "recorded_at": "2026-09-01T19:54:46.694836Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/generate_fixtures.py b/tests/test_litellm/ocr/generate_fixtures.py index 5ec498eca56..82533aecf20 100644 --- a/tests/test_litellm/ocr/generate_fixtures.py +++ b/tests/test_litellm/ocr/generate_fixtures.py @@ -1,5 +1,6 @@ from __future__ import annotations +import base64 import logging import os from collections.abc import Callable, Mapping @@ -26,8 +27,11 @@ from tests.route_parity.fixture_recorder import ( fixture_directory, ) from tests.test_litellm.ocr.fixture_models import ( + AzureDocumentIntelligenceOcrSdkInput, + AzureMistralOcrSdkInput, JsonSchemaDefinition, JsonSchemaResponseFormat, + MistralDocumentUrlDocument, MistralImageUrlDocument, MistralOcrSdkInput, OcrParityCase, @@ -39,13 +43,14 @@ from tests.test_litellm.ocr.fixture_models import ( ReductoParseV3SdkInput, ReductoRetrieval, ReductoSettings, + VertexDeepSeekOcrSdkInput, + VertexMistralOcrSdkInput, ) FIXTURE_DIR_ENV: Final = "LITELLM_OCR_FIXTURE_DIR" -_TEXT: Final = st.just("invoice 123") _VALUE_TEXT: Final = st.just("case-1") -_FONT_SIZE: Final = st.just(24) _MISTRAL_MODEL: Final = "mistral/mistral-ocr-latest" +_REDUCTO_API_BASE: Final = "https://platform.reducto.ai" OcrFixtureTarget = FixtureTarget[OcrSdkInputBase] @@ -56,6 +61,20 @@ def _image_document(text: str, font_size: int) -> MistralImageUrlDocument: return MistralImageUrlDocument(type="image_url", image_url=url) +def _fixture_pdf_data_uri() -> str: + fixture: Final = Path(__file__).resolve().parents[2] / "llm_translation" / "fixtures" / "dummy.pdf" + encoded: Final = base64.b64encode(fixture.read_bytes()).decode("ascii") + return f"data:application/pdf;base64,{encoded}" + + +def _pdf_document() -> MistralDocumentUrlDocument: + return MistralDocumentUrlDocument(type="document_url", document_url=_fixture_pdf_data_uri()) + + +def _public_document_strategy() -> SearchStrategy[MistralImageUrlDocument | MistralDocumentUrlDocument]: + return st.sampled_from((_image_document("invoice 123", 24), _pdf_document())) + + def _annotation_format(name: str) -> JsonSchemaResponseFormat: return JsonSchemaResponseFormat( type="json_schema", @@ -79,7 +98,7 @@ def _mistral_confidence_strategy() -> SearchStrategy[str]: @st.composite def mistral_input_strategy(draw: DrawFn, model: str) -> MistralOcrSdkInput: - document: Final = draw(st.builds(_image_document, _TEXT, _FONT_SIZE)) + document: Final = draw(_public_document_strategy()) annotation: Final = draw( st.sampled_from( ( @@ -120,6 +139,145 @@ def mistral_input_strategy(draw: DrawFn, model: str) -> MistralOcrSdkInput: ) +def _as_azure_mistral(case_input: MistralOcrSdkInput, model: str) -> AzureMistralOcrSdkInput: + values: Final = case_input.model_dump(mode="python", exclude={"boundary", "model", "custom_llm_provider"}) + return AzureMistralOcrSdkInput.model_validate({**values, "model": model}) + + +def _as_vertex_mistral(case_input: MistralOcrSdkInput, project: str, location: str) -> VertexMistralOcrSdkInput: + values: Final = case_input.model_dump(mode="python", exclude={"boundary", "model", "custom_llm_provider"}) + return VertexMistralOcrSdkInput.model_validate( + {**values, "vertex_project": project, "vertex_location": location} + ) + + +def _required_mistral_inputs(model: str) -> tuple[MistralOcrSdkInput, ...]: + document: Final = _image_document("invoice 123", 24) + annotation: Final = _annotation_format("document_title") + bbox_annotation: Final = _annotation_format("bounding_boxes") + cases: Final[tuple[dict[str, object], ...]] = ( + {}, + {"pages": [0]}, + {"include_image_base64": True}, + {"image_limit": 1}, + {"image_min_size": 300}, + {"bbox_annotation_format": bbox_annotation}, + {"document_annotation_format": annotation}, + {"document_annotation_format": annotation, "document_annotation_prompt": "Extract the visible title"}, + {"extract_header": True}, + {"extract_footer": True}, + {"table_format": "markdown"}, + {"confidence_scores_granularity": "page"}, + {"include_blocks": False}, + {"id": "case-1"}, + ) + return tuple( + MistralOcrSdkInput.model_validate({"model": model, "document": document, **case}) for case in cases + ) + + +def _required_azure_document_intelligence_inputs() -> tuple[AzureDocumentIntelligenceOcrSdkInput, ...]: + document: Final = _pdf_document() + model: Final = "azure_ai/doc-intelligence/prebuilt-layout" + return ( + AzureDocumentIntelligenceOcrSdkInput(model=model, document=document), + AzureDocumentIntelligenceOcrSdkInput(model=model, document=document, pages=[0, 1]), + AzureDocumentIntelligenceOcrSdkInput(model=model, document=document, features=["languages"]), + AzureDocumentIntelligenceOcrSdkInput(model=model, document=document, req_format="litellm"), + ) + + +def _required_vertex_deepseek_inputs(project: str, location: str) -> tuple[VertexDeepSeekOcrSdkInput, ...]: + document: Final = _image_document("invoice 123", 24) + common: Final = {"document": document, "vertex_project": project, "vertex_location": location} + cases: Final[tuple[dict[str, object], ...]] = ( + {}, + {"stream": False}, + {"temperature": 0.5}, + {"max_tokens": 256}, + {"top_p": 0.9}, + {"n": 1}, + {"stop": ["END", "STOP"]}, + ) + return tuple(VertexDeepSeekOcrSdkInput.model_validate({**common, **case}) for case in cases) + + +def _required_reducto_v3_inputs( + document: ReductoDocumentUrlDocument, +) -> tuple[ReductoParseV3SdkInput, ...]: + return ( + ReductoParseV3SdkInput(model="reducto/parse-v3", document=document), + ReductoParseV3SdkInput( + model="reducto/parse-v3", + document=document, + formatting=ReductoFormatting(table_output_format="md"), + ), + ReductoParseV3SdkInput( + model="reducto/parse-v3", + document=document, + retrieval=ReductoRetrieval(chunking=ReductoChunking(chunk_mode="page")), + ), + ReductoParseV3SdkInput( + model="reducto/parse-v3", + document=document, + settings=ReductoSettings(return_ocr_data=True), + ), + ) + + +@st.composite +def azure_document_intelligence_input_strategy(draw: DrawFn) -> AzureDocumentIntelligenceOcrSdkInput: + optional_params: Final = draw( + st.fixed_dictionaries( + {}, + optional={ + "pages": st.sampled_from(([0], [0, 1], "1-2")), + "features": st.sampled_from((["languages"], ["keyValuePairs"], "languages,keyValuePairs")), + }, + ) + ) + return AzureDocumentIntelligenceOcrSdkInput.model_validate( + { + "model": draw( + st.sampled_from( + ( + "azure_ai/doc-intelligence/prebuilt-read", + "azure_ai/doc-intelligence/prebuilt-layout", + "azure_ai/doc-intelligence/prebuilt-document", + ) + ) + ), + "document": draw(_public_document_strategy()), + **optional_params, + } + ) + + +@st.composite +def vertex_deepseek_input_strategy(draw: DrawFn, project: str, location: str) -> VertexDeepSeekOcrSdkInput: + optional_params: Final = draw( + st.fixed_dictionaries( + {}, + optional={ + "stream": st.just(False), + "temperature": st.sampled_from((0.0, 0.5, 1.0)), + "max_tokens": st.sampled_from((1, 256, 1024)), + "top_p": st.sampled_from((0.1, 0.9, 1.0)), + "n": st.just(1), + "stop": st.sampled_from(("END", ["END", "STOP"])), + }, + ) + ) + return VertexDeepSeekOcrSdkInput.model_validate( + { + "document": draw(_public_document_strategy()), + "vertex_project": project, + "vertex_location": location, + **optional_params, + } + ) + + def _reducto_formatting_strategy() -> SearchStrategy[ReductoFormatting]: return st.builds( ReductoFormatting, @@ -175,7 +333,7 @@ def _reducto_settings_strategy() -> SearchStrategy[ReductoSettings]: ReductoSettings, ocr_system=st.sampled_from(("standard", "legacy")), extraction_mode=st.sampled_from(("ocr", "hybrid")), - force_url_result=st.booleans(), + force_url_result=st.just(False), return_ocr_data=st.booleans(), return_images=st.sampled_from(([], ["figure"], ["table"], ["page"], ["figure", "table", "page"])), embed_pdf_metadata=st.booleans(), @@ -187,7 +345,9 @@ def _reducto_settings_strategy() -> SearchStrategy[ReductoSettings]: @st.composite -def reducto_v3_input_strategy(draw: DrawFn) -> ReductoParseV3SdkInput: +def reducto_v3_input_strategy( + draw: DrawFn, document: ReductoDocumentUrlDocument | None = None +) -> ReductoParseV3SdkInput: model, custom_llm_provider = draw(st.sampled_from((("reducto/parse-v3", None), ("parse-v3", "reducto")))) options: Final = draw( st.fixed_dictionaries( @@ -203,32 +363,34 @@ def reducto_v3_input_strategy(draw: DrawFn) -> ReductoParseV3SdkInput: { "model": model, "custom_llm_provider": custom_llm_provider, - "document": ReductoDocumentUrlDocument( - type="document_url", - document_url="reducto://fixture-document.pdf", - ), + "document": document + or ReductoDocumentUrlDocument(type="document_url", document_url="reducto://fixture-document.pdf"), **options, } ) -def reducto_legacy_input_strategy() -> SearchStrategy[ReductoParseLegacySdkInput]: +def reducto_legacy_input_strategy( + document: ReductoDocumentUrlDocument | None = None, +) -> SearchStrategy[ReductoParseLegacySdkInput]: + selected_document: Final = document or ReductoDocumentUrlDocument( + type="document_url", document_url="reducto://fixture-document.pdf" + ) return st.sampled_from( ( ReductoParseLegacySdkInput( model="reducto/parse-legacy", - document=ReductoDocumentUrlDocument( - type="document_url", - document_url="reducto://fixture-document.pdf", - ), + document=selected_document, ), ReductoParseLegacySdkInput( model="parse-legacy", custom_llm_provider="reducto", - document=ReductoDocumentUrlDocument( - type="document_url", - document_url="reducto://fixture-document.pdf", - ), + document=selected_document, + ), + ReductoParseLegacySdkInput( + model="reducto/parse-legacy", + document=selected_document, + enhance={}, ), ) ) @@ -264,6 +426,142 @@ def _mistral_target( provider_spec=ProviderSpec(upstream_base=_mistral_upstream_base(environ)), strategy=cast(SearchStrategy[OcrSdkInputBase], mistral_input_strategy(_MISTRAL_MODEL)), invoke=invoke, + required_inputs=cast(tuple[OcrSdkInputBase, ...], _required_mistral_inputs(_MISTRAL_MODEL)), + ) + + +def _azure_mistral_target( + environ: Mapping[str, str], sdk_call: Callable[..., object] +) -> OcrFixtureTarget | None: + api_key: Final = environ.get("AZURE_AI_API_KEY") + upstream_base: Final = environ.get("AZURE_AI_API_BASE") + configured_model: Final = environ.get("AZURE_AI_OCR_MODEL") + if not api_key or not upstream_base or not configured_model: + return None + model: Final = configured_model if configured_model.startswith("azure_ai/") else f"azure_ai/{configured_model}" + + def invoke(api_base: str, case_input: OcrSdkInputBase) -> object: + return sdk_call(api_base=api_base, api_key=api_key, **case_input.as_sdk_kwargs()) + + return OcrFixtureTarget( + name="azure-mistral", + provider_spec=ProviderSpec(upstream_base=upstream_base.rstrip("/")), + strategy=cast( + SearchStrategy[OcrSdkInputBase], + mistral_input_strategy(_MISTRAL_MODEL).map(lambda case_input: _as_azure_mistral(case_input, model)), + ), + invoke=invoke, + required_inputs=cast( + tuple[OcrSdkInputBase, ...], + tuple(_as_azure_mistral(case_input, model) for case_input in _required_mistral_inputs(_MISTRAL_MODEL)), + ), + ) + + +def _azure_document_intelligence_target( + environ: Mapping[str, str], sdk_call: Callable[..., object] +) -> OcrFixtureTarget | None: + api_key: Final = environ.get("AZURE_DOCUMENT_INTELLIGENCE_API_KEY") + upstream_base: Final = environ.get("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT") + if not api_key or not upstream_base: + return None + + def invoke(api_base: str, case_input: OcrSdkInputBase) -> object: + return sdk_call(api_base=api_base, api_key=api_key, **case_input.as_sdk_kwargs()) + + return OcrFixtureTarget( + name="azure-document-intelligence", + provider_spec=ProviderSpec(upstream_base=upstream_base.rstrip("/")), + strategy=cast(SearchStrategy[OcrSdkInputBase], azure_document_intelligence_input_strategy()), + invoke=invoke, + required_inputs=cast( + tuple[OcrSdkInputBase, ...], _required_azure_document_intelligence_inputs() + ), + ) + + +def _vertex_targets( + environ: Mapping[str, str], sdk_call: Callable[..., object] +) -> tuple[OcrFixtureTarget, ...]: + api_key: Final = environ.get("VERTEX_AI_API_KEY") + project: Final = environ.get("VERTEXAI_PROJECT") or environ.get("VERTEX_PROJECT") + location: Final = environ.get("VERTEXAI_LOCATION") or environ.get("VERTEX_LOCATION") or "us-central1" + if not api_key or not project: + return () + upstream_base: Final = environ.get("VERTEX_AI_API_BASE") or f"https://{location}-aiplatform.googleapis.com" + + def invoke(api_base: str, case_input: OcrSdkInputBase) -> object: + return sdk_call(api_base=api_base, api_key=api_key, **case_input.as_sdk_kwargs()) + + return ( + OcrFixtureTarget( + name="vertex-mistral", + provider_spec=ProviderSpec(upstream_base=upstream_base.rstrip("/")), + strategy=cast( + SearchStrategy[OcrSdkInputBase], + st.builds( + _as_vertex_mistral, + case_input=mistral_input_strategy(_MISTRAL_MODEL), + project=st.just(project), + location=st.just(location), + ), + ), + invoke=invoke, + required_inputs=cast( + tuple[OcrSdkInputBase, ...], + tuple( + _as_vertex_mistral(case_input, project, location) + for case_input in _required_mistral_inputs(_MISTRAL_MODEL) + ), + ), + ), + OcrFixtureTarget( + name="vertex-deepseek", + provider_spec=ProviderSpec(upstream_base=upstream_base.rstrip("/")), + strategy=cast( + SearchStrategy[OcrSdkInputBase], vertex_deepseek_input_strategy(project, location) + ), + invoke=invoke, + required_inputs=cast( + tuple[OcrSdkInputBase, ...], _required_vertex_deepseek_inputs(project, location) + ), + ), + ) + + +def _reducto_targets( + environ: Mapping[str, str], sdk_call: Callable[..., object] +) -> tuple[OcrFixtureTarget, ...]: + api_key: Final = environ.get("REDUCTO_API_KEY") + if not api_key: + return () + upstream_base: Final = environ.get("REDUCTO_API_BASE", _REDUCTO_API_BASE).rstrip("/") + document: Final = ReductoDocumentUrlDocument(type="document_url", document_url=_fixture_pdf_data_uri()) + + def invoke(api_base: str, case_input: OcrSdkInputBase) -> object: + return sdk_call(api_base=api_base, api_key=api_key, **case_input.as_sdk_kwargs()) + + return ( + OcrFixtureTarget( + name="reducto-v3", + provider_spec=ProviderSpec(upstream_base=upstream_base), + strategy=cast(SearchStrategy[OcrSdkInputBase], reducto_v3_input_strategy(document)), + invoke=invoke, + required_inputs=cast(tuple[OcrSdkInputBase, ...], _required_reducto_v3_inputs(document)), + ), + OcrFixtureTarget( + name="reducto-legacy", + provider_spec=ProviderSpec(upstream_base=upstream_base), + strategy=cast(SearchStrategy[OcrSdkInputBase], reducto_legacy_input_strategy(document)), + invoke=invoke, + required_inputs=cast( + tuple[OcrSdkInputBase, ...], + ( + ReductoParseLegacySdkInput(model="reducto/parse-legacy", document=document), + ReductoParseLegacySdkInput(model="reducto/parse-legacy", document=document, enhance={}), + ), + ), + ), ) @@ -271,14 +569,19 @@ def discover_targets( environ: Mapping[str, str], sdk_call: Callable[..., object], ) -> tuple[OcrFixtureTarget, ...]: - candidates: Final = (_mistral_target(environ, sdk_call),) - return tuple(target for target in candidates if target is not None) + optional_targets: Final = ( + _mistral_target(environ, sdk_call), + _azure_mistral_target(environ, sdk_call), + _azure_document_intelligence_target(environ, sdk_call), + ) + direct_targets: Final = tuple(target for target in optional_targets if target is not None) + return (*direct_targets, *_vertex_targets(environ, sdk_call), *_reducto_targets(environ, sdk_call)) def require_targets(targets: tuple[OcrFixtureTarget, ...]) -> tuple[OcrFixtureTarget, ...]: return require_fixture_targets( targets, - "No OCR fixture providers are configured. Set MISTRAL_API_KEY", + "No OCR fixture providers are configured. Set a supported provider API key and endpoint", ) diff --git a/tests/test_litellm/ocr/test_fixture_models.py b/tests/test_litellm/ocr/test_fixture_models.py index 70966e47bda..e44dd5c573c 100644 --- a/tests/test_litellm/ocr/test_fixture_models.py +++ b/tests/test_litellm/ocr/test_fixture_models.py @@ -10,7 +10,10 @@ from pydantic import ValidationError from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig from litellm.llms.mistral.ocr.transformation import MistralOCRConfig from litellm.llms.reducto.ocr.transformation import ReductoParseLegacyConfig, ReductoParseV3Config +from litellm.llms.vertex_ai.ocr.deepseek_transformation import VertexAIDeepSeekOCRConfig from tests.test_litellm.ocr.fixture_models import ( + AzureDocumentIntelligenceOcrSdkInput, + AzureMistralOcrSdkInput, JsonSchemaDefinition, JsonSchemaResponseFormat, MistralDocumentUrlDocument, @@ -26,14 +29,20 @@ from tests.test_litellm.ocr.fixture_models import ( ReductoParseV3SdkInput, ReductoRetrieval, ReductoSettings, + VertexDeepSeekOcrSdkInput, + VertexMistralOcrSdkInput, ) from tests.test_litellm.ocr.generate_fixtures import ( + azure_document_intelligence_input_strategy, mistral_input_strategy, reducto_legacy_input_strategy, reducto_v3_input_strategy, + vertex_deepseek_input_strategy, ) -COMMON_FIELDS: Final = frozenset({"model", "document", "custom_llm_provider"}) +COMMON_FIELDS: Final = frozenset( + {"boundary", "model", "document", "custom_llm_provider", "vertex_project", "vertex_location"} +) def _provider_fields(model: type[OcrSdkInputBase]) -> set[str]: @@ -71,6 +80,65 @@ def test_reducto_fixture_fields_match_provider_configs() -> None: assert _provider_fields(ReductoParseLegacySdkInput) == _supported_params(ReductoParseLegacyConfig(), "parse-legacy") +def test_deepseek_fixture_fields_match_provider_config() -> None: + assert _provider_fields(VertexDeepSeekOcrSdkInput) == _supported_params( + VertexAIDeepSeekOCRConfig(), "deepseek-ai/deepseek-ocr-maas" + ) + + +def test_deepseek_maps_litellm_params_without_duplicating_model_namespace() -> None: + config: Final = VertexAIDeepSeekOCRConfig() + optional_params: Final = config.map_ocr_params( + non_default_params={"temperature": 0.5, "max_tokens": 256}, + optional_params={}, + model="deepseek-ai/deepseek-ocr-maas", + ) + request: Final = config.transform_ocr_request( + model="deepseek-ai/deepseek-ocr-maas", + document={"type": "image_url", "image_url": "gs://bucket/document.png"}, + optional_params=optional_params, + headers={}, + ) + + assert request.data == { + "model": "deepseek-ai/deepseek-ocr-maas", + "messages": [ + { + "role": "user", + "content": [{"type": "image_url", "image_url": "gs://bucket/document.png"}], + } + ], + "temperature": 0.5, + "max_tokens": 256, + } + + +@pytest.mark.parametrize( + "sdk_input", + ( + AzureMistralOcrSdkInput( + model="azure_ai/mistral-ocr-deployment", + document=MistralImageUrlDocument(type="image_url", image_url="data:image/png;base64,AA=="), + ), + VertexMistralOcrSdkInput( + document=MistralImageUrlDocument(type="image_url", image_url="data:image/png;base64,AA=="), + vertex_project="project-1", + ), + AzureDocumentIntelligenceOcrSdkInput( + model="azure_ai/doc-intelligence/prebuilt-layout", + document=MistralImageUrlDocument(type="image_url", image_url="data:image/png;base64,AA=="), + ), + VertexDeepSeekOcrSdkInput( + document=MistralImageUrlDocument(type="image_url", image_url="data:image/png;base64,AA=="), + vertex_project="project-1", + ), + ), +) +def test_provider_boundary_is_explicit_but_not_forwarded(sdk_input: OcrSdkInputBase) -> None: + assert sdk_input.canonical_input()["boundary"] == sdk_input.boundary + assert "boundary" not in sdk_input.as_sdk_kwargs() + + def test_mistral_input_preserves_omission_and_explicit_boolean_values() -> None: omitted: Final = _mistral_input().as_sdk_kwargs() explicit: Final = _mistral_input(extract_header=False, include_blocks=True).as_sdk_kwargs() @@ -227,5 +295,23 @@ def test_reducto_v3_strategy_only_generates_valid_sdk_inputs(sdk_input: ReductoP @settings(max_examples=10, deadline=None) @given(sdk_input=reducto_legacy_input_strategy()) -def test_reducto_legacy_strategy_omits_undocumented_enhance(sdk_input: ReductoParseLegacySdkInput) -> None: - assert "enhance" not in sdk_input.as_sdk_kwargs() +def test_reducto_legacy_strategy_generates_valid_litellm_inputs(sdk_input: ReductoParseLegacySdkInput) -> None: + assert ReductoParseLegacySdkInput.model_validate(sdk_input.canonical_input()) == sdk_input + + +@settings(max_examples=30, deadline=None) +@given(sdk_input=azure_document_intelligence_input_strategy()) +def test_azure_document_intelligence_strategy_only_generates_litellm_inputs( + sdk_input: AzureDocumentIntelligenceOcrSdkInput, +) -> None: + assert sdk_input.req_format == "litellm" + assert "boundary" not in sdk_input.as_sdk_kwargs() + + +@settings(max_examples=30, deadline=None) +@given(sdk_input=vertex_deepseek_input_strategy("project-1", "us-central1")) +def test_vertex_deepseek_strategy_only_generates_litellm_inputs( + sdk_input: VertexDeepSeekOcrSdkInput, +) -> None: + assert sdk_input.vertex_project == "project-1" + assert "boundary" not in sdk_input.as_sdk_kwargs() diff --git a/tests/test_litellm/ocr/test_generate_fixtures.py b/tests/test_litellm/ocr/test_generate_fixtures.py index ec1a4509ff6..83f867e86ca 100644 --- a/tests/test_litellm/ocr/test_generate_fixtures.py +++ b/tests/test_litellm/ocr/test_generate_fixtures.py @@ -41,10 +41,51 @@ def test_discovery_requires_provider_specific_key(environ: dict[str, str]) -> No def test_no_discovered_targets_has_actionable_error() -> None: - with pytest.raises(SystemExit, match="Set MISTRAL_API_KEY"): + with pytest.raises(SystemExit, match="supported provider API key"): require_targets(()) +def test_discovery_is_explicit_per_available_provider_boundary() -> None: + targets: Final = discover_targets( + { + "MISTRAL_API_KEY": "mistral-secret", + "REDUCTO_API_KEY": "reducto-secret", + "AZURE_AI_API_KEY": "azure-secret", + "AZURE_AI_API_BASE": "https://azure.example", + "AZURE_AI_OCR_MODEL": "mistral-ocr-deployment", + "AZURE_DOCUMENT_INTELLIGENCE_API_KEY": "document-secret", + "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT": "https://document.example", + "VERTEX_AI_API_KEY": "vertex-secret", + "VERTEXAI_PROJECT": "project-1", + }, + _unused_sdk_call, + ) + + assert tuple(target.name for target in targets) == ( + "mistral-ocr", + "azure-mistral", + "azure-document-intelligence", + "vertex-mistral", + "vertex-deepseek", + "reducto-v3", + "reducto-legacy", + ) + assert all("secret" not in repr(target) for target in targets) + + +def test_azure_mistral_discovery_requires_and_normalizes_deployment_model() -> None: + incomplete: Final = { + "AZURE_AI_API_KEY": "azure-secret", + "AZURE_AI_API_BASE": "https://azure.example", + } + assert discover_targets(incomplete, _unused_sdk_call) == () + + target: Final = discover_targets( + {**incomplete, "AZURE_AI_OCR_MODEL": "mistral-ocr-deployment"}, _unused_sdk_call + )[0] + assert target.required_inputs[0].model == "azure_ai/mistral-ocr-deployment" + + @pytest.mark.parametrize( ("configured", "expected"), ( @@ -71,6 +112,28 @@ def test_mistral_target_uses_canonical_model_and_normalized_base( case_inputs: Final = generate_case_inputs(target.strategy, examples=1) assert len(case_inputs) == 1 assert case_inputs[0].canonical_input()["model"] == "mistral/mistral-ocr-latest" + assert len(target.required_inputs) == 14 + covered_params: Final = { + key + for case_input in target.required_inputs + for key in case_input.as_sdk_kwargs() + if key not in {"model", "document", "custom_llm_provider"} + } + assert covered_params == { + "pages", + "include_image_base64", + "image_limit", + "image_min_size", + "bbox_annotation_format", + "document_annotation_format", + "document_annotation_prompt", + "extract_header", + "extract_footer", + "table_format", + "confidence_scores_granularity", + "include_blocks", + "id", + } def test_mistral_target_invocation_forwards_discovered_credentials() -> None: diff --git a/tests/test_litellm/ocr/test_sdk_parity.py b/tests/test_litellm/ocr/test_sdk_parity.py index 97d836e849c..2f4d16460a8 100644 --- a/tests/test_litellm/ocr/test_sdk_parity.py +++ b/tests/test_litellm/ocr/test_sdk_parity.py @@ -28,7 +28,7 @@ from tests.route_parity.runner import ( parity_worker_main, run_execution, ) -from tests.test_litellm.ocr.fixture_models import MistralOcrSdkInput, OcrParityCase +from tests.test_litellm.ocr.fixture_models import OcrParityCase, OcrSdkInput API_KEY: Final = "test-key" PYTHON_HTTP_SENTINEL: Final = "python-ocr-parity-fallback" @@ -40,7 +40,7 @@ class SDKRoute(str, Enum): AOCR = "aocr" -def _call_kwargs(sdk_input: MistralOcrSdkInput, mock_url: str, route: SDKRoute) -> dict[str, object]: +def _call_kwargs(sdk_input: OcrSdkInput, mock_url: str, route: SDKRoute) -> dict[str, object]: return { **sdk_input.as_sdk_kwargs(), "api_base": mock_url, @@ -50,7 +50,7 @@ def _call_kwargs(sdk_input: MistralOcrSdkInput, mock_url: str, route: SDKRoute) def _execute_sdk_case( - sdk_input: MistralOcrSdkInput, + sdk_input: OcrSdkInput, route: SDKRoute, mock_url: str, event_loop: asyncio.AbstractEventLoop, @@ -67,7 +67,7 @@ def _execute_sdk_case( return SDKReport(response=async_response.model_dump(mode="json")) -def _call_sdk_case(sdk_input: MistralOcrSdkInput, route: SDKRoute, mock_url: str) -> OCRResponse: +def _call_sdk_case(sdk_input: OcrSdkInput, route: SDKRoute, mock_url: str) -> OCRResponse: import litellm call_kwargs: Final = _call_kwargs(sdk_input, mock_url, route) @@ -185,12 +185,14 @@ def test_recorded_ocr_sdk_parity( ocr_fixture: OcrParityCase, route: SDKRoute, ) -> None: + if ocr_fixture.litellm_input.boundary in {"reducto_v3", "reducto_legacy"}: + pytest.skip("Reducto does not have a Rust OCR boundary") sync_spy, async_spy = _native_spies() with _restore_rust_ocr_state(), replay_server() as provider: rust_ocr_bridge.use_litellm_rust(False, ocr=sync_spy, aocr=async_spy) python: Final = run_in_process( provider, - ocr_fixture.provider_response, + ocr_fixture.provider_responses, lambda mock_url: _call_sdk_case(ocr_fixture.litellm_input, route, mock_url), ) assert sync_spy.calls == 0 @@ -199,13 +201,13 @@ def test_recorded_ocr_sdk_parity( rust_ocr_bridge.use_litellm_rust(True, ocr=sync_spy, aocr=async_spy) rust: Final = run_in_process( provider, - ocr_fixture.provider_response, + ocr_fixture.provider_responses, lambda mock_url: _call_sdk_case(ocr_fixture.litellm_input, route, mock_url), ) assert sync_spy.calls == (1 if route is SDKRoute.OCR else 0) assert async_spy.calls == (1 if route is SDKRoute.AOCR else 0) - assert_request_parity(python.request, rust.request) + assert_request_parity(python.requests, rust.requests) assert_model_parity(python.response, rust.response) @@ -221,13 +223,13 @@ def test_ocr_subprocess_startup_smoke( python_worker, case_file, SDKRoute.OCR.value, - startup_ocr_fixture.provider_response, + startup_ocr_fixture.provider_responses, ) rust: Final = run_execution( rust_worker, case_file, SDKRoute.OCR.value, - startup_ocr_fixture.provider_response, + startup_ocr_fixture.provider_responses, ) assert_parity(python, rust, PYTHON_HTTP_SENTINEL)