From 64be0c18b97cf1aea6ad2edef323ee0981131ea7 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Sat, 29 Aug 2026 16:26:03 -0700 Subject: [PATCH] refactor(tests): generalize parity fixture recorder --- tests/test_litellm/_fixture_recorder.py | 177 +++++++++++++----- tests/test_litellm/_recorded_http.py | 63 +++++++ ...4a77b92d76edfd00e485fca93a0efbb82bda3.json | 93 +++++++++ ...6d274496e0065223342a76cc04e5ae4491a4c.json | 95 ---------- ...b0f431a36cc237657c498638a6c676bb6a28d.json | 90 --------- ...cb73438844c5eb7b4f5eae1a86ecce7b60742.json | 139 ++++++++++++++ ...1895ccc6bdbcf6f9fff3aab098998c7bc538d.json | 87 --------- ...e0a149493bcf609378295206bf5719a369059.json | 88 --------- ...910adf782ab9194e309d3c62fe0f2fadc326c.json | 92 +++++++++ ...f788cf6f2b3d5f4fdae5d7b9b620c398f47dc.json | 117 ++++++++++++ tests/test_litellm/ocr/conftest.py | 15 +- tests/test_litellm/ocr/fixture_models.py | 32 +--- tests/test_litellm/ocr/generate_fixtures.py | 43 ++--- tests/test_litellm/ocr/test_sdk_parity.py | 10 +- tests/test_litellm/parity/replay.py | 39 ++-- tests/test_litellm/parity/runner.py | 13 +- tests/test_litellm/test__fixture_recorder.py | 144 ++++++++++++-- 17 files changed, 826 insertions(+), 511 deletions(-) create mode 100644 tests/test_litellm/_recorded_http.py create mode 100644 tests/test_litellm/ocr/.fixtures/067fe064938269ae58ac14030064a77b92d76edfd00e485fca93a0efbb82bda3.json delete mode 100644 tests/test_litellm/ocr/.fixtures/099863d538eee61d30ece33c67b6d274496e0065223342a76cc04e5ae4491a4c.json delete mode 100644 tests/test_litellm/ocr/.fixtures/177537cc3eb91a26336033c6016b0f431a36cc237657c498638a6c676bb6a28d.json create mode 100644 tests/test_litellm/ocr/.fixtures/6aa18aa0bb41abea634cac9c77fcb73438844c5eb7b4f5eae1a86ecce7b60742.json delete mode 100644 tests/test_litellm/ocr/.fixtures/b6cee08db583cae6ed091afd4f81895ccc6bdbcf6f9fff3aab098998c7bc538d.json delete mode 100644 tests/test_litellm/ocr/.fixtures/c8702579eea0ba7fb7989604401e0a149493bcf609378295206bf5719a369059.json create mode 100644 tests/test_litellm/ocr/.fixtures/d7a77e2b36580d60ba22ef0a9cd910adf782ab9194e309d3c62fe0f2fadc326c.json create mode 100644 tests/test_litellm/ocr/.fixtures/ecfb251cbe6cb13e52dd11a4706f788cf6f2b3d5f4fdae5d7b9b620c398f47dc.json diff --git a/tests/test_litellm/_fixture_recorder.py b/tests/test_litellm/_fixture_recorder.py index e40a915e6f3..f791b08d13f 100644 --- a/tests/test_litellm/_fixture_recorder.py +++ b/tests/test_litellm/_fixture_recorder.py @@ -3,25 +3,31 @@ from __future__ import annotations import hashlib import queue import threading -from collections.abc import Callable, Generator +from collections.abc import Callable, Generator, Iterable from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from dataclasses import dataclass +from datetime import datetime, timezone from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path -from typing import Final, cast +from typing import Final, Generic, Protocol, TypeVar, cast import httpx -from pydantic import ValidationError +from hypothesis import given, settings +from hypothesis.strategies import SearchStrategy +from pydantic import AwareDatetime, BaseModel, ConfigDict, ValidationError from tests.test_litellm._json_fs_cache import JsonFileCache, canonical_json -from tests.test_litellm.ocr.fixture_models import ( +from tests.test_litellm._recorded_http import ( HttpHeader, - MistralOcrSdkInput, - OcrParityCase, RecordedHttpResponse, + RecordedHttpStreamResponse, + RecordedResponse, + RecordedStreamChunk, ) +FIXTURE_SCHEMA_VERSION: Final = 1 + _HOP_BY_HOP_HEADERS: Final = frozenset( { "connection", @@ -36,19 +42,33 @@ _HOP_BY_HOP_HEADERS: Final = frozenset( ) +class FixtureInput(Protocol): + def canonical_input(self) -> dict[str, object]: ... + + +InputT = TypeVar("InputT", bound=FixtureInput) +CaseT = TypeVar("CaseT", bound=BaseModel) + + @dataclass(frozen=True, slots=True) class ProviderSpec: - model: str upstream_base: str - api_key: str @dataclass(frozen=True, slots=True) -class RecorderResult: - case: OcrParityCase +class RecorderResult(Generic[CaseT]): + case: CaseT cache_hit: bool +class FixtureEnvelope(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + schema_version: int + recorded_at: AwareDatetime + case: dict[str, object] + + def _excluded_headers(headers: tuple[tuple[str, str], ...]) -> frozenset[str]: connection_values: Final = tuple(value for name, value in headers if name.lower() == "connection") connection_headers: Final = frozenset( @@ -69,13 +89,13 @@ class _RecordingProvider(ThreadingHTTPServer): def __init__(self, spec: ProviderSpec) -> None: super().__init__(("127.0.0.1", 0), _RecordingHandler) self.spec: Final = spec - self.responses: queue.Queue[RecordedHttpResponse] = queue.Queue() + self.responses: queue.Queue[RecordedResponse] = queue.Queue() @property def url(self) -> str: return f"http://127.0.0.1:{self.server_address[1]}" - def take_response(self) -> RecordedHttpResponse: + def take_response(self) -> RecordedResponse: try: return self.responses.get(timeout=5) except queue.Empty as error: @@ -103,22 +123,59 @@ class _RecordingHandler(BaseHTTPRequestHandler): content=request_body, timeout=120, ) as upstream: - response_body: Final = b"".join(upstream.iter_bytes()) - recorded_response: Final = RecordedHttpResponse.from_bytes( - status_code=upstream.status_code, - headers=_end_to_end_headers(upstream.headers), - body=response_body, - ) + headers: Final = _end_to_end_headers(upstream.headers) + recorded_response: Final = self._record_upstream_response(upstream, headers) except httpx.HTTPError as error: self._send_response(502, (), str(error).encode("utf-8")) return if 200 <= recorded_response.status_code < 300: provider.responses.put(recorded_response) - self._send_recorded_response(recorded_response) + if isinstance(recorded_response, RecordedHttpResponse): + self._send_response(recorded_response.status_code, recorded_response.headers, recorded_response.body_bytes()) - def _send_recorded_response(self, response: RecordedHttpResponse) -> None: - self._send_response(response.status_code, response.headers, response.body_bytes()) + def _record_upstream_response( + self, + upstream: httpx.Response, + headers: tuple[HttpHeader, ...], + ) -> RecordedResponse: + content_type: Final = cast(str, upstream.headers.get("content-type", "")) + if content_type.lower().startswith("text/event-stream"): + return self._record_stream(upstream, headers) + response_body: Final = b"".join(upstream.iter_bytes()) + return RecordedHttpResponse.from_bytes( + status_code=upstream.status_code, + headers=headers, + body=response_body, + ) + + def _record_stream( + self, + upstream: httpx.Response, + headers: tuple[HttpHeader, ...], + ) -> RecordedHttpStreamResponse: + self.send_response_only(upstream.status_code) + for header in headers: + self.send_header(header.name, header.value) + self.send_header("transfer-encoding", "chunked") + self.end_headers() + chunks: Final = tuple(self._relay_chunks(upstream.iter_bytes())) + self.wfile.write(b"0\r\n\r\n") + self.wfile.flush() + return RecordedHttpStreamResponse( + kind="http_stream", + status_code=upstream.status_code, + headers=headers, + chunks=chunks, + ) + + def _relay_chunks(self, chunks: Iterable[bytes]) -> Generator[RecordedStreamChunk, None, None]: + for chunk in chunks: + self.wfile.write(f"{len(chunk):X}\r\n".encode("ascii")) + self.wfile.write(chunk) + self.wfile.write(b"\r\n") + self.wfile.flush() + yield RecordedStreamChunk.from_bytes(chunk) def _send_response(self, status_code: int, headers: tuple[HttpHeader, ...], body: bytes) -> None: self.send_response_only(status_code) @@ -145,38 +202,72 @@ def _recording_provider(spec: ProviderSpec) -> Generator[_RecordingProvider]: thread.join(timeout=5) -def fixture_cache_key(case_input: MistralOcrSdkInput) -> dict[str, object]: +def generate_case_inputs(strategy: SearchStrategy[InputT], examples: int) -> tuple[InputT, ...]: + generated: Final[queue.SimpleQueue[InputT | None]] = queue.SimpleQueue() + + @settings(max_examples=examples, deadline=None, derandomize=True) + @given(case_input=strategy) + def generate_case(case_input: InputT) -> None: + generated.put(case_input) + + generate_case() + generated.put(None) + return tuple(iter(generated.get, None)) + + +def fixture_cache_key(case_input: FixtureInput) -> dict[str, object]: return case_input.canonical_input() +def _load_fixture(raw_fixture: dict[str, object], path: Path, case_type: type[CaseT]) -> CaseT: + schema_version: Final = raw_fixture.get("schema_version") + if schema_version != FIXTURE_SCHEMA_VERSION: + raise ValueError( + f"fixture {path} has schema_version {schema_version!r}, expected {FIXTURE_SCHEMA_VERSION}; " + "delete it and regenerate the fixture bundle" + ) + try: + envelope: Final = FixtureEnvelope.model_validate(raw_fixture) + return case_type.model_validate(envelope.case) + except ValidationError as error: + raise ValueError(f"invalid parity fixture {path} ({len(error.errors())} validation errors)") from error + + def record_case( spec: ProviderSpec, root: Path, - case_input: MistralOcrSdkInput, - sdk_call: Callable[..., object], -) -> RecorderResult: + case_input: InputT, + sdk_call: Callable[[str, InputT], object], + case_type: type[CaseT], +) -> RecorderResult[CaseT]: cache: Final = JsonFileCache(root) cache_key: Final = fixture_cache_key(case_input) cached: Final = cache.get(cache_key) if cached is not None: - return RecorderResult(case=OcrParityCase.model_validate(cached), cache_hit=True) + return RecorderResult(case=_load_fixture(cached, cache.path_for(cache_key), case_type), cache_hit=True) with _recording_provider(spec) as recorder: - sdk_call(api_base=recorder.url, api_key=spec.api_key, **case_input.as_sdk_kwargs()) + sdk_call(recorder.url, case_input) upstream_response: Final = recorder.take_response() - case: Final = OcrParityCase(litellm_input=case_input, provider_response=upstream_response) - cache.put(cache_key, cast(dict[str, object], case.model_dump(mode="json", exclude_unset=True))) + case: Final = case_type.model_validate({"litellm_input": case_input, "provider_response": upstream_response}) + envelope: Final = FixtureEnvelope( + schema_version=FIXTURE_SCHEMA_VERSION, + recorded_at=datetime.now(timezone.utc), + case=cast(dict[str, object], case.model_dump(mode="json", exclude_unset=True)), + ) + cache.put(cache_key, cast(dict[str, object], envelope.model_dump(mode="json", exclude_unset=True))) return RecorderResult(case=case, cache_hit=False) def record_cases( spec: ProviderSpec, root: Path, - case_inputs: tuple[MistralOcrSdkInput, ...], - sdk_call: Callable[..., object], + case_inputs: tuple[InputT, ...], + sdk_call: Callable[[str, InputT], object], + case_type: type[CaseT], max_concurrency: int, -) -> tuple[RecorderResult, ...]: +) -> tuple[RecorderResult[CaseT], ...]: if max_concurrency < 1: raise ValueError("max_concurrency must be at least 1") unique_inputs: Final = tuple( @@ -184,7 +275,7 @@ def record_cases( ) with ThreadPoolExecutor(max_workers=max_concurrency) as executor: futures: Final = tuple( - executor.submit(record_case, spec, root, case_input, sdk_call) for case_input in unique_inputs + executor.submit(record_case, spec, root, case_input, sdk_call, case_type) for case_input in unique_inputs ) return tuple(future.result() for future in futures) @@ -193,24 +284,12 @@ def fixture_directory(configured: Path | None, env_value: str | None, default: P return (configured or Path(env_value or default)).expanduser() -def recorded_fixtures(directory: Path) -> tuple[OcrParityCase, ...]: +def recorded_fixtures(directory: Path, case_type: type[CaseT]) -> tuple[CaseT, ...]: cache: Final = JsonFileCache(directory) - fixtures: list[OcrParityCase] = [] - for path, raw_fixture in cache.values_with_paths(): - try: - fixtures.append(OcrParityCase.model_validate(raw_fixture)) - except ValidationError as error: - raise ValueError( - f"invalid OCR parity fixture {path}: expected exactly `litellm_input` and `provider_response` " - f"({len(error.errors())} validation errors)" - ) from error - return tuple(fixtures) + return tuple(_load_fixture(raw_fixture, path, case_type) for path, raw_fixture in cache.values_with_paths()) -def fixture_id(fixture: OcrParityCase) -> str: - input_json: Final = canonical_json(fixture.litellm_input.canonical_input()) +def fixture_id(case_input: FixtureInput, prefix: str) -> str: + input_json: Final = canonical_json(case_input.canonical_input()) digest: Final = hashlib.sha256(input_json.encode("utf-8")).hexdigest()[:8] - model_id: Final = fixture.litellm_input.model.replace("/", "-") - provider: Final = fixture.litellm_input.custom_llm_provider - prefix: Final = f"{provider}-{model_id}" if provider and not model_id.startswith(f"{provider}-") else model_id return f"{prefix}-{digest}" diff --git a/tests/test_litellm/_recorded_http.py b/tests/test_litellm/_recorded_http.py new file mode 100644 index 00000000000..940c5211753 --- /dev/null +++ b/tests/test_litellm/_recorded_http.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import base64 +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class _RecordedHttpModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + +class HttpHeader(_RecordedHttpModel): + name: str + value: str + + +class RecordedHttpResponse(_RecordedHttpModel): + kind: Literal["http"] + status_code: int + headers: tuple[HttpHeader, ...] + body_b64: str + + @classmethod + def from_bytes( + cls, + status_code: int, + headers: tuple[HttpHeader, ...], + body: bytes, + ) -> RecordedHttpResponse: + return cls( + kind="http", + status_code=status_code, + headers=headers, + body_b64=base64.b64encode(body).decode("ascii"), + ) + + def body_bytes(self) -> bytes: + return base64.b64decode(self.body_b64, validate=True) + + +class RecordedStreamChunk(_RecordedHttpModel): + data_b64: str + + @classmethod + def from_bytes(cls, data: bytes) -> RecordedStreamChunk: + return cls(data_b64=base64.b64encode(data).decode("ascii")) + + def data_bytes(self) -> bytes: + return base64.b64decode(self.data_b64, validate=True) + + +class RecordedHttpStreamResponse(_RecordedHttpModel): + kind: Literal["http_stream"] + status_code: int + headers: tuple[HttpHeader, ...] + chunks: tuple[RecordedStreamChunk, ...] + + +RecordedResponse = Annotated[ + RecordedHttpResponse | RecordedHttpStreamResponse, + Field(discriminator="kind"), +] diff --git a/tests/test_litellm/ocr/.fixtures/067fe064938269ae58ac14030064a77b92d76edfd00e485fca93a0efbb82bda3.json b/tests/test_litellm/ocr/.fixtures/067fe064938269ae58ac14030064a77b92d76edfd00e485fca93a0efbb82bda3.json new file mode 100644 index 00000000000..7c6168d48a7 --- /dev/null +++ b/tests/test_litellm/ocr/.fixtures/067fe064938269ae58ac14030064a77b92d76edfd00e485fca93a0efbb82bda3.json @@ -0,0 +1,93 @@ +{ + "case": { + "litellm_input": { + "document": { + "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24", + "type": "image_url" + }, + "id": "case-1", + "model": "mistral/mistral-ocr-latest" + }, + "provider_response": { + "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsImJsb2NrcyI6W3sidG9wX2xlZnRfeCI6MzM1LCJ0b3BfbGVmdF95IjoxMzgsImJvdHRvbV9yaWdodF94Ijo0NjQsImJvdHRvbV9yaWdodF95IjoxNjIsImNvbnRlbnQiOiJpbnZvaWNlIDEyMyIsImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJ0eXBlIjoidGV4dCJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOm51bGwsInVzYWdlX2luZm8iOnsicGFnZXNfcHJvY2Vzc2VkIjoxLCJkb2Nfc2l6ZV9ieXRlcyI6NDEyNH19", + "headers": [ + { + "name": "Date", + "value": "Sat, 29 Aug 2026 23:18:02 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "set-cookie", + "value": "__cf_bm=xhZ4oMdiKbThFP71tBxvQtwQRHpnzzS31o6bhmCrX_o-1788045481.8243725-1.0.1.1-ju2kNNBEq.Ff7jRRXRNgRw7jODlxu2bjMlA9ni_JBOIo6Mw.5QKyVVN8wN38J.gLItzZCZAKXFSCWAqVM39oZ9NMBCNTuwXEbf6bM5NIOCn0LgE2DrvxielM2VJkpDan; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Sat, 29 Aug 2026 23:48:02 GMT" + }, + { + "name": "mistral-correlation-id", + "value": "01a04fd0-d7b3-7279-a78d-c48ecdcde419" + }, + { + "name": "x-kong-request-id", + "value": "01a04fd0-d7b3-7279-a78d-c48ecdcde419" + }, + { + "name": "x-ratelimit-limit-ocr-pages-minute", + "value": "60" + }, + { + "name": "x-ratelimit-remaining-ocr-pages-minute", + "value": "59" + }, + { + "name": "x-ratelimit-ocr-pages-query-cost", + "value": "1" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "379" + }, + { + "name": "Server", + "value": "cloudflare" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "x-kong-upstream-latency", + "value": "380" + }, + { + "name": "x-kong-proxy-latency", + "value": "12" + }, + { + "name": "X-Content-Type-Options", + "value": "nosniff" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "name": "Strict-Transport-Security", + "value": "max-age=15552000; includeSubDomains; preload" + }, + { + "name": "CF-RAY", + "value": "a32f45c56be31975-SJC" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "kind": "http", + "status_code": 200 + } + }, + "recorded_at": "2026-08-29T23:18:02.736046Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/.fixtures/099863d538eee61d30ece33c67b6d274496e0065223342a76cc04e5ae4491a4c.json b/tests/test_litellm/ocr/.fixtures/099863d538eee61d30ece33c67b6d274496e0065223342a76cc04e5ae4491a4c.json deleted file mode 100644 index 5892eae7fa3..00000000000 --- a/tests/test_litellm/ocr/.fixtures/099863d538eee61d30ece33c67b6d274496e0065223342a76cc04e5ae4491a4c.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "litellm_input": { - "document": { - "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=N&fontSize=33", - "type": "image_url" - }, - "extract_footer": true, - "extract_header": false, - "id": "-0mq4q9pmig_rn9x2-vn", - "image_limit": 66, - "image_min_size": 1320, - "include_blocks": true, - "include_image_base64": false, - "model": "mistral/mistral-ocr-latest", - "table_format": "html" - }, - "provider_response": { - "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiTiIsImltYWdlcyI6W10sInRhYmxlcyI6W10sImh5cGVybGlua3MiOltdLCJoZWFkZXIiOm51bGwsImZvb3RlciI6bnVsbCwiZGltZW5zaW9ucyI6eyJkcGkiOjIwMCwiaGVpZ2h0IjozMDAsIndpZHRoIjo4MDB9LCJjb25maWRlbmNlX3Njb3JlcyI6bnVsbCwiYmxvY2tzIjpbeyJ0b3BfbGVmdF94IjozODYsInRvcF9sZWZ0X3kiOjEzNSwiYm90dG9tX3JpZ2h0X3giOjQxMiwiYm90dG9tX3JpZ2h0X3kiOjE2MiwiY29udGVudCI6Ik4iLCJjb25maWRlbmNlX3Njb3JlcyI6bnVsbCwidHlwZSI6InRleHQifV19XSwibW9kZWwiOiJtaXN0cmFsLW9jci1sYXRlc3QiLCJkb2N1bWVudF9hbm5vdGF0aW9uIjpudWxsLCJ1c2FnZV9pbmZvIjp7InBhZ2VzX3Byb2Nlc3NlZCI6MSwiZG9jX3NpemVfYnl0ZXMiOjIyOTh9fQ==", - "headers": [ - { - "name": "Date", - "value": "Sat, 29 Aug 2026 21:29:07 GMT" - }, - { - "name": "Content-Type", - "value": "application/json" - }, - { - "name": "set-cookie", - "value": "__cf_bm=vwrWUQPvG4As7fL9Gl1e.lxaZ4I1lG_9uVgrmykYadg-1788038947.3819609-1.0.1.1-4ibtTa5G29xCi1ADh.G_kJ.3l0T2gPlt.1mPNdaJOJNjekO4DR4yYlCCXhucA3Lq45RPrVMxxkPH2YFZ887YrReeGS9x0E05ZO58vqbcJhwfTGLh.3xJSmr.0471Xw4Q; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Sat, 29 Aug 2026 21:59:07 GMT" - }, - { - "name": "mistral-correlation-id", - "value": "01a04f6d-228b-7cdc-9a1a-b99123c027c3" - }, - { - "name": "x-kong-request-id", - "value": "01a04f6d-228b-7cdc-9a1a-b99123c027c3" - }, - { - "name": "x-ratelimit-limit-ocr-pages-minute", - "value": "60" - }, - { - "name": "x-ratelimit-remaining-ocr-pages-minute", - "value": "56" - }, - { - "name": "x-ratelimit-ocr-pages-query-cost", - "value": "1" - }, - { - "name": "x-envoy-upstream-service-time", - "value": "395" - }, - { - "name": "Server", - "value": "cloudflare" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "x-kong-upstream-latency", - "value": "396" - }, - { - "name": "x-kong-proxy-latency", - "value": "17" - }, - { - "name": "X-Content-Type-Options", - "value": "nosniff" - }, - { - "name": "cf-cache-status", - "value": "DYNAMIC" - }, - { - "name": "Strict-Transport-Security", - "value": "max-age=15552000; includeSubDomains; preload" - }, - { - "name": "CF-RAY", - "value": "a32ea63d2ad30065-SJC" - }, - { - "name": "alt-svc", - "value": "h3=\":443\"; ma=86400" - } - ], - "status_code": 200 - } -} diff --git a/tests/test_litellm/ocr/.fixtures/177537cc3eb91a26336033c6016b0f431a36cc237657c498638a6c676bb6a28d.json b/tests/test_litellm/ocr/.fixtures/177537cc3eb91a26336033c6016b0f431a36cc237657c498638a6c676bb6a28d.json deleted file mode 100644 index 4d8267d248d..00000000000 --- a/tests/test_litellm/ocr/.fixtures/177537cc3eb91a26336033c6016b0f431a36cc237657c498638a6c676bb6a28d.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "litellm_input": { - "document": { - "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=ZPay&fontSize=24", - "type": "image_url" - }, - "id": "utdwr1axd37", - "image_min_size": 4099, - "model": "mistral/mistral-ocr-latest", - "table_format": "markdown" - }, - "provider_response": { - "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiWlBheSIsImltYWdlcyI6W10sInRhYmxlcyI6W10sImh5cGVybGlua3MiOltdLCJoZWFkZXIiOm51bGwsImZvb3RlciI6bnVsbCwiZGltZW5zaW9ucyI6eyJkcGkiOjIwMCwiaGVpZ2h0IjozMDAsIndpZHRoIjo4MDB9LCJjb25maWRlbmNlX3Njb3JlcyI6bnVsbCwiYmxvY2tzIjpbeyJ0b3BfbGVmdF94IjozNzAsInRvcF9sZWZ0X3kiOjEzOCwiYm90dG9tX3JpZ2h0X3giOjQzMCwiYm90dG9tX3JpZ2h0X3kiOjE2NSwiY29udGVudCI6IlpQYXkiLCJjb25maWRlbmNlX3Njb3JlcyI6bnVsbCwidHlwZSI6InRleHQifV19XSwibW9kZWwiOiJtaXN0cmFsLW9jci1sYXRlc3QiLCJkb2N1bWVudF9hbm5vdGF0aW9uIjpudWxsLCJ1c2FnZV9pbmZvIjp7InBhZ2VzX3Byb2Nlc3NlZCI6MSwiZG9jX3NpemVfYnl0ZXMiOjMxNDd9fQ==", - "headers": [ - { - "name": "Date", - "value": "Sat, 29 Aug 2026 21:29:05 GMT" - }, - { - "name": "Content-Type", - "value": "application/json" - }, - { - "name": "set-cookie", - "value": "__cf_bm=_.yDONMxI8Wj6bdeKvgLqylB1uRdIzfx64ReOxc5C1o-1788038945.3530984-1.0.1.1-uwAg3dYolu9XCaCCJsie9eFRumBnKC1ZyQDkHA1Otpe_IUrpEsnokSCiyyPun1WOFenihkYZTLs7xMCwSkdbEUs1gN2O2WkWnKL3v_PA36X9McSO1o5iIlm84TAqmGOJ; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Sat, 29 Aug 2026 21:59:05 GMT" - }, - { - "name": "mistral-correlation-id", - "value": "01a04f6d-1a9e-7b54-a1ef-e40d7988227f" - }, - { - "name": "x-kong-request-id", - "value": "01a04f6d-1a9e-7b54-a1ef-e40d7988227f" - }, - { - "name": "x-ratelimit-limit-ocr-pages-minute", - "value": "60" - }, - { - "name": "x-ratelimit-remaining-ocr-pages-minute", - "value": "58" - }, - { - "name": "x-ratelimit-ocr-pages-query-cost", - "value": "1" - }, - { - "name": "x-envoy-upstream-service-time", - "value": "388" - }, - { - "name": "Server", - "value": "cloudflare" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "x-kong-upstream-latency", - "value": "389" - }, - { - "name": "x-kong-proxy-latency", - "value": "13" - }, - { - "name": "X-Content-Type-Options", - "value": "nosniff" - }, - { - "name": "cf-cache-status", - "value": "DYNAMIC" - }, - { - "name": "Strict-Transport-Security", - "value": "max-age=15552000; includeSubDomains; preload" - }, - { - "name": "CF-RAY", - "value": "a32ea6307e391be5-SJC" - }, - { - "name": "alt-svc", - "value": "h3=\":443\"; ma=86400" - } - ], - "status_code": 200 - } -} diff --git a/tests/test_litellm/ocr/.fixtures/6aa18aa0bb41abea634cac9c77fcb73438844c5eb7b4f5eae1a86ecce7b60742.json b/tests/test_litellm/ocr/.fixtures/6aa18aa0bb41abea634cac9c77fcb73438844c5eb7b4f5eae1a86ecce7b60742.json new file mode 100644 index 00000000000..961ac826f68 --- /dev/null +++ b/tests/test_litellm/ocr/.fixtures/6aa18aa0bb41abea634cac9c77fcb73438844c5eb7b4f5eae1a86ecce7b60742.json @@ -0,0 +1,139 @@ +{ + "case": { + "litellm_input": { + "bbox_annotation_format": { + "json_schema": { + "description": "Extract the visible document fields", + "name": "bounding_boxes", + "schema": { + "additionalProperties": false, + "properties": { + "title": { + "type": "string" + } + }, + "required": [ + "title" + ], + "type": "object" + }, + "strict": true + }, + "type": "json_schema" + }, + "confidence_scores_granularity": "page", + "document": { + "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24", + "type": "image_url" + }, + "document_annotation_format": { + "json_schema": { + "description": "Extract the visible document fields", + "name": "document_title", + "schema": { + "additionalProperties": false, + "properties": { + "title": { + "type": "string" + } + }, + "required": [ + "title" + ], + "type": "object" + }, + "strict": true + }, + "type": "json_schema" + }, + "id": "case-1", + "image_min_size": 300, + "include_blocks": false, + "model": "mistral/mistral-ocr-latest", + "pages": [ + 0 + ] + }, + "provider_response": { + "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOnsid29yZF9jb25maWRlbmNlX3Njb3JlcyI6W10sImF2ZXJhZ2VfcGFnZV9jb25maWRlbmNlX3Njb3JlIjowLjkwNjc4MjgyOTY4MDA1MzIsIm1pbmltdW1fcGFnZV9jb25maWRlbmNlX3Njb3JlIjowLjE1NTE4MTc1NjU3MDQ0MDZ9LCJibG9ja3MiOm51bGx9XSwibW9kZWwiOiJtaXN0cmFsLW9jci1sYXRlc3QiLCJkb2N1bWVudF9hbm5vdGF0aW9uIjoie1widGl0bGVcIjogXCJJbnZvaWNlXzEyM1wifSIsInVzYWdlX2luZm8iOnsicGFnZXNfcHJvY2Vzc2VkIjoxLCJkb2Nfc2l6ZV9ieXRlcyI6NDEyNH19", + "headers": [ + { + "name": "Date", + "value": "Sat, 29 Aug 2026 23:18:02 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "mistral-correlation-id", + "value": "01a04fd0-d7c1-7f2e-b3c0-a5901da37f33" + }, + { + "name": "x-kong-request-id", + "value": "01a04fd0-d7c1-7f2e-b3c0-a5901da37f33" + }, + { + "name": "x-ratelimit-limit-ocr-pages-minute", + "value": "60" + }, + { + "name": "x-ratelimit-remaining-ocr-pages-minute", + "value": "59" + }, + { + "name": "x-ratelimit-ocr-pages-query-cost", + "value": "1" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "783" + }, + { + "name": "Server", + "value": "cloudflare" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "x-kong-upstream-latency", + "value": "784" + }, + { + "name": "x-kong-proxy-latency", + "value": "17" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "name": "set-cookie", + "value": "__cf_bm=MplX_p4RI9Z_0LCIvJqPXtjSY6xAps03mWqX2ICuNHM-1788045481.837575-1.0.1.1-8RtNOBgVlARI3DaJl3.wto8CKnDIMFj4rqK5kX7i8rw5N03V7zwUSwaL6q7nus4bqA.3tX_qbM7iqdjvnFGJSRiV4ytibS5ajPPs645Xh_8vVTDzPSgpCzXDFMRnhmnk; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Sat, 29 Aug 2026 23:48:02 GMT" + }, + { + "name": "Strict-Transport-Security", + "value": "max-age=15552000; includeSubDomains; preload" + }, + { + "name": "X-Content-Type-Options", + "value": "nosniff" + }, + { + "name": "CF-RAY", + "value": "a32f45c57d28faec-SJC" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "kind": "http", + "status_code": 200 + } + }, + "recorded_at": "2026-08-29T23:18:03.235228Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/.fixtures/b6cee08db583cae6ed091afd4f81895ccc6bdbcf6f9fff3aab098998c7bc538d.json b/tests/test_litellm/ocr/.fixtures/b6cee08db583cae6ed091afd4f81895ccc6bdbcf6f9fff3aab098998c7bc538d.json deleted file mode 100644 index a2a937befa7..00000000000 --- a/tests/test_litellm/ocr/.fixtures/b6cee08db583cae6ed091afd4f81895ccc6bdbcf6f9fff3aab098998c7bc538d.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "litellm_input": { - "document": { - "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=0&fontSize=12", - "type": "image_url" - }, - "model": "mistral/mistral-ocr-latest" - }, - "provider_response": { - "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiMCIsImltYWdlcyI6W10sInRhYmxlcyI6W10sImh5cGVybGlua3MiOltdLCJoZWFkZXIiOm51bGwsImZvb3RlciI6bnVsbCwiZGltZW5zaW9ucyI6eyJkcGkiOjIwMCwiaGVpZ2h0IjozMDAsIndpZHRoIjo4MDB9LCJjb25maWRlbmNlX3Njb3JlcyI6bnVsbCwiYmxvY2tzIjpbeyJ0b3BfbGVmdF94IjozOTIsInRvcF9sZWZ0X3kiOjE0NCwiYm90dG9tX3JpZ2h0X3giOjQwOCwiYm90dG9tX3JpZ2h0X3kiOjE1NywiY29udGVudCI6IjAiLCJjb25maWRlbmNlX3Njb3JlcyI6bnVsbCwidHlwZSI6ImZvb3RlciJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOm51bGwsInVzYWdlX2luZm8iOnsicGFnZXNfcHJvY2Vzc2VkIjoxLCJkb2Nfc2l6ZV9ieXRlcyI6MTkxMn19", - "headers": [ - { - "name": "Date", - "value": "Sat, 29 Aug 2026 21:29:05 GMT" - }, - { - "name": "Content-Type", - "value": "application/json" - }, - { - "name": "mistral-correlation-id", - "value": "01a04f6d-17f4-7ff9-a890-8e93430da5a7" - }, - { - "name": "x-kong-request-id", - "value": "01a04f6d-17f4-7ff9-a890-8e93430da5a7" - }, - { - "name": "x-ratelimit-limit-ocr-pages-minute", - "value": "60" - }, - { - "name": "x-ratelimit-remaining-ocr-pages-minute", - "value": "59" - }, - { - "name": "x-ratelimit-ocr-pages-query-cost", - "value": "1" - }, - { - "name": "x-envoy-upstream-service-time", - "value": "252" - }, - { - "name": "Server", - "value": "cloudflare" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "x-kong-upstream-latency", - "value": "252" - }, - { - "name": "x-kong-proxy-latency", - "value": "16" - }, - { - "name": "cf-cache-status", - "value": "DYNAMIC" - }, - { - "name": "set-cookie", - "value": "__cf_bm=Bu1ZsZArEglH4RxFKY1uja22ICsIoWxY3EHWLjhl0.U-1788038944.6771533-1.0.1.1-sq3d4oadvs6YUQHDueFHRtdC9k2PXHPeqfHHJ1aD95baBbevXjDnDKJVfn_7Yb.pLQ3kNVEzucNmUGFFbyYk0W7q_rFe54PLou5QgObpMaWYgrectVNvGi0cwNZeqH24; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Sat, 29 Aug 2026 21:59:05 GMT" - }, - { - "name": "Strict-Transport-Security", - "value": "max-age=15552000; includeSubDomains; preload" - }, - { - "name": "X-Content-Type-Options", - "value": "nosniff" - }, - { - "name": "CF-RAY", - "value": "a32ea62c3f91cf22-SJC" - }, - { - "name": "alt-svc", - "value": "h3=\":443\"; ma=86400" - } - ], - "status_code": 200 - } -} diff --git a/tests/test_litellm/ocr/.fixtures/c8702579eea0ba7fb7989604401e0a149493bcf609378295206bf5719a369059.json b/tests/test_litellm/ocr/.fixtures/c8702579eea0ba7fb7989604401e0a149493bcf609378295206bf5719a369059.json deleted file mode 100644 index ed7f9ef38ba..00000000000 --- a/tests/test_litellm/ocr/.fixtures/c8702579eea0ba7fb7989604401e0a149493bcf609378295206bf5719a369059.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "litellm_input": { - "document": { - "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=AuUPy5&fontSize=35", - "type": "image_url" - }, - "include_image_base64": true, - "model": "mistral/mistral-ocr-latest" - }, - "provider_response": { - "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiQXVVUHk1IiwiaW1hZ2VzIjpbXSwidGFibGVzIjpbXSwiaHlwZXJsaW5rcyI6W10sImhlYWRlciI6bnVsbCwiZm9vdGVyIjpudWxsLCJkaW1lbnNpb25zIjp7ImRwaSI6MjAwLCJoZWlnaHQiOjMwMCwid2lkdGgiOjgwMH0sImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJibG9ja3MiOlt7InRvcF9sZWZ0X3giOjMyOSwidG9wX2xlZnRfeSI6MTM0LCJib3R0b21fcmlnaHRfeCI6NDY5LCJib3R0b21fcmlnaHRfeSI6MTcyLCJjb250ZW50IjoiQXVVUHk1IiwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsInR5cGUiOiJ0ZXh0In1dfV0sIm1vZGVsIjoibWlzdHJhbC1vY3ItbGF0ZXN0IiwiZG9jdW1lbnRfYW5ub3RhdGlvbiI6bnVsbCwidXNhZ2VfaW5mbyI6eyJwYWdlc19wcm9jZXNzZWQiOjEsImRvY19zaXplX2J5dGVzIjo0NDQ5fX0=", - "headers": [ - { - "name": "Date", - "value": "Sat, 29 Aug 2026 21:29:06 GMT" - }, - { - "name": "Content-Type", - "value": "application/json" - }, - { - "name": "mistral-correlation-id", - "value": "01a04f6d-1e94-7169-acf2-331366511208" - }, - { - "name": "x-kong-request-id", - "value": "01a04f6d-1e94-7169-acf2-331366511208" - }, - { - "name": "x-ratelimit-limit-ocr-pages-minute", - "value": "60" - }, - { - "name": "x-ratelimit-remaining-ocr-pages-minute", - "value": "57" - }, - { - "name": "x-ratelimit-ocr-pages-query-cost", - "value": "1" - }, - { - "name": "x-envoy-upstream-service-time", - "value": "299" - }, - { - "name": "Server", - "value": "cloudflare" - }, - { - "name": "access-control-allow-origin", - "value": "*" - }, - { - "name": "x-kong-upstream-latency", - "value": "301" - }, - { - "name": "x-kong-proxy-latency", - "value": "13" - }, - { - "name": "cf-cache-status", - "value": "DYNAMIC" - }, - { - "name": "set-cookie", - "value": "__cf_bm=L.iPQSMbMkUfZ6bZqRUnIvFouaL6uaka9sb1RADqE7I-1788038946.371394-1.0.1.1-eP7TRle_nNnnCddZ21LX.mzRWslGV0sv5wZONgMz.g2Es_qbFOGUuPv6K1C0p86Pz6xULpoaVjmqxUOkza6Ug_53VGsNkwHRvyxXuKXvMSB62zZPyrO0PcwTqRZKYFLC; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Sat, 29 Aug 2026 21:59:06 GMT" - }, - { - "name": "Strict-Transport-Security", - "value": "max-age=15552000; includeSubDomains; preload" - }, - { - "name": "X-Content-Type-Options", - "value": "nosniff" - }, - { - "name": "CF-RAY", - "value": "a32ea636dd4eebe2-SJC" - }, - { - "name": "alt-svc", - "value": "h3=\":443\"; ma=86400" - } - ], - "status_code": 200 - } -} diff --git a/tests/test_litellm/ocr/.fixtures/d7a77e2b36580d60ba22ef0a9cd910adf782ab9194e309d3c62fe0f2fadc326c.json b/tests/test_litellm/ocr/.fixtures/d7a77e2b36580d60ba22ef0a9cd910adf782ab9194e309d3c62fe0f2fadc326c.json new file mode 100644 index 00000000000..4abf13f66ce --- /dev/null +++ b/tests/test_litellm/ocr/.fixtures/d7a77e2b36580d60ba22ef0a9cd910adf782ab9194e309d3c62fe0f2fadc326c.json @@ -0,0 +1,92 @@ +{ + "case": { + "litellm_input": { + "document": { + "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24", + "type": "image_url" + }, + "model": "mistral/mistral-ocr-latest" + }, + "provider_response": { + "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsImJsb2NrcyI6W3sidG9wX2xlZnRfeCI6MzM1LCJ0b3BfbGVmdF95IjoxMzgsImJvdHRvbV9yaWdodF94Ijo0NjQsImJvdHRvbV9yaWdodF95IjoxNjIsImNvbnRlbnQiOiJpbnZvaWNlIDEyMyIsImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJ0eXBlIjoidGV4dCJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOm51bGwsInVzYWdlX2luZm8iOnsicGFnZXNfcHJvY2Vzc2VkIjoxLCJkb2Nfc2l6ZV9ieXRlcyI6NDEyNH19", + "headers": [ + { + "name": "Date", + "value": "Sat, 29 Aug 2026 23:18:02 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "set-cookie", + "value": "__cf_bm=4Y4BNG5seFc4759BsUBjy4Z2odUy3Rgf0SOYkyESdPA-1788045481.8373353-1.0.1.1-eirQC2neaOT8r6_9qxcWeUjOMH7L1NmsXj2YyMbcVmTO62TWrDCjf7VqxKigiJ1lViQQkJQwVpMsSHTfUOaziX4zDTloRi4b.oaA37lqE.gBbH9FylA_nMTl2mYg65X9; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Sat, 29 Aug 2026 23:48:02 GMT" + }, + { + "name": "mistral-correlation-id", + "value": "01a04fd0-d7b8-799c-9ac2-48ab880605d6" + }, + { + "name": "x-kong-request-id", + "value": "01a04fd0-d7b8-799c-9ac2-48ab880605d6" + }, + { + "name": "x-ratelimit-limit-ocr-pages-minute", + "value": "60" + }, + { + "name": "x-ratelimit-remaining-ocr-pages-minute", + "value": "59" + }, + { + "name": "x-ratelimit-ocr-pages-query-cost", + "value": "1" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "355" + }, + { + "name": "Server", + "value": "cloudflare" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "x-kong-upstream-latency", + "value": "356" + }, + { + "name": "x-kong-proxy-latency", + "value": "17" + }, + { + "name": "X-Content-Type-Options", + "value": "nosniff" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "name": "Strict-Transport-Security", + "value": "max-age=15552000; includeSubDomains; preload" + }, + { + "name": "CF-RAY", + "value": "a32f45c57f8e7834-SJC" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "kind": "http", + "status_code": 200 + } + }, + "recorded_at": "2026-08-29T23:18:02.735198Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/.fixtures/ecfb251cbe6cb13e52dd11a4706f788cf6f2b3d5f4fdae5d7b9b620c398f47dc.json b/tests/test_litellm/ocr/.fixtures/ecfb251cbe6cb13e52dd11a4706f788cf6f2b3d5f4fdae5d7b9b620c398f47dc.json new file mode 100644 index 00000000000..57f1e223133 --- /dev/null +++ b/tests/test_litellm/ocr/.fixtures/ecfb251cbe6cb13e52dd11a4706f788cf6f2b3d5f4fdae5d7b9b620c398f47dc.json @@ -0,0 +1,117 @@ +{ + "case": { + "litellm_input": { + "document": { + "image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24", + "type": "image_url" + }, + "document_annotation_format": { + "json_schema": { + "description": "Extract the visible document fields", + "name": "prompted_document_title", + "schema": { + "additionalProperties": false, + "properties": { + "title": { + "type": "string" + } + }, + "required": [ + "title" + ], + "type": "object" + }, + "strict": true + }, + "type": "json_schema" + }, + "document_annotation_prompt": "Extract the visible title", + "include_image_base64": true, + "model": "mistral/mistral-ocr-latest", + "pages": [ + 0 + ] + }, + "provider_response": { + "body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsImJsb2NrcyI6W3sidG9wX2xlZnRfeCI6MzM1LCJ0b3BfbGVmdF95IjoxMzgsImJvdHRvbV9yaWdodF94Ijo0NjQsImJvdHRvbV9yaWdodF95IjoxNjIsImNvbnRlbnQiOiJpbnZvaWNlIDEyMyIsImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJ0eXBlIjoidGV4dCJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOiJ7XCJ0aXRsZVwiOiBcImludm9pY2UgMTIzXCJ9IiwidXNhZ2VfaW5mbyI6eyJwYWdlc19wcm9jZXNzZWQiOjEsImRvY19zaXplX2J5dGVzIjo0MTI0fX0=", + "headers": [ + { + "name": "Date", + "value": "Sat, 29 Aug 2026 23:18:02 GMT" + }, + { + "name": "Content-Type", + "value": "application/json" + }, + { + "name": "set-cookie", + "value": "__cf_bm=vLaAHFXIItKaEkQQMMAH_E.rAMYc7fuSd6Evsu2SbhY-1788045481.8225453-1.0.1.1-L.S7ARgAGlXrNvVQ7ETwoVMFhVRT25B9gxsNxfaHcYVZ9Tt_36StIe5sxCWGZzWeMfhVJhVBsxNEADhO._6d_iPdnanYVrN0CA0EmP2pt8mDq0j2sEb_bdco7eTaJinM; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Sat, 29 Aug 2026 23:48:02 GMT" + }, + { + "name": "mistral-correlation-id", + "value": "01a04fd0-d7b6-7819-aa2d-0d1906d915e2" + }, + { + "name": "x-kong-request-id", + "value": "01a04fd0-d7b6-7819-aa2d-0d1906d915e2" + }, + { + "name": "x-ratelimit-limit-ocr-pages-minute", + "value": "60" + }, + { + "name": "x-ratelimit-remaining-ocr-pages-minute", + "value": "59" + }, + { + "name": "x-ratelimit-ocr-pages-query-cost", + "value": "1" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "620" + }, + { + "name": "Server", + "value": "cloudflare" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "x-kong-upstream-latency", + "value": "621" + }, + { + "name": "x-kong-proxy-latency", + "value": "12" + }, + { + "name": "X-Content-Type-Options", + "value": "nosniff" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "name": "Strict-Transport-Security", + "value": "max-age=15552000; includeSubDomains; preload" + }, + { + "name": "CF-RAY", + "value": "a32f45c56cd35d19-SJC" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "kind": "http", + "status_code": 200 + } + }, + "recorded_at": "2026-08-29T23:18:02.736175Z", + "schema_version": 1 +} diff --git a/tests/test_litellm/ocr/conftest.py b/tests/test_litellm/ocr/conftest.py index 6a3947e2fdb..ba2bd84a9d8 100644 --- a/tests/test_litellm/ocr/conftest.py +++ b/tests/test_litellm/ocr/conftest.py @@ -8,6 +8,7 @@ import pytest from pydantic import ValidationError from tests.test_litellm._fixture_recorder import fixture_id, recorded_fixtures +from tests.test_litellm.ocr.fixture_models import OcrParityCase FIXTURE_DIR_ENV: Final = "LITELLM_OCR_FIXTURE_DIR" @@ -21,16 +22,24 @@ def _fixture_directory() -> Path: return Path(configured).expanduser() +def _fixture_id(fixture: OcrParityCase) -> str: + case_input: Final = fixture.litellm_input + model_id: Final = case_input.model.replace("/", "-") + provider: Final = case_input.custom_llm_provider + prefix: Final = f"{provider}-{model_id}" if provider and not model_id.startswith(f"{provider}-") else model_id + return fixture_id(case_input, prefix) + + def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: if "ocr_fixture" not in metafunc.fixturenames: return directory: Final = _fixture_directory() try: - fixtures: Final = recorded_fixtures(directory) + fixtures: Final = recorded_fixtures(directory, OcrParityCase) except (ValidationError, ValueError) as error: raise pytest.UsageError( f"Invalid OCR parity fixture bundle at {directory}. " - "Each fixture must contain exactly `litellm_input` and `provider_response`. " + "Each fixture must use the current versioned envelope. " "Record fresh fixtures in an empty directory with: " f"`uv run python tests/test_litellm/ocr/generate_fixtures.py --fixture-dir {directory}`. " f"Validation details: {error}" @@ -49,4 +58,4 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: ), ) return - metafunc.parametrize("ocr_fixture", fixtures, ids=tuple(fixture_id(fixture) for fixture in fixtures)) + metafunc.parametrize("ocr_fixture", fixtures, ids=tuple(_fixture_id(fixture) for fixture in fixtures)) diff --git a/tests/test_litellm/ocr/fixture_models.py b/tests/test_litellm/ocr/fixture_models.py index dc247e6a145..67070c33fde 100644 --- a/tests/test_litellm/ocr/fixture_models.py +++ b/tests/test_litellm/ocr/fixture_models.py @@ -7,6 +7,8 @@ from typing import Annotated, Literal, cast from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator from typing_extensions import Self +from tests.test_litellm._recorded_http import RecordedResponse + JsonObject = dict[str, JsonValue] @@ -283,34 +285,6 @@ class ReductoParseLegacySdkInput(OcrSdkInputBase): return self -class HttpHeader(_FixtureModel): - name: str - value: str - - -class RecordedHttpResponse(_FixtureModel): - kind: Literal["http"] = "http" - status_code: int - headers: tuple[HttpHeader, ...] - body_b64: str - - @classmethod - def from_bytes( - cls, - status_code: int, - headers: tuple[HttpHeader, ...], - body: bytes, - ) -> RecordedHttpResponse: - return cls( - status_code=status_code, - headers=headers, - body_b64=base64.b64encode(body).decode("ascii"), - ) - - def body_bytes(self) -> bytes: - return base64.b64decode(self.body_b64, validate=True) - - class OcrParityCase(_FixtureModel): litellm_input: MistralOcrSdkInput - provider_response: RecordedHttpResponse + provider_response: RecordedResponse diff --git a/tests/test_litellm/ocr/generate_fixtures.py b/tests/test_litellm/ocr/generate_fixtures.py index 173f4ef3966..56490f78f26 100644 --- a/tests/test_litellm/ocr/generate_fixtures.py +++ b/tests/test_litellm/ocr/generate_fixtures.py @@ -3,7 +3,6 @@ from __future__ import annotations import argparse import logging import os -import queue from collections.abc import Callable from dataclasses import dataclass from pathlib import Path @@ -11,7 +10,6 @@ from typing import Final, cast from urllib.parse import quote from dotenv import load_dotenv -from hypothesis import given, settings from hypothesis import strategies as st from hypothesis.strategies import DrawFn, SearchStrategy @@ -20,14 +18,15 @@ from litellm.rust_bridge.ocr import use_litellm_rust from tests.test_litellm._fixture_recorder import ( ProviderSpec, fixture_directory, + generate_case_inputs, record_cases, ) from tests.test_litellm.ocr.fixture_models import ( JsonSchemaDefinition, JsonSchemaResponseFormat, MistralImageUrlDocument, - MistralModel, MistralOcrSdkInput, + OcrParityCase, ReductoChunking, ReductoDocumentUrlDocument, ReductoFormatting, @@ -42,13 +41,6 @@ LOGGER: Final = logging.getLogger(__name__) _TEXT: Final = st.just("invoice 123") _VALUE_TEXT: Final = st.just("case-1") _FONT_SIZE: Final = st.just(24) -_MISTRAL_MODELS: Final = ( - "mistral/mistral-ocr-2512", - "mistral/mistral-ocr-4-0", - "mistral/mistral-ocr-4-1", - "mistral/mistral-ocr-4", - "mistral/mistral-ocr-latest", -) @dataclass(frozen=True, slots=True) @@ -245,21 +237,18 @@ def reducto_legacy_input_strategy() -> SearchStrategy[ReductoParseLegacySdkInput def _generate_examples( spec: ProviderSpec, root: Path, + model: str, + api_key: str, examples: int, concurrency: int, sdk_call: Callable[..., object], ) -> None: - generated: Final[queue.SimpleQueue[MistralOcrSdkInput | None]] = queue.SimpleQueue() + case_inputs: Final = generate_case_inputs(mistral_input_strategy(model), examples) - @settings(max_examples=examples, deadline=None, derandomize=True) - @given(case_input=mistral_input_strategy(spec.model)) - def generate_case(case_input: MistralOcrSdkInput) -> None: - generated.put(case_input) + def invoke(api_base: str, case_input: MistralOcrSdkInput) -> object: + return sdk_call(api_base=api_base, api_key=api_key, **case_input.as_sdk_kwargs()) - generate_case() - generated.put(None) - case_inputs: Final = tuple(iter(generated.get, None)) - results: Final = record_cases(spec, root, case_inputs, sdk_call, concurrency) + results: Final = record_cases(spec, root, case_inputs, invoke, OcrParityCase, concurrency) for result in results: LOGGER.info("%s %s", "cached" if result.cache_hit else "recorded", result.case.litellm_input.model) @@ -274,13 +263,13 @@ def _parse_args() -> GeneratorArgs: parser.add_argument("--concurrency", type=int, default=4) parser.add_argument("--examples", type=int, default=4) parser.add_argument("--fixture-dir", type=Path) - parser.add_argument("--model", choices=_MISTRAL_MODELS, default="mistral/mistral-ocr-latest") + parser.add_argument("--model", required=True) namespace: Final = parser.parse_args() return GeneratorArgs( concurrency=cast(int, namespace.concurrency), examples=cast(int, namespace.examples), fixture_dir=cast(Path | None, namespace.fixture_dir), - model=cast(MistralModel, namespace.model), + model=cast(str, namespace.model), ) @@ -296,9 +285,17 @@ def main() -> None: os.environ.get(FIXTURE_DIR_ENV), Path(__file__).with_name(".fixtures"), ) - spec: Final = ProviderSpec(model=args.model, upstream_base=_mistral_upstream_base(), api_key=api_key) + spec: Final = ProviderSpec(upstream_base=_mistral_upstream_base()) use_litellm_rust(False, ocr=None, aocr=None) - _generate_examples(spec, root, args.examples, args.concurrency, cast(Callable[..., object], litellm.ocr)) + _generate_examples( + spec, + root, + args.model, + api_key, + args.examples, + args.concurrency, + cast(Callable[..., object], litellm.ocr), + ) if __name__ == "__main__": diff --git a/tests/test_litellm/ocr/test_sdk_parity.py b/tests/test_litellm/ocr/test_sdk_parity.py index 324fc90db9d..01ce7af10b1 100644 --- a/tests/test_litellm/ocr/test_sdk_parity.py +++ b/tests/test_litellm/ocr/test_sdk_parity.py @@ -80,24 +80,18 @@ def test_recorded_ocr_sdk_parity( case_file: Final = tmp_path / f"{route.value}-ocr-parity-case.json" case_file.write_text(ocr_fixture.model_dump_json(indent=2, exclude_unset=True), encoding="utf-8") response: Final = ocr_fixture.provider_response - response_body: Final = response.body_bytes() - response_headers: Final = tuple((header.name, header.value) for header in response.headers) python_worker, rust_worker = sdk_workers python: Final = run_execution( python_worker, case_file, route.value, - response.status_code, - response_headers, - response_body, + response, ) rust: Final = run_execution( rust_worker, case_file, route.value, - response.status_code, - response_headers, - response_body, + response, ) assert_parity(python, rust, PYTHON_HTTP_SENTINEL) diff --git a/tests/test_litellm/parity/replay.py b/tests/test_litellm/parity/replay.py index 017f9002303..c2228c70be6 100644 --- a/tests/test_litellm/parity/replay.py +++ b/tests/test_litellm/parity/replay.py @@ -4,12 +4,12 @@ import queue import threading from collections.abc import Generator from contextlib import contextmanager -from dataclasses import dataclass from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Final from pydantic import JsonValue, TypeAdapter +from tests.test_litellm._recorded_http import RecordedHttpResponse, RecordedHttpStreamResponse, RecordedResponse from tests.test_litellm.parity.models import CapturedRequest JSON_VALUE: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) @@ -31,15 +31,15 @@ class ReplayServer(ThreadingHTTPServer): def __init__(self) -> None: super().__init__(("127.0.0.1", 0), _ReplayHandler) - self.responses: queue.Queue[ReplayResponse] = queue.Queue() + self.responses: queue.Queue[RecordedResponse] = queue.Queue() self.requests: queue.Queue[CapturedRequest] = queue.Queue() @property def url(self) -> str: return f"http://127.0.0.1:{self.server_address[1]}" - def enqueue_response(self, status_code: int, headers: tuple[tuple[str, str], ...], body: bytes) -> None: - self.responses.put(ReplayResponse(status_code=status_code, headers=headers, body=body)) + def enqueue_response(self, response: RecordedResponse) -> None: + self.responses.put(response) def take_request(self) -> CapturedRequest: request_count: Final = self.requests.qsize() @@ -54,13 +54,6 @@ class ReplayServer(ThreadingHTTPServer): self.requests.get_nowait() -@dataclass(frozen=True, slots=True) -class ReplayResponse: - status_code: int - headers: tuple[tuple[str, str], ...] - body: bytes - - class _ReplayHandler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" @@ -91,12 +84,26 @@ class _ReplayHandler(BaseHTTPRequestHandler): self.send_error(500, "no replay response queued") return self.send_response_only(response.status_code) - for name, value in response.headers: - if name.lower() not in EXCLUDED_RESPONSE_HEADERS: - self.send_header(name, value) - self.send_header("content-length", str(len(response.body))) + for header in response.headers: + if header.name.lower() not in EXCLUDED_RESPONSE_HEADERS: + self.send_header(header.name, header.value) + if isinstance(response, RecordedHttpResponse): + response_body: Final = response.body_bytes() + self.send_header("content-length", str(len(response_body))) + self.end_headers() + self.wfile.write(response_body) + return + assert isinstance(response, RecordedHttpStreamResponse) + self.send_header("transfer-encoding", "chunked") self.end_headers() - self.wfile.write(response.body) + for chunk in response.chunks: + data = chunk.data_bytes() + self.wfile.write(f"{len(data):X}\r\n".encode("ascii")) + self.wfile.write(data) + self.wfile.write(b"\r\n") + self.wfile.flush() + self.wfile.write(b"0\r\n\r\n") + self.wfile.flush() def log_message(self, format: str, *args: object) -> None: return diff --git a/tests/test_litellm/parity/runner.py b/tests/test_litellm/parity/runner.py index 83f652a582c..eaec9bcb05d 100644 --- a/tests/test_litellm/parity/runner.py +++ b/tests/test_litellm/parity/runner.py @@ -13,6 +13,7 @@ from typing import Final, TextIO, cast from pydantic import TypeAdapter, ValidationError +from tests.test_litellm._recorded_http import RecordedResponse from tests.test_litellm.parity.models import ( Execution, SDKCommand, @@ -69,14 +70,12 @@ class PythonScriptWorker: self, case_file: Path, route: str, - status_code: int, - headers: tuple[tuple[str, str], ...], - body: bytes, + response: RecordedResponse, ) -> Execution: stdin: Final = self.process.stdin if stdin is None or self.process.poll() is not None: raise AssertionError(f"{self.mode} OCR worker exited before processing {case_file}") - self.provider.enqueue_response(status_code, headers, body) + self.provider.enqueue_response(response) command: Final = SDKCommand(case_file=str(case_file), route=route) try: stdin.write(f"{command.model_dump_json()}\n") @@ -153,8 +152,6 @@ def run_execution( worker: PythonScriptWorker, case_file: Path, route: str, - status_code: int, - headers: tuple[tuple[str, str], ...], - body: bytes, + response: RecordedResponse, ) -> Execution: - return worker.execute(case_file, route, status_code, headers, body) + return worker.execute(case_file, route, response) diff --git a/tests/test_litellm/test__fixture_recorder.py b/tests/test_litellm/test__fixture_recorder.py index 44a0cf88cf9..38f86057abc 100644 --- a/tests/test_litellm/test__fixture_recorder.py +++ b/tests/test_litellm/test__fixture_recorder.py @@ -5,12 +5,52 @@ from collections.abc import Generator from contextlib import contextmanager from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path -from typing import Final, cast +from typing import Final import httpx +import pytest +from hypothesis import strategies as st +from pydantic import BaseModel, ConfigDict -from tests.test_litellm._fixture_recorder import ProviderSpec, record_cases -from tests.test_litellm.ocr.fixture_models import MistralImageUrlDocument, MistralOcrSdkInput +from tests.test_litellm._fixture_recorder import ( + FIXTURE_SCHEMA_VERSION, + ProviderSpec, + fixture_cache_key, + generate_case_inputs, + record_case, + record_cases, + recorded_fixtures, +) +from tests.test_litellm._json_fs_cache import JsonFileCache +from tests.test_litellm._recorded_http import ( + HttpHeader, + RecordedHttpStreamResponse, + RecordedResponse, + RecordedStreamChunk, +) +from tests.test_litellm.parity.replay import replay_server + +_SSE_CHUNKS: Final = ( + b'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n', + b'data: {"choices":[{"delta":{"content":" world"}}]}\n\n', + b"data: [DONE]\n\n", +) + + +class _FixtureInput(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + identifier: str + + def canonical_input(self) -> dict[str, object]: + return {"identifier": self.identifier} + + +class _ParityCase(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + litellm_input: _FixtureInput + provider_response: RecordedResponse class _ControlledUpstream(ThreadingHTTPServer): @@ -50,6 +90,21 @@ class _ControlledUpstreamHandler(BaseHTTPRequestHandler): assert isinstance(upstream, _ControlledUpstream) length: Final = int(self.headers.get("content-length") or "0") self.rfile.read(length) + if self.path == "/v1/chat/completions": + with upstream.lock: + upstream.request_count += 1 + self.send_response(200) + self.send_header("content-type", "text/event-stream") + self.send_header("transfer-encoding", "chunked") + self.end_headers() + for chunk in _SSE_CHUNKS: + self.wfile.write(f"{len(chunk):X}\r\n".encode("ascii")) + self.wfile.write(chunk) + self.wfile.write(b"\r\n") + self.wfile.flush() + self.wfile.write(b"0\r\n\r\n") + self.wfile.flush() + return upstream.start_request() try: body: Final = b"{}" @@ -78,27 +133,86 @@ def _controlled_upstream() -> Generator[_ControlledUpstream]: thread.join(timeout=5) -def _case(identifier: str) -> MistralOcrSdkInput: - return MistralOcrSdkInput.model_validate( - { - "model": "mistral/mistral-ocr-latest", - "document": MistralImageUrlDocument(type="image_url", image_url="https://example.com/image.png"), - "id": identifier, - } - ) +def _case(identifier: str) -> _FixtureInput: + return _FixtureInput(identifier=identifier) -def _sdk_call(**kwargs: object) -> object: - api_base: Final = cast(str, kwargs["api_base"]) +def _sdk_call(api_base: str, case_input: _FixtureInput) -> object: return httpx.post(f"{api_base}/v1/ocr", content=b"{}", timeout=5) +def _stream_sdk_call(api_base: str, case_input: _FixtureInput) -> object: + return httpx.post(f"{api_base}/v1/chat/completions", content=b"{}", timeout=5) + + +def test_generate_case_inputs_is_deterministic() -> None: + strategy: Final = st.builds(_FixtureInput, identifier=st.integers().map(str)) + + assert generate_case_inputs(strategy, examples=4) == generate_case_inputs(strategy, examples=4) + + def test_record_cases_deduplicates_and_limits_concurrency(tmp_path: Path) -> None: case_inputs: Final = (_case("one"), _case("two"), _case("one"), _case("three")) with _controlled_upstream() as upstream: - spec: Final = ProviderSpec(model="mistral/mistral-ocr-latest", upstream_base=upstream.url, api_key="test-key") - results: Final = record_cases(spec, tmp_path, case_inputs, _sdk_call, max_concurrency=2) + spec: Final = ProviderSpec(upstream_base=upstream.url) + results: Final = record_cases(spec, tmp_path, case_inputs, _sdk_call, _ParityCase, max_concurrency=2) assert len(results) == 3 assert upstream.request_count == 3 assert upstream.max_active_requests == 2 + assert len(recorded_fixtures(tmp_path, _ParityCase)) == 3 + for fixture_path in tmp_path.glob("*.json"): + contents = fixture_path.read_text(encoding="utf-8") + assert f'"schema_version": {FIXTURE_SCHEMA_VERSION}' in contents + assert '"recorded_at":' in contents + + +def test_record_case_rejects_stale_fixture_before_provider_call(tmp_path: Path) -> None: + case_input: Final = _case("stale") + cache: Final = JsonFileCache(tmp_path) + fixture_path: Final = cache.put(fixture_cache_key(case_input), {"schema_version": 0}) + + with pytest.raises(ValueError, match=f"{fixture_path} has schema_version 0, expected {FIXTURE_SCHEMA_VERSION}"): + record_case( + ProviderSpec(upstream_base="http://127.0.0.1:1"), + tmp_path, + case_input, + _sdk_call, + _ParityCase, + ) + + +def test_streaming_response_records_and_replays_chunks(tmp_path: Path) -> None: + with _controlled_upstream() as upstream: + result: Final = record_case( + ProviderSpec(upstream_base=upstream.url), + tmp_path, + _case("stream"), + _stream_sdk_call, + _ParityCase, + ) + + response: Final = result.case.provider_response + assert isinstance(response, RecordedHttpStreamResponse) + assert tuple(chunk.data_bytes() for chunk in response.chunks) == _SSE_CHUNKS + + with replay_server() as provider: + provider.enqueue_response(response) + with httpx.stream("POST", f"{provider.url}/v1/chat/completions", json={}) as replayed: + replayed_chunks: Final = tuple(replayed.iter_raw()) + provider.take_request() + + assert replayed_chunks == _SSE_CHUNKS + + +def test_stream_response_model_rejects_buffered_body() -> None: + with pytest.raises(ValueError): + RecordedHttpStreamResponse.model_validate( + { + "kind": "http_stream", + "status_code": 200, + "headers": [HttpHeader(name="content-type", value="text/event-stream")], + "chunks": [RecordedStreamChunk.from_bytes(b"data: [DONE]\n\n")], + "body_b64": "", + } + )