mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-19 00:01:29 +00:00
refactor(tests): generalize parity fixture recorder
This commit is contained in:
Yujong Lee
GitHub
parent
50ed0c9ca7
commit
64be0c18b9
17 changed files with 826 additions and 511 deletions
|
|
@ -3,25 +3,31 @@ from __future__ import annotations
|
|||
import hashlib
|
||||
import queue
|
||||
import threading
|
||||
from collections.abc import Callable, Generator
|
||||
from collections.abc import Callable, Generator, Iterable
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Final, cast
|
||||
from typing import Final, Generic, Protocol, TypeVar, cast
|
||||
|
||||
import httpx
|
||||
from pydantic import ValidationError
|
||||
from hypothesis import given, settings
|
||||
from hypothesis.strategies import SearchStrategy
|
||||
from pydantic import AwareDatetime, BaseModel, ConfigDict, ValidationError
|
||||
|
||||
from tests.test_litellm._json_fs_cache import JsonFileCache, canonical_json
|
||||
from tests.test_litellm.ocr.fixture_models import (
|
||||
from tests.test_litellm._recorded_http import (
|
||||
HttpHeader,
|
||||
MistralOcrSdkInput,
|
||||
OcrParityCase,
|
||||
RecordedHttpResponse,
|
||||
RecordedHttpStreamResponse,
|
||||
RecordedResponse,
|
||||
RecordedStreamChunk,
|
||||
)
|
||||
|
||||
FIXTURE_SCHEMA_VERSION: Final = 1
|
||||
|
||||
_HOP_BY_HOP_HEADERS: Final = frozenset(
|
||||
{
|
||||
"connection",
|
||||
|
|
@ -36,19 +42,33 @@ _HOP_BY_HOP_HEADERS: Final = frozenset(
|
|||
)
|
||||
|
||||
|
||||
class FixtureInput(Protocol):
|
||||
def canonical_input(self) -> dict[str, object]: ...
|
||||
|
||||
|
||||
InputT = TypeVar("InputT", bound=FixtureInput)
|
||||
CaseT = TypeVar("CaseT", bound=BaseModel)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProviderSpec:
|
||||
model: str
|
||||
upstream_base: str
|
||||
api_key: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RecorderResult:
|
||||
case: OcrParityCase
|
||||
class RecorderResult(Generic[CaseT]):
|
||||
case: CaseT
|
||||
cache_hit: bool
|
||||
|
||||
|
||||
class FixtureEnvelope(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
schema_version: int
|
||||
recorded_at: AwareDatetime
|
||||
case: dict[str, object]
|
||||
|
||||
|
||||
def _excluded_headers(headers: tuple[tuple[str, str], ...]) -> frozenset[str]:
|
||||
connection_values: Final = tuple(value for name, value in headers if name.lower() == "connection")
|
||||
connection_headers: Final = frozenset(
|
||||
|
|
@ -69,13 +89,13 @@ class _RecordingProvider(ThreadingHTTPServer):
|
|||
def __init__(self, spec: ProviderSpec) -> None:
|
||||
super().__init__(("127.0.0.1", 0), _RecordingHandler)
|
||||
self.spec: Final = spec
|
||||
self.responses: queue.Queue[RecordedHttpResponse] = queue.Queue()
|
||||
self.responses: queue.Queue[RecordedResponse] = queue.Queue()
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
return f"http://127.0.0.1:{self.server_address[1]}"
|
||||
|
||||
def take_response(self) -> RecordedHttpResponse:
|
||||
def take_response(self) -> RecordedResponse:
|
||||
try:
|
||||
return self.responses.get(timeout=5)
|
||||
except queue.Empty as error:
|
||||
|
|
@ -103,22 +123,59 @@ class _RecordingHandler(BaseHTTPRequestHandler):
|
|||
content=request_body,
|
||||
timeout=120,
|
||||
) as upstream:
|
||||
response_body: Final = b"".join(upstream.iter_bytes())
|
||||
recorded_response: Final = RecordedHttpResponse.from_bytes(
|
||||
status_code=upstream.status_code,
|
||||
headers=_end_to_end_headers(upstream.headers),
|
||||
body=response_body,
|
||||
)
|
||||
headers: Final = _end_to_end_headers(upstream.headers)
|
||||
recorded_response: Final = self._record_upstream_response(upstream, headers)
|
||||
except httpx.HTTPError as error:
|
||||
self._send_response(502, (), str(error).encode("utf-8"))
|
||||
return
|
||||
|
||||
if 200 <= recorded_response.status_code < 300:
|
||||
provider.responses.put(recorded_response)
|
||||
self._send_recorded_response(recorded_response)
|
||||
if isinstance(recorded_response, RecordedHttpResponse):
|
||||
self._send_response(recorded_response.status_code, recorded_response.headers, recorded_response.body_bytes())
|
||||
|
||||
def _send_recorded_response(self, response: RecordedHttpResponse) -> None:
|
||||
self._send_response(response.status_code, response.headers, response.body_bytes())
|
||||
def _record_upstream_response(
|
||||
self,
|
||||
upstream: httpx.Response,
|
||||
headers: tuple[HttpHeader, ...],
|
||||
) -> RecordedResponse:
|
||||
content_type: Final = cast(str, upstream.headers.get("content-type", ""))
|
||||
if content_type.lower().startswith("text/event-stream"):
|
||||
return self._record_stream(upstream, headers)
|
||||
response_body: Final = b"".join(upstream.iter_bytes())
|
||||
return RecordedHttpResponse.from_bytes(
|
||||
status_code=upstream.status_code,
|
||||
headers=headers,
|
||||
body=response_body,
|
||||
)
|
||||
|
||||
def _record_stream(
|
||||
self,
|
||||
upstream: httpx.Response,
|
||||
headers: tuple[HttpHeader, ...],
|
||||
) -> RecordedHttpStreamResponse:
|
||||
self.send_response_only(upstream.status_code)
|
||||
for header in headers:
|
||||
self.send_header(header.name, header.value)
|
||||
self.send_header("transfer-encoding", "chunked")
|
||||
self.end_headers()
|
||||
chunks: Final = tuple(self._relay_chunks(upstream.iter_bytes()))
|
||||
self.wfile.write(b"0\r\n\r\n")
|
||||
self.wfile.flush()
|
||||
return RecordedHttpStreamResponse(
|
||||
kind="http_stream",
|
||||
status_code=upstream.status_code,
|
||||
headers=headers,
|
||||
chunks=chunks,
|
||||
)
|
||||
|
||||
def _relay_chunks(self, chunks: Iterable[bytes]) -> Generator[RecordedStreamChunk, None, None]:
|
||||
for chunk in chunks:
|
||||
self.wfile.write(f"{len(chunk):X}\r\n".encode("ascii"))
|
||||
self.wfile.write(chunk)
|
||||
self.wfile.write(b"\r\n")
|
||||
self.wfile.flush()
|
||||
yield RecordedStreamChunk.from_bytes(chunk)
|
||||
|
||||
def _send_response(self, status_code: int, headers: tuple[HttpHeader, ...], body: bytes) -> None:
|
||||
self.send_response_only(status_code)
|
||||
|
|
@ -145,38 +202,72 @@ def _recording_provider(spec: ProviderSpec) -> Generator[_RecordingProvider]:
|
|||
thread.join(timeout=5)
|
||||
|
||||
|
||||
def fixture_cache_key(case_input: MistralOcrSdkInput) -> dict[str, object]:
|
||||
def generate_case_inputs(strategy: SearchStrategy[InputT], examples: int) -> tuple[InputT, ...]:
|
||||
generated: Final[queue.SimpleQueue[InputT | None]] = queue.SimpleQueue()
|
||||
|
||||
@settings(max_examples=examples, deadline=None, derandomize=True)
|
||||
@given(case_input=strategy)
|
||||
def generate_case(case_input: InputT) -> None:
|
||||
generated.put(case_input)
|
||||
|
||||
generate_case()
|
||||
generated.put(None)
|
||||
return tuple(iter(generated.get, None))
|
||||
|
||||
|
||||
def fixture_cache_key(case_input: FixtureInput) -> dict[str, object]:
|
||||
return case_input.canonical_input()
|
||||
|
||||
|
||||
def _load_fixture(raw_fixture: dict[str, object], path: Path, case_type: type[CaseT]) -> CaseT:
|
||||
schema_version: Final = raw_fixture.get("schema_version")
|
||||
if schema_version != FIXTURE_SCHEMA_VERSION:
|
||||
raise ValueError(
|
||||
f"fixture {path} has schema_version {schema_version!r}, expected {FIXTURE_SCHEMA_VERSION}; "
|
||||
"delete it and regenerate the fixture bundle"
|
||||
)
|
||||
try:
|
||||
envelope: Final = FixtureEnvelope.model_validate(raw_fixture)
|
||||
return case_type.model_validate(envelope.case)
|
||||
except ValidationError as error:
|
||||
raise ValueError(f"invalid parity fixture {path} ({len(error.errors())} validation errors)") from error
|
||||
|
||||
|
||||
def record_case(
|
||||
spec: ProviderSpec,
|
||||
root: Path,
|
||||
case_input: MistralOcrSdkInput,
|
||||
sdk_call: Callable[..., object],
|
||||
) -> RecorderResult:
|
||||
case_input: InputT,
|
||||
sdk_call: Callable[[str, InputT], object],
|
||||
case_type: type[CaseT],
|
||||
) -> RecorderResult[CaseT]:
|
||||
cache: Final = JsonFileCache(root)
|
||||
cache_key: Final = fixture_cache_key(case_input)
|
||||
cached: Final = cache.get(cache_key)
|
||||
if cached is not None:
|
||||
return RecorderResult(case=OcrParityCase.model_validate(cached), cache_hit=True)
|
||||
return RecorderResult(case=_load_fixture(cached, cache.path_for(cache_key), case_type), cache_hit=True)
|
||||
|
||||
with _recording_provider(spec) as recorder:
|
||||
sdk_call(api_base=recorder.url, api_key=spec.api_key, **case_input.as_sdk_kwargs())
|
||||
sdk_call(recorder.url, case_input)
|
||||
upstream_response: Final = recorder.take_response()
|
||||
|
||||
case: Final = OcrParityCase(litellm_input=case_input, provider_response=upstream_response)
|
||||
cache.put(cache_key, cast(dict[str, object], case.model_dump(mode="json", exclude_unset=True)))
|
||||
case: Final = case_type.model_validate({"litellm_input": case_input, "provider_response": upstream_response})
|
||||
envelope: Final = FixtureEnvelope(
|
||||
schema_version=FIXTURE_SCHEMA_VERSION,
|
||||
recorded_at=datetime.now(timezone.utc),
|
||||
case=cast(dict[str, object], case.model_dump(mode="json", exclude_unset=True)),
|
||||
)
|
||||
cache.put(cache_key, cast(dict[str, object], envelope.model_dump(mode="json", exclude_unset=True)))
|
||||
return RecorderResult(case=case, cache_hit=False)
|
||||
|
||||
|
||||
def record_cases(
|
||||
spec: ProviderSpec,
|
||||
root: Path,
|
||||
case_inputs: tuple[MistralOcrSdkInput, ...],
|
||||
sdk_call: Callable[..., object],
|
||||
case_inputs: tuple[InputT, ...],
|
||||
sdk_call: Callable[[str, InputT], object],
|
||||
case_type: type[CaseT],
|
||||
max_concurrency: int,
|
||||
) -> tuple[RecorderResult, ...]:
|
||||
) -> tuple[RecorderResult[CaseT], ...]:
|
||||
if max_concurrency < 1:
|
||||
raise ValueError("max_concurrency must be at least 1")
|
||||
unique_inputs: Final = tuple(
|
||||
|
|
@ -184,7 +275,7 @@ def record_cases(
|
|||
)
|
||||
with ThreadPoolExecutor(max_workers=max_concurrency) as executor:
|
||||
futures: Final = tuple(
|
||||
executor.submit(record_case, spec, root, case_input, sdk_call) for case_input in unique_inputs
|
||||
executor.submit(record_case, spec, root, case_input, sdk_call, case_type) for case_input in unique_inputs
|
||||
)
|
||||
return tuple(future.result() for future in futures)
|
||||
|
||||
|
|
@ -193,24 +284,12 @@ def fixture_directory(configured: Path | None, env_value: str | None, default: P
|
|||
return (configured or Path(env_value or default)).expanduser()
|
||||
|
||||
|
||||
def recorded_fixtures(directory: Path) -> tuple[OcrParityCase, ...]:
|
||||
def recorded_fixtures(directory: Path, case_type: type[CaseT]) -> tuple[CaseT, ...]:
|
||||
cache: Final = JsonFileCache(directory)
|
||||
fixtures: list[OcrParityCase] = []
|
||||
for path, raw_fixture in cache.values_with_paths():
|
||||
try:
|
||||
fixtures.append(OcrParityCase.model_validate(raw_fixture))
|
||||
except ValidationError as error:
|
||||
raise ValueError(
|
||||
f"invalid OCR parity fixture {path}: expected exactly `litellm_input` and `provider_response` "
|
||||
f"({len(error.errors())} validation errors)"
|
||||
) from error
|
||||
return tuple(fixtures)
|
||||
return tuple(_load_fixture(raw_fixture, path, case_type) for path, raw_fixture in cache.values_with_paths())
|
||||
|
||||
|
||||
def fixture_id(fixture: OcrParityCase) -> str:
|
||||
input_json: Final = canonical_json(fixture.litellm_input.canonical_input())
|
||||
def fixture_id(case_input: FixtureInput, prefix: str) -> str:
|
||||
input_json: Final = canonical_json(case_input.canonical_input())
|
||||
digest: Final = hashlib.sha256(input_json.encode("utf-8")).hexdigest()[:8]
|
||||
model_id: Final = fixture.litellm_input.model.replace("/", "-")
|
||||
provider: Final = fixture.litellm_input.custom_llm_provider
|
||||
prefix: Final = f"{provider}-{model_id}" if provider and not model_id.startswith(f"{provider}-") else model_id
|
||||
return f"{prefix}-{digest}"
|
||||
|
|
|
|||
63
tests/test_litellm/_recorded_http.py
Normal file
63
tests/test_litellm/_recorded_http.py
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
class _RecordedHttpModel(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
|
||||
class HttpHeader(_RecordedHttpModel):
|
||||
name: str
|
||||
value: str
|
||||
|
||||
|
||||
class RecordedHttpResponse(_RecordedHttpModel):
|
||||
kind: Literal["http"]
|
||||
status_code: int
|
||||
headers: tuple[HttpHeader, ...]
|
||||
body_b64: str
|
||||
|
||||
@classmethod
|
||||
def from_bytes(
|
||||
cls,
|
||||
status_code: int,
|
||||
headers: tuple[HttpHeader, ...],
|
||||
body: bytes,
|
||||
) -> RecordedHttpResponse:
|
||||
return cls(
|
||||
kind="http",
|
||||
status_code=status_code,
|
||||
headers=headers,
|
||||
body_b64=base64.b64encode(body).decode("ascii"),
|
||||
)
|
||||
|
||||
def body_bytes(self) -> bytes:
|
||||
return base64.b64decode(self.body_b64, validate=True)
|
||||
|
||||
|
||||
class RecordedStreamChunk(_RecordedHttpModel):
|
||||
data_b64: str
|
||||
|
||||
@classmethod
|
||||
def from_bytes(cls, data: bytes) -> RecordedStreamChunk:
|
||||
return cls(data_b64=base64.b64encode(data).decode("ascii"))
|
||||
|
||||
def data_bytes(self) -> bytes:
|
||||
return base64.b64decode(self.data_b64, validate=True)
|
||||
|
||||
|
||||
class RecordedHttpStreamResponse(_RecordedHttpModel):
|
||||
kind: Literal["http_stream"]
|
||||
status_code: int
|
||||
headers: tuple[HttpHeader, ...]
|
||||
chunks: tuple[RecordedStreamChunk, ...]
|
||||
|
||||
|
||||
RecordedResponse = Annotated[
|
||||
RecordedHttpResponse | RecordedHttpStreamResponse,
|
||||
Field(discriminator="kind"),
|
||||
]
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
{
|
||||
"case": {
|
||||
"litellm_input": {
|
||||
"document": {
|
||||
"image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24",
|
||||
"type": "image_url"
|
||||
},
|
||||
"id": "case-1",
|
||||
"model": "mistral/mistral-ocr-latest"
|
||||
},
|
||||
"provider_response": {
|
||||
"body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsImJsb2NrcyI6W3sidG9wX2xlZnRfeCI6MzM1LCJ0b3BfbGVmdF95IjoxMzgsImJvdHRvbV9yaWdodF94Ijo0NjQsImJvdHRvbV9yaWdodF95IjoxNjIsImNvbnRlbnQiOiJpbnZvaWNlIDEyMyIsImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJ0eXBlIjoidGV4dCJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOm51bGwsInVzYWdlX2luZm8iOnsicGFnZXNfcHJvY2Vzc2VkIjoxLCJkb2Nfc2l6ZV9ieXRlcyI6NDEyNH19",
|
||||
"headers": [
|
||||
{
|
||||
"name": "Date",
|
||||
"value": "Sat, 29 Aug 2026 23:18:02 GMT"
|
||||
},
|
||||
{
|
||||
"name": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=xhZ4oMdiKbThFP71tBxvQtwQRHpnzzS31o6bhmCrX_o-1788045481.8243725-1.0.1.1-ju2kNNBEq.Ff7jRRXRNgRw7jODlxu2bjMlA9ni_JBOIo6Mw.5QKyVVN8wN38J.gLItzZCZAKXFSCWAqVM39oZ9NMBCNTuwXEbf6bM5NIOCn0LgE2DrvxielM2VJkpDan; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Sat, 29 Aug 2026 23:48:02 GMT"
|
||||
},
|
||||
{
|
||||
"name": "mistral-correlation-id",
|
||||
"value": "01a04fd0-d7b3-7279-a78d-c48ecdcde419"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-request-id",
|
||||
"value": "01a04fd0-d7b3-7279-a78d-c48ecdcde419"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-limit-ocr-pages-minute",
|
||||
"value": "60"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-remaining-ocr-pages-minute",
|
||||
"value": "59"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-ocr-pages-query-cost",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"name": "x-envoy-upstream-service-time",
|
||||
"value": "379"
|
||||
},
|
||||
{
|
||||
"name": "Server",
|
||||
"value": "cloudflare"
|
||||
},
|
||||
{
|
||||
"name": "access-control-allow-origin",
|
||||
"value": "*"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-upstream-latency",
|
||||
"value": "380"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-proxy-latency",
|
||||
"value": "12"
|
||||
},
|
||||
{
|
||||
"name": "X-Content-Type-Options",
|
||||
"value": "nosniff"
|
||||
},
|
||||
{
|
||||
"name": "cf-cache-status",
|
||||
"value": "DYNAMIC"
|
||||
},
|
||||
{
|
||||
"name": "Strict-Transport-Security",
|
||||
"value": "max-age=15552000; includeSubDomains; preload"
|
||||
},
|
||||
{
|
||||
"name": "CF-RAY",
|
||||
"value": "a32f45c56be31975-SJC"
|
||||
},
|
||||
{
|
||||
"name": "alt-svc",
|
||||
"value": "h3=\":443\"; ma=86400"
|
||||
}
|
||||
],
|
||||
"kind": "http",
|
||||
"status_code": 200
|
||||
}
|
||||
},
|
||||
"recorded_at": "2026-08-29T23:18:02.736046Z",
|
||||
"schema_version": 1
|
||||
}
|
||||
|
|
@ -1,95 +0,0 @@
|
|||
{
|
||||
"litellm_input": {
|
||||
"document": {
|
||||
"image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=N&fontSize=33",
|
||||
"type": "image_url"
|
||||
},
|
||||
"extract_footer": true,
|
||||
"extract_header": false,
|
||||
"id": "-0mq4q9pmig_rn9x2-vn",
|
||||
"image_limit": 66,
|
||||
"image_min_size": 1320,
|
||||
"include_blocks": true,
|
||||
"include_image_base64": false,
|
||||
"model": "mistral/mistral-ocr-latest",
|
||||
"table_format": "html"
|
||||
},
|
||||
"provider_response": {
|
||||
"body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiTiIsImltYWdlcyI6W10sInRhYmxlcyI6W10sImh5cGVybGlua3MiOltdLCJoZWFkZXIiOm51bGwsImZvb3RlciI6bnVsbCwiZGltZW5zaW9ucyI6eyJkcGkiOjIwMCwiaGVpZ2h0IjozMDAsIndpZHRoIjo4MDB9LCJjb25maWRlbmNlX3Njb3JlcyI6bnVsbCwiYmxvY2tzIjpbeyJ0b3BfbGVmdF94IjozODYsInRvcF9sZWZ0X3kiOjEzNSwiYm90dG9tX3JpZ2h0X3giOjQxMiwiYm90dG9tX3JpZ2h0X3kiOjE2MiwiY29udGVudCI6Ik4iLCJjb25maWRlbmNlX3Njb3JlcyI6bnVsbCwidHlwZSI6InRleHQifV19XSwibW9kZWwiOiJtaXN0cmFsLW9jci1sYXRlc3QiLCJkb2N1bWVudF9hbm5vdGF0aW9uIjpudWxsLCJ1c2FnZV9pbmZvIjp7InBhZ2VzX3Byb2Nlc3NlZCI6MSwiZG9jX3NpemVfYnl0ZXMiOjIyOTh9fQ==",
|
||||
"headers": [
|
||||
{
|
||||
"name": "Date",
|
||||
"value": "Sat, 29 Aug 2026 21:29:07 GMT"
|
||||
},
|
||||
{
|
||||
"name": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=vwrWUQPvG4As7fL9Gl1e.lxaZ4I1lG_9uVgrmykYadg-1788038947.3819609-1.0.1.1-4ibtTa5G29xCi1ADh.G_kJ.3l0T2gPlt.1mPNdaJOJNjekO4DR4yYlCCXhucA3Lq45RPrVMxxkPH2YFZ887YrReeGS9x0E05ZO58vqbcJhwfTGLh.3xJSmr.0471Xw4Q; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Sat, 29 Aug 2026 21:59:07 GMT"
|
||||
},
|
||||
{
|
||||
"name": "mistral-correlation-id",
|
||||
"value": "01a04f6d-228b-7cdc-9a1a-b99123c027c3"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-request-id",
|
||||
"value": "01a04f6d-228b-7cdc-9a1a-b99123c027c3"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-limit-ocr-pages-minute",
|
||||
"value": "60"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-remaining-ocr-pages-minute",
|
||||
"value": "56"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-ocr-pages-query-cost",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"name": "x-envoy-upstream-service-time",
|
||||
"value": "395"
|
||||
},
|
||||
{
|
||||
"name": "Server",
|
||||
"value": "cloudflare"
|
||||
},
|
||||
{
|
||||
"name": "access-control-allow-origin",
|
||||
"value": "*"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-upstream-latency",
|
||||
"value": "396"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-proxy-latency",
|
||||
"value": "17"
|
||||
},
|
||||
{
|
||||
"name": "X-Content-Type-Options",
|
||||
"value": "nosniff"
|
||||
},
|
||||
{
|
||||
"name": "cf-cache-status",
|
||||
"value": "DYNAMIC"
|
||||
},
|
||||
{
|
||||
"name": "Strict-Transport-Security",
|
||||
"value": "max-age=15552000; includeSubDomains; preload"
|
||||
},
|
||||
{
|
||||
"name": "CF-RAY",
|
||||
"value": "a32ea63d2ad30065-SJC"
|
||||
},
|
||||
{
|
||||
"name": "alt-svc",
|
||||
"value": "h3=\":443\"; ma=86400"
|
||||
}
|
||||
],
|
||||
"status_code": 200
|
||||
}
|
||||
}
|
||||
|
|
@ -1,90 +0,0 @@
|
|||
{
|
||||
"litellm_input": {
|
||||
"document": {
|
||||
"image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=ZPay&fontSize=24",
|
||||
"type": "image_url"
|
||||
},
|
||||
"id": "utdwr1axd37",
|
||||
"image_min_size": 4099,
|
||||
"model": "mistral/mistral-ocr-latest",
|
||||
"table_format": "markdown"
|
||||
},
|
||||
"provider_response": {
|
||||
"body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiWlBheSIsImltYWdlcyI6W10sInRhYmxlcyI6W10sImh5cGVybGlua3MiOltdLCJoZWFkZXIiOm51bGwsImZvb3RlciI6bnVsbCwiZGltZW5zaW9ucyI6eyJkcGkiOjIwMCwiaGVpZ2h0IjozMDAsIndpZHRoIjo4MDB9LCJjb25maWRlbmNlX3Njb3JlcyI6bnVsbCwiYmxvY2tzIjpbeyJ0b3BfbGVmdF94IjozNzAsInRvcF9sZWZ0X3kiOjEzOCwiYm90dG9tX3JpZ2h0X3giOjQzMCwiYm90dG9tX3JpZ2h0X3kiOjE2NSwiY29udGVudCI6IlpQYXkiLCJjb25maWRlbmNlX3Njb3JlcyI6bnVsbCwidHlwZSI6InRleHQifV19XSwibW9kZWwiOiJtaXN0cmFsLW9jci1sYXRlc3QiLCJkb2N1bWVudF9hbm5vdGF0aW9uIjpudWxsLCJ1c2FnZV9pbmZvIjp7InBhZ2VzX3Byb2Nlc3NlZCI6MSwiZG9jX3NpemVfYnl0ZXMiOjMxNDd9fQ==",
|
||||
"headers": [
|
||||
{
|
||||
"name": "Date",
|
||||
"value": "Sat, 29 Aug 2026 21:29:05 GMT"
|
||||
},
|
||||
{
|
||||
"name": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=_.yDONMxI8Wj6bdeKvgLqylB1uRdIzfx64ReOxc5C1o-1788038945.3530984-1.0.1.1-uwAg3dYolu9XCaCCJsie9eFRumBnKC1ZyQDkHA1Otpe_IUrpEsnokSCiyyPun1WOFenihkYZTLs7xMCwSkdbEUs1gN2O2WkWnKL3v_PA36X9McSO1o5iIlm84TAqmGOJ; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Sat, 29 Aug 2026 21:59:05 GMT"
|
||||
},
|
||||
{
|
||||
"name": "mistral-correlation-id",
|
||||
"value": "01a04f6d-1a9e-7b54-a1ef-e40d7988227f"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-request-id",
|
||||
"value": "01a04f6d-1a9e-7b54-a1ef-e40d7988227f"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-limit-ocr-pages-minute",
|
||||
"value": "60"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-remaining-ocr-pages-minute",
|
||||
"value": "58"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-ocr-pages-query-cost",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"name": "x-envoy-upstream-service-time",
|
||||
"value": "388"
|
||||
},
|
||||
{
|
||||
"name": "Server",
|
||||
"value": "cloudflare"
|
||||
},
|
||||
{
|
||||
"name": "access-control-allow-origin",
|
||||
"value": "*"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-upstream-latency",
|
||||
"value": "389"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-proxy-latency",
|
||||
"value": "13"
|
||||
},
|
||||
{
|
||||
"name": "X-Content-Type-Options",
|
||||
"value": "nosniff"
|
||||
},
|
||||
{
|
||||
"name": "cf-cache-status",
|
||||
"value": "DYNAMIC"
|
||||
},
|
||||
{
|
||||
"name": "Strict-Transport-Security",
|
||||
"value": "max-age=15552000; includeSubDomains; preload"
|
||||
},
|
||||
{
|
||||
"name": "CF-RAY",
|
||||
"value": "a32ea6307e391be5-SJC"
|
||||
},
|
||||
{
|
||||
"name": "alt-svc",
|
||||
"value": "h3=\":443\"; ma=86400"
|
||||
}
|
||||
],
|
||||
"status_code": 200
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,139 @@
|
|||
{
|
||||
"case": {
|
||||
"litellm_input": {
|
||||
"bbox_annotation_format": {
|
||||
"json_schema": {
|
||||
"description": "Extract the visible document fields",
|
||||
"name": "bounding_boxes",
|
||||
"schema": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"title": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"title"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"strict": true
|
||||
},
|
||||
"type": "json_schema"
|
||||
},
|
||||
"confidence_scores_granularity": "page",
|
||||
"document": {
|
||||
"image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24",
|
||||
"type": "image_url"
|
||||
},
|
||||
"document_annotation_format": {
|
||||
"json_schema": {
|
||||
"description": "Extract the visible document fields",
|
||||
"name": "document_title",
|
||||
"schema": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"title": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"title"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"strict": true
|
||||
},
|
||||
"type": "json_schema"
|
||||
},
|
||||
"id": "case-1",
|
||||
"image_min_size": 300,
|
||||
"include_blocks": false,
|
||||
"model": "mistral/mistral-ocr-latest",
|
||||
"pages": [
|
||||
0
|
||||
]
|
||||
},
|
||||
"provider_response": {
|
||||
"body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOnsid29yZF9jb25maWRlbmNlX3Njb3JlcyI6W10sImF2ZXJhZ2VfcGFnZV9jb25maWRlbmNlX3Njb3JlIjowLjkwNjc4MjgyOTY4MDA1MzIsIm1pbmltdW1fcGFnZV9jb25maWRlbmNlX3Njb3JlIjowLjE1NTE4MTc1NjU3MDQ0MDZ9LCJibG9ja3MiOm51bGx9XSwibW9kZWwiOiJtaXN0cmFsLW9jci1sYXRlc3QiLCJkb2N1bWVudF9hbm5vdGF0aW9uIjoie1widGl0bGVcIjogXCJJbnZvaWNlXzEyM1wifSIsInVzYWdlX2luZm8iOnsicGFnZXNfcHJvY2Vzc2VkIjoxLCJkb2Nfc2l6ZV9ieXRlcyI6NDEyNH19",
|
||||
"headers": [
|
||||
{
|
||||
"name": "Date",
|
||||
"value": "Sat, 29 Aug 2026 23:18:02 GMT"
|
||||
},
|
||||
{
|
||||
"name": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"name": "mistral-correlation-id",
|
||||
"value": "01a04fd0-d7c1-7f2e-b3c0-a5901da37f33"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-request-id",
|
||||
"value": "01a04fd0-d7c1-7f2e-b3c0-a5901da37f33"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-limit-ocr-pages-minute",
|
||||
"value": "60"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-remaining-ocr-pages-minute",
|
||||
"value": "59"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-ocr-pages-query-cost",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"name": "x-envoy-upstream-service-time",
|
||||
"value": "783"
|
||||
},
|
||||
{
|
||||
"name": "Server",
|
||||
"value": "cloudflare"
|
||||
},
|
||||
{
|
||||
"name": "access-control-allow-origin",
|
||||
"value": "*"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-upstream-latency",
|
||||
"value": "784"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-proxy-latency",
|
||||
"value": "17"
|
||||
},
|
||||
{
|
||||
"name": "cf-cache-status",
|
||||
"value": "DYNAMIC"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=MplX_p4RI9Z_0LCIvJqPXtjSY6xAps03mWqX2ICuNHM-1788045481.837575-1.0.1.1-8RtNOBgVlARI3DaJl3.wto8CKnDIMFj4rqK5kX7i8rw5N03V7zwUSwaL6q7nus4bqA.3tX_qbM7iqdjvnFGJSRiV4ytibS5ajPPs645Xh_8vVTDzPSgpCzXDFMRnhmnk; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Sat, 29 Aug 2026 23:48:02 GMT"
|
||||
},
|
||||
{
|
||||
"name": "Strict-Transport-Security",
|
||||
"value": "max-age=15552000; includeSubDomains; preload"
|
||||
},
|
||||
{
|
||||
"name": "X-Content-Type-Options",
|
||||
"value": "nosniff"
|
||||
},
|
||||
{
|
||||
"name": "CF-RAY",
|
||||
"value": "a32f45c57d28faec-SJC"
|
||||
},
|
||||
{
|
||||
"name": "alt-svc",
|
||||
"value": "h3=\":443\"; ma=86400"
|
||||
}
|
||||
],
|
||||
"kind": "http",
|
||||
"status_code": 200
|
||||
}
|
||||
},
|
||||
"recorded_at": "2026-08-29T23:18:03.235228Z",
|
||||
"schema_version": 1
|
||||
}
|
||||
|
|
@ -1,87 +0,0 @@
|
|||
{
|
||||
"litellm_input": {
|
||||
"document": {
|
||||
"image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=0&fontSize=12",
|
||||
"type": "image_url"
|
||||
},
|
||||
"model": "mistral/mistral-ocr-latest"
|
||||
},
|
||||
"provider_response": {
|
||||
"body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiMCIsImltYWdlcyI6W10sInRhYmxlcyI6W10sImh5cGVybGlua3MiOltdLCJoZWFkZXIiOm51bGwsImZvb3RlciI6bnVsbCwiZGltZW5zaW9ucyI6eyJkcGkiOjIwMCwiaGVpZ2h0IjozMDAsIndpZHRoIjo4MDB9LCJjb25maWRlbmNlX3Njb3JlcyI6bnVsbCwiYmxvY2tzIjpbeyJ0b3BfbGVmdF94IjozOTIsInRvcF9sZWZ0X3kiOjE0NCwiYm90dG9tX3JpZ2h0X3giOjQwOCwiYm90dG9tX3JpZ2h0X3kiOjE1NywiY29udGVudCI6IjAiLCJjb25maWRlbmNlX3Njb3JlcyI6bnVsbCwidHlwZSI6ImZvb3RlciJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOm51bGwsInVzYWdlX2luZm8iOnsicGFnZXNfcHJvY2Vzc2VkIjoxLCJkb2Nfc2l6ZV9ieXRlcyI6MTkxMn19",
|
||||
"headers": [
|
||||
{
|
||||
"name": "Date",
|
||||
"value": "Sat, 29 Aug 2026 21:29:05 GMT"
|
||||
},
|
||||
{
|
||||
"name": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"name": "mistral-correlation-id",
|
||||
"value": "01a04f6d-17f4-7ff9-a890-8e93430da5a7"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-request-id",
|
||||
"value": "01a04f6d-17f4-7ff9-a890-8e93430da5a7"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-limit-ocr-pages-minute",
|
||||
"value": "60"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-remaining-ocr-pages-minute",
|
||||
"value": "59"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-ocr-pages-query-cost",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"name": "x-envoy-upstream-service-time",
|
||||
"value": "252"
|
||||
},
|
||||
{
|
||||
"name": "Server",
|
||||
"value": "cloudflare"
|
||||
},
|
||||
{
|
||||
"name": "access-control-allow-origin",
|
||||
"value": "*"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-upstream-latency",
|
||||
"value": "252"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-proxy-latency",
|
||||
"value": "16"
|
||||
},
|
||||
{
|
||||
"name": "cf-cache-status",
|
||||
"value": "DYNAMIC"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=Bu1ZsZArEglH4RxFKY1uja22ICsIoWxY3EHWLjhl0.U-1788038944.6771533-1.0.1.1-sq3d4oadvs6YUQHDueFHRtdC9k2PXHPeqfHHJ1aD95baBbevXjDnDKJVfn_7Yb.pLQ3kNVEzucNmUGFFbyYk0W7q_rFe54PLou5QgObpMaWYgrectVNvGi0cwNZeqH24; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Sat, 29 Aug 2026 21:59:05 GMT"
|
||||
},
|
||||
{
|
||||
"name": "Strict-Transport-Security",
|
||||
"value": "max-age=15552000; includeSubDomains; preload"
|
||||
},
|
||||
{
|
||||
"name": "X-Content-Type-Options",
|
||||
"value": "nosniff"
|
||||
},
|
||||
{
|
||||
"name": "CF-RAY",
|
||||
"value": "a32ea62c3f91cf22-SJC"
|
||||
},
|
||||
{
|
||||
"name": "alt-svc",
|
||||
"value": "h3=\":443\"; ma=86400"
|
||||
}
|
||||
],
|
||||
"status_code": 200
|
||||
}
|
||||
}
|
||||
|
|
@ -1,88 +0,0 @@
|
|||
{
|
||||
"litellm_input": {
|
||||
"document": {
|
||||
"image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=AuUPy5&fontSize=35",
|
||||
"type": "image_url"
|
||||
},
|
||||
"include_image_base64": true,
|
||||
"model": "mistral/mistral-ocr-latest"
|
||||
},
|
||||
"provider_response": {
|
||||
"body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiQXVVUHk1IiwiaW1hZ2VzIjpbXSwidGFibGVzIjpbXSwiaHlwZXJsaW5rcyI6W10sImhlYWRlciI6bnVsbCwiZm9vdGVyIjpudWxsLCJkaW1lbnNpb25zIjp7ImRwaSI6MjAwLCJoZWlnaHQiOjMwMCwid2lkdGgiOjgwMH0sImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJibG9ja3MiOlt7InRvcF9sZWZ0X3giOjMyOSwidG9wX2xlZnRfeSI6MTM0LCJib3R0b21fcmlnaHRfeCI6NDY5LCJib3R0b21fcmlnaHRfeSI6MTcyLCJjb250ZW50IjoiQXVVUHk1IiwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsInR5cGUiOiJ0ZXh0In1dfV0sIm1vZGVsIjoibWlzdHJhbC1vY3ItbGF0ZXN0IiwiZG9jdW1lbnRfYW5ub3RhdGlvbiI6bnVsbCwidXNhZ2VfaW5mbyI6eyJwYWdlc19wcm9jZXNzZWQiOjEsImRvY19zaXplX2J5dGVzIjo0NDQ5fX0=",
|
||||
"headers": [
|
||||
{
|
||||
"name": "Date",
|
||||
"value": "Sat, 29 Aug 2026 21:29:06 GMT"
|
||||
},
|
||||
{
|
||||
"name": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"name": "mistral-correlation-id",
|
||||
"value": "01a04f6d-1e94-7169-acf2-331366511208"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-request-id",
|
||||
"value": "01a04f6d-1e94-7169-acf2-331366511208"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-limit-ocr-pages-minute",
|
||||
"value": "60"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-remaining-ocr-pages-minute",
|
||||
"value": "57"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-ocr-pages-query-cost",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"name": "x-envoy-upstream-service-time",
|
||||
"value": "299"
|
||||
},
|
||||
{
|
||||
"name": "Server",
|
||||
"value": "cloudflare"
|
||||
},
|
||||
{
|
||||
"name": "access-control-allow-origin",
|
||||
"value": "*"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-upstream-latency",
|
||||
"value": "301"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-proxy-latency",
|
||||
"value": "13"
|
||||
},
|
||||
{
|
||||
"name": "cf-cache-status",
|
||||
"value": "DYNAMIC"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=L.iPQSMbMkUfZ6bZqRUnIvFouaL6uaka9sb1RADqE7I-1788038946.371394-1.0.1.1-eP7TRle_nNnnCddZ21LX.mzRWslGV0sv5wZONgMz.g2Es_qbFOGUuPv6K1C0p86Pz6xULpoaVjmqxUOkza6Ug_53VGsNkwHRvyxXuKXvMSB62zZPyrO0PcwTqRZKYFLC; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Sat, 29 Aug 2026 21:59:06 GMT"
|
||||
},
|
||||
{
|
||||
"name": "Strict-Transport-Security",
|
||||
"value": "max-age=15552000; includeSubDomains; preload"
|
||||
},
|
||||
{
|
||||
"name": "X-Content-Type-Options",
|
||||
"value": "nosniff"
|
||||
},
|
||||
{
|
||||
"name": "CF-RAY",
|
||||
"value": "a32ea636dd4eebe2-SJC"
|
||||
},
|
||||
{
|
||||
"name": "alt-svc",
|
||||
"value": "h3=\":443\"; ma=86400"
|
||||
}
|
||||
],
|
||||
"status_code": 200
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
{
|
||||
"case": {
|
||||
"litellm_input": {
|
||||
"document": {
|
||||
"image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24",
|
||||
"type": "image_url"
|
||||
},
|
||||
"model": "mistral/mistral-ocr-latest"
|
||||
},
|
||||
"provider_response": {
|
||||
"body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsImJsb2NrcyI6W3sidG9wX2xlZnRfeCI6MzM1LCJ0b3BfbGVmdF95IjoxMzgsImJvdHRvbV9yaWdodF94Ijo0NjQsImJvdHRvbV9yaWdodF95IjoxNjIsImNvbnRlbnQiOiJpbnZvaWNlIDEyMyIsImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJ0eXBlIjoidGV4dCJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOm51bGwsInVzYWdlX2luZm8iOnsicGFnZXNfcHJvY2Vzc2VkIjoxLCJkb2Nfc2l6ZV9ieXRlcyI6NDEyNH19",
|
||||
"headers": [
|
||||
{
|
||||
"name": "Date",
|
||||
"value": "Sat, 29 Aug 2026 23:18:02 GMT"
|
||||
},
|
||||
{
|
||||
"name": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=4Y4BNG5seFc4759BsUBjy4Z2odUy3Rgf0SOYkyESdPA-1788045481.8373353-1.0.1.1-eirQC2neaOT8r6_9qxcWeUjOMH7L1NmsXj2YyMbcVmTO62TWrDCjf7VqxKigiJ1lViQQkJQwVpMsSHTfUOaziX4zDTloRi4b.oaA37lqE.gBbH9FylA_nMTl2mYg65X9; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Sat, 29 Aug 2026 23:48:02 GMT"
|
||||
},
|
||||
{
|
||||
"name": "mistral-correlation-id",
|
||||
"value": "01a04fd0-d7b8-799c-9ac2-48ab880605d6"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-request-id",
|
||||
"value": "01a04fd0-d7b8-799c-9ac2-48ab880605d6"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-limit-ocr-pages-minute",
|
||||
"value": "60"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-remaining-ocr-pages-minute",
|
||||
"value": "59"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-ocr-pages-query-cost",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"name": "x-envoy-upstream-service-time",
|
||||
"value": "355"
|
||||
},
|
||||
{
|
||||
"name": "Server",
|
||||
"value": "cloudflare"
|
||||
},
|
||||
{
|
||||
"name": "access-control-allow-origin",
|
||||
"value": "*"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-upstream-latency",
|
||||
"value": "356"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-proxy-latency",
|
||||
"value": "17"
|
||||
},
|
||||
{
|
||||
"name": "X-Content-Type-Options",
|
||||
"value": "nosniff"
|
||||
},
|
||||
{
|
||||
"name": "cf-cache-status",
|
||||
"value": "DYNAMIC"
|
||||
},
|
||||
{
|
||||
"name": "Strict-Transport-Security",
|
||||
"value": "max-age=15552000; includeSubDomains; preload"
|
||||
},
|
||||
{
|
||||
"name": "CF-RAY",
|
||||
"value": "a32f45c57f8e7834-SJC"
|
||||
},
|
||||
{
|
||||
"name": "alt-svc",
|
||||
"value": "h3=\":443\"; ma=86400"
|
||||
}
|
||||
],
|
||||
"kind": "http",
|
||||
"status_code": 200
|
||||
}
|
||||
},
|
||||
"recorded_at": "2026-08-29T23:18:02.735198Z",
|
||||
"schema_version": 1
|
||||
}
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
{
|
||||
"case": {
|
||||
"litellm_input": {
|
||||
"document": {
|
||||
"image_url": "https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24",
|
||||
"type": "image_url"
|
||||
},
|
||||
"document_annotation_format": {
|
||||
"json_schema": {
|
||||
"description": "Extract the visible document fields",
|
||||
"name": "prompted_document_title",
|
||||
"schema": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"title": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"title"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"strict": true
|
||||
},
|
||||
"type": "json_schema"
|
||||
},
|
||||
"document_annotation_prompt": "Extract the visible title",
|
||||
"include_image_base64": true,
|
||||
"model": "mistral/mistral-ocr-latest",
|
||||
"pages": [
|
||||
0
|
||||
]
|
||||
},
|
||||
"provider_response": {
|
||||
"body_b64": "eyJwYWdlcyI6W3siaW5kZXgiOjAsIm1hcmtkb3duIjoiaW52b2ljZSAxMjMiLCJpbWFnZXMiOltdLCJ0YWJsZXMiOltdLCJoeXBlcmxpbmtzIjpbXSwiaGVhZGVyIjpudWxsLCJmb290ZXIiOm51bGwsImRpbWVuc2lvbnMiOnsiZHBpIjoyMDAsImhlaWdodCI6MzAwLCJ3aWR0aCI6ODAwfSwiY29uZmlkZW5jZV9zY29yZXMiOm51bGwsImJsb2NrcyI6W3sidG9wX2xlZnRfeCI6MzM1LCJ0b3BfbGVmdF95IjoxMzgsImJvdHRvbV9yaWdodF94Ijo0NjQsImJvdHRvbV9yaWdodF95IjoxNjIsImNvbnRlbnQiOiJpbnZvaWNlIDEyMyIsImNvbmZpZGVuY2Vfc2NvcmVzIjpudWxsLCJ0eXBlIjoidGV4dCJ9XX1dLCJtb2RlbCI6Im1pc3RyYWwtb2NyLWxhdGVzdCIsImRvY3VtZW50X2Fubm90YXRpb24iOiJ7XCJ0aXRsZVwiOiBcImludm9pY2UgMTIzXCJ9IiwidXNhZ2VfaW5mbyI6eyJwYWdlc19wcm9jZXNzZWQiOjEsImRvY19zaXplX2J5dGVzIjo0MTI0fX0=",
|
||||
"headers": [
|
||||
{
|
||||
"name": "Date",
|
||||
"value": "Sat, 29 Aug 2026 23:18:02 GMT"
|
||||
},
|
||||
{
|
||||
"name": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=vLaAHFXIItKaEkQQMMAH_E.rAMYc7fuSd6Evsu2SbhY-1788045481.8225453-1.0.1.1-L.S7ARgAGlXrNvVQ7ETwoVMFhVRT25B9gxsNxfaHcYVZ9Tt_36StIe5sxCWGZzWeMfhVJhVBsxNEADhO._6d_iPdnanYVrN0CA0EmP2pt8mDq0j2sEb_bdco7eTaJinM; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Sat, 29 Aug 2026 23:48:02 GMT"
|
||||
},
|
||||
{
|
||||
"name": "mistral-correlation-id",
|
||||
"value": "01a04fd0-d7b6-7819-aa2d-0d1906d915e2"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-request-id",
|
||||
"value": "01a04fd0-d7b6-7819-aa2d-0d1906d915e2"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-limit-ocr-pages-minute",
|
||||
"value": "60"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-remaining-ocr-pages-minute",
|
||||
"value": "59"
|
||||
},
|
||||
{
|
||||
"name": "x-ratelimit-ocr-pages-query-cost",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"name": "x-envoy-upstream-service-time",
|
||||
"value": "620"
|
||||
},
|
||||
{
|
||||
"name": "Server",
|
||||
"value": "cloudflare"
|
||||
},
|
||||
{
|
||||
"name": "access-control-allow-origin",
|
||||
"value": "*"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-upstream-latency",
|
||||
"value": "621"
|
||||
},
|
||||
{
|
||||
"name": "x-kong-proxy-latency",
|
||||
"value": "12"
|
||||
},
|
||||
{
|
||||
"name": "X-Content-Type-Options",
|
||||
"value": "nosniff"
|
||||
},
|
||||
{
|
||||
"name": "cf-cache-status",
|
||||
"value": "DYNAMIC"
|
||||
},
|
||||
{
|
||||
"name": "Strict-Transport-Security",
|
||||
"value": "max-age=15552000; includeSubDomains; preload"
|
||||
},
|
||||
{
|
||||
"name": "CF-RAY",
|
||||
"value": "a32f45c56cd35d19-SJC"
|
||||
},
|
||||
{
|
||||
"name": "alt-svc",
|
||||
"value": "h3=\":443\"; ma=86400"
|
||||
}
|
||||
],
|
||||
"kind": "http",
|
||||
"status_code": 200
|
||||
}
|
||||
},
|
||||
"recorded_at": "2026-08-29T23:18:02.736175Z",
|
||||
"schema_version": 1
|
||||
}
|
||||
|
|
@ -8,6 +8,7 @@ import pytest
|
|||
from pydantic import ValidationError
|
||||
|
||||
from tests.test_litellm._fixture_recorder import fixture_id, recorded_fixtures
|
||||
from tests.test_litellm.ocr.fixture_models import OcrParityCase
|
||||
|
||||
FIXTURE_DIR_ENV: Final = "LITELLM_OCR_FIXTURE_DIR"
|
||||
|
||||
|
|
@ -21,16 +22,24 @@ def _fixture_directory() -> Path:
|
|||
return Path(configured).expanduser()
|
||||
|
||||
|
||||
def _fixture_id(fixture: OcrParityCase) -> str:
|
||||
case_input: Final = fixture.litellm_input
|
||||
model_id: Final = case_input.model.replace("/", "-")
|
||||
provider: Final = case_input.custom_llm_provider
|
||||
prefix: Final = f"{provider}-{model_id}" if provider and not model_id.startswith(f"{provider}-") else model_id
|
||||
return fixture_id(case_input, prefix)
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
|
||||
if "ocr_fixture" not in metafunc.fixturenames:
|
||||
return
|
||||
directory: Final = _fixture_directory()
|
||||
try:
|
||||
fixtures: Final = recorded_fixtures(directory)
|
||||
fixtures: Final = recorded_fixtures(directory, OcrParityCase)
|
||||
except (ValidationError, ValueError) as error:
|
||||
raise pytest.UsageError(
|
||||
f"Invalid OCR parity fixture bundle at {directory}. "
|
||||
"Each fixture must contain exactly `litellm_input` and `provider_response`. "
|
||||
"Each fixture must use the current versioned envelope. "
|
||||
"Record fresh fixtures in an empty directory with: "
|
||||
f"`uv run python tests/test_litellm/ocr/generate_fixtures.py --fixture-dir {directory}`. "
|
||||
f"Validation details: {error}"
|
||||
|
|
@ -49,4 +58,4 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
|
|||
),
|
||||
)
|
||||
return
|
||||
metafunc.parametrize("ocr_fixture", fixtures, ids=tuple(fixture_id(fixture) for fixture in fixtures))
|
||||
metafunc.parametrize("ocr_fixture", fixtures, ids=tuple(_fixture_id(fixture) for fixture in fixtures))
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ from typing import Annotated, Literal, cast
|
|||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator
|
||||
from typing_extensions import Self
|
||||
|
||||
from tests.test_litellm._recorded_http import RecordedResponse
|
||||
|
||||
JsonObject = dict[str, JsonValue]
|
||||
|
||||
|
||||
|
|
@ -283,34 +285,6 @@ class ReductoParseLegacySdkInput(OcrSdkInputBase):
|
|||
return self
|
||||
|
||||
|
||||
class HttpHeader(_FixtureModel):
|
||||
name: str
|
||||
value: str
|
||||
|
||||
|
||||
class RecordedHttpResponse(_FixtureModel):
|
||||
kind: Literal["http"] = "http"
|
||||
status_code: int
|
||||
headers: tuple[HttpHeader, ...]
|
||||
body_b64: str
|
||||
|
||||
@classmethod
|
||||
def from_bytes(
|
||||
cls,
|
||||
status_code: int,
|
||||
headers: tuple[HttpHeader, ...],
|
||||
body: bytes,
|
||||
) -> RecordedHttpResponse:
|
||||
return cls(
|
||||
status_code=status_code,
|
||||
headers=headers,
|
||||
body_b64=base64.b64encode(body).decode("ascii"),
|
||||
)
|
||||
|
||||
def body_bytes(self) -> bytes:
|
||||
return base64.b64decode(self.body_b64, validate=True)
|
||||
|
||||
|
||||
class OcrParityCase(_FixtureModel):
|
||||
litellm_input: MistralOcrSdkInput
|
||||
provider_response: RecordedHttpResponse
|
||||
provider_response: RecordedResponse
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ from __future__ import annotations
|
|||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
|
@ -11,7 +10,6 @@ from typing import Final, cast
|
|||
from urllib.parse import quote
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from hypothesis import given, settings
|
||||
from hypothesis import strategies as st
|
||||
from hypothesis.strategies import DrawFn, SearchStrategy
|
||||
|
||||
|
|
@ -20,14 +18,15 @@ from litellm.rust_bridge.ocr import use_litellm_rust
|
|||
from tests.test_litellm._fixture_recorder import (
|
||||
ProviderSpec,
|
||||
fixture_directory,
|
||||
generate_case_inputs,
|
||||
record_cases,
|
||||
)
|
||||
from tests.test_litellm.ocr.fixture_models import (
|
||||
JsonSchemaDefinition,
|
||||
JsonSchemaResponseFormat,
|
||||
MistralImageUrlDocument,
|
||||
MistralModel,
|
||||
MistralOcrSdkInput,
|
||||
OcrParityCase,
|
||||
ReductoChunking,
|
||||
ReductoDocumentUrlDocument,
|
||||
ReductoFormatting,
|
||||
|
|
@ -42,13 +41,6 @@ LOGGER: Final = logging.getLogger(__name__)
|
|||
_TEXT: Final = st.just("invoice 123")
|
||||
_VALUE_TEXT: Final = st.just("case-1")
|
||||
_FONT_SIZE: Final = st.just(24)
|
||||
_MISTRAL_MODELS: Final = (
|
||||
"mistral/mistral-ocr-2512",
|
||||
"mistral/mistral-ocr-4-0",
|
||||
"mistral/mistral-ocr-4-1",
|
||||
"mistral/mistral-ocr-4",
|
||||
"mistral/mistral-ocr-latest",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -245,21 +237,18 @@ def reducto_legacy_input_strategy() -> SearchStrategy[ReductoParseLegacySdkInput
|
|||
def _generate_examples(
|
||||
spec: ProviderSpec,
|
||||
root: Path,
|
||||
model: str,
|
||||
api_key: str,
|
||||
examples: int,
|
||||
concurrency: int,
|
||||
sdk_call: Callable[..., object],
|
||||
) -> None:
|
||||
generated: Final[queue.SimpleQueue[MistralOcrSdkInput | None]] = queue.SimpleQueue()
|
||||
case_inputs: Final = generate_case_inputs(mistral_input_strategy(model), examples)
|
||||
|
||||
@settings(max_examples=examples, deadline=None, derandomize=True)
|
||||
@given(case_input=mistral_input_strategy(spec.model))
|
||||
def generate_case(case_input: MistralOcrSdkInput) -> None:
|
||||
generated.put(case_input)
|
||||
def invoke(api_base: str, case_input: MistralOcrSdkInput) -> object:
|
||||
return sdk_call(api_base=api_base, api_key=api_key, **case_input.as_sdk_kwargs())
|
||||
|
||||
generate_case()
|
||||
generated.put(None)
|
||||
case_inputs: Final = tuple(iter(generated.get, None))
|
||||
results: Final = record_cases(spec, root, case_inputs, sdk_call, concurrency)
|
||||
results: Final = record_cases(spec, root, case_inputs, invoke, OcrParityCase, concurrency)
|
||||
for result in results:
|
||||
LOGGER.info("%s %s", "cached" if result.cache_hit else "recorded", result.case.litellm_input.model)
|
||||
|
||||
|
|
@ -274,13 +263,13 @@ def _parse_args() -> GeneratorArgs:
|
|||
parser.add_argument("--concurrency", type=int, default=4)
|
||||
parser.add_argument("--examples", type=int, default=4)
|
||||
parser.add_argument("--fixture-dir", type=Path)
|
||||
parser.add_argument("--model", choices=_MISTRAL_MODELS, default="mistral/mistral-ocr-latest")
|
||||
parser.add_argument("--model", required=True)
|
||||
namespace: Final = parser.parse_args()
|
||||
return GeneratorArgs(
|
||||
concurrency=cast(int, namespace.concurrency),
|
||||
examples=cast(int, namespace.examples),
|
||||
fixture_dir=cast(Path | None, namespace.fixture_dir),
|
||||
model=cast(MistralModel, namespace.model),
|
||||
model=cast(str, namespace.model),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -296,9 +285,17 @@ def main() -> None:
|
|||
os.environ.get(FIXTURE_DIR_ENV),
|
||||
Path(__file__).with_name(".fixtures"),
|
||||
)
|
||||
spec: Final = ProviderSpec(model=args.model, upstream_base=_mistral_upstream_base(), api_key=api_key)
|
||||
spec: Final = ProviderSpec(upstream_base=_mistral_upstream_base())
|
||||
use_litellm_rust(False, ocr=None, aocr=None)
|
||||
_generate_examples(spec, root, args.examples, args.concurrency, cast(Callable[..., object], litellm.ocr))
|
||||
_generate_examples(
|
||||
spec,
|
||||
root,
|
||||
args.model,
|
||||
api_key,
|
||||
args.examples,
|
||||
args.concurrency,
|
||||
cast(Callable[..., object], litellm.ocr),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -80,24 +80,18 @@ def test_recorded_ocr_sdk_parity(
|
|||
case_file: Final = tmp_path / f"{route.value}-ocr-parity-case.json"
|
||||
case_file.write_text(ocr_fixture.model_dump_json(indent=2, exclude_unset=True), encoding="utf-8")
|
||||
response: Final = ocr_fixture.provider_response
|
||||
response_body: Final = response.body_bytes()
|
||||
response_headers: Final = tuple((header.name, header.value) for header in response.headers)
|
||||
python_worker, rust_worker = sdk_workers
|
||||
python: Final = run_execution(
|
||||
python_worker,
|
||||
case_file,
|
||||
route.value,
|
||||
response.status_code,
|
||||
response_headers,
|
||||
response_body,
|
||||
response,
|
||||
)
|
||||
rust: Final = run_execution(
|
||||
rust_worker,
|
||||
case_file,
|
||||
route.value,
|
||||
response.status_code,
|
||||
response_headers,
|
||||
response_body,
|
||||
response,
|
||||
)
|
||||
|
||||
assert_parity(python, rust, PYTHON_HTTP_SENTINEL)
|
||||
|
|
|
|||
|
|
@ -4,12 +4,12 @@ import queue
|
|||
import threading
|
||||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from typing import Final
|
||||
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
from tests.test_litellm._recorded_http import RecordedHttpResponse, RecordedHttpStreamResponse, RecordedResponse
|
||||
from tests.test_litellm.parity.models import CapturedRequest
|
||||
|
||||
JSON_VALUE: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
|
||||
|
|
@ -31,15 +31,15 @@ class ReplayServer(ThreadingHTTPServer):
|
|||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(("127.0.0.1", 0), _ReplayHandler)
|
||||
self.responses: queue.Queue[ReplayResponse] = queue.Queue()
|
||||
self.responses: queue.Queue[RecordedResponse] = queue.Queue()
|
||||
self.requests: queue.Queue[CapturedRequest] = queue.Queue()
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
return f"http://127.0.0.1:{self.server_address[1]}"
|
||||
|
||||
def enqueue_response(self, status_code: int, headers: tuple[tuple[str, str], ...], body: bytes) -> None:
|
||||
self.responses.put(ReplayResponse(status_code=status_code, headers=headers, body=body))
|
||||
def enqueue_response(self, response: RecordedResponse) -> None:
|
||||
self.responses.put(response)
|
||||
|
||||
def take_request(self) -> CapturedRequest:
|
||||
request_count: Final = self.requests.qsize()
|
||||
|
|
@ -54,13 +54,6 @@ class ReplayServer(ThreadingHTTPServer):
|
|||
self.requests.get_nowait()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReplayResponse:
|
||||
status_code: int
|
||||
headers: tuple[tuple[str, str], ...]
|
||||
body: bytes
|
||||
|
||||
|
||||
class _ReplayHandler(BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
|
||||
|
|
@ -91,12 +84,26 @@ class _ReplayHandler(BaseHTTPRequestHandler):
|
|||
self.send_error(500, "no replay response queued")
|
||||
return
|
||||
self.send_response_only(response.status_code)
|
||||
for name, value in response.headers:
|
||||
if name.lower() not in EXCLUDED_RESPONSE_HEADERS:
|
||||
self.send_header(name, value)
|
||||
self.send_header("content-length", str(len(response.body)))
|
||||
for header in response.headers:
|
||||
if header.name.lower() not in EXCLUDED_RESPONSE_HEADERS:
|
||||
self.send_header(header.name, header.value)
|
||||
if isinstance(response, RecordedHttpResponse):
|
||||
response_body: Final = response.body_bytes()
|
||||
self.send_header("content-length", str(len(response_body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(response_body)
|
||||
return
|
||||
assert isinstance(response, RecordedHttpStreamResponse)
|
||||
self.send_header("transfer-encoding", "chunked")
|
||||
self.end_headers()
|
||||
self.wfile.write(response.body)
|
||||
for chunk in response.chunks:
|
||||
data = chunk.data_bytes()
|
||||
self.wfile.write(f"{len(data):X}\r\n".encode("ascii"))
|
||||
self.wfile.write(data)
|
||||
self.wfile.write(b"\r\n")
|
||||
self.wfile.flush()
|
||||
self.wfile.write(b"0\r\n\r\n")
|
||||
self.wfile.flush()
|
||||
|
||||
def log_message(self, format: str, *args: object) -> None:
|
||||
return
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from typing import Final, TextIO, cast
|
|||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from tests.test_litellm._recorded_http import RecordedResponse
|
||||
from tests.test_litellm.parity.models import (
|
||||
Execution,
|
||||
SDKCommand,
|
||||
|
|
@ -69,14 +70,12 @@ class PythonScriptWorker:
|
|||
self,
|
||||
case_file: Path,
|
||||
route: str,
|
||||
status_code: int,
|
||||
headers: tuple[tuple[str, str], ...],
|
||||
body: bytes,
|
||||
response: RecordedResponse,
|
||||
) -> Execution:
|
||||
stdin: Final = self.process.stdin
|
||||
if stdin is None or self.process.poll() is not None:
|
||||
raise AssertionError(f"{self.mode} OCR worker exited before processing {case_file}")
|
||||
self.provider.enqueue_response(status_code, headers, body)
|
||||
self.provider.enqueue_response(response)
|
||||
command: Final = SDKCommand(case_file=str(case_file), route=route)
|
||||
try:
|
||||
stdin.write(f"{command.model_dump_json()}\n")
|
||||
|
|
@ -153,8 +152,6 @@ def run_execution(
|
|||
worker: PythonScriptWorker,
|
||||
case_file: Path,
|
||||
route: str,
|
||||
status_code: int,
|
||||
headers: tuple[tuple[str, str], ...],
|
||||
body: bytes,
|
||||
response: RecordedResponse,
|
||||
) -> Execution:
|
||||
return worker.execute(case_file, route, status_code, headers, body)
|
||||
return worker.execute(case_file, route, response)
|
||||
|
|
|
|||
|
|
@ -5,12 +5,52 @@ from collections.abc import Generator
|
|||
from contextlib import contextmanager
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Final, cast
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from hypothesis import strategies as st
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from tests.test_litellm._fixture_recorder import ProviderSpec, record_cases
|
||||
from tests.test_litellm.ocr.fixture_models import MistralImageUrlDocument, MistralOcrSdkInput
|
||||
from tests.test_litellm._fixture_recorder import (
|
||||
FIXTURE_SCHEMA_VERSION,
|
||||
ProviderSpec,
|
||||
fixture_cache_key,
|
||||
generate_case_inputs,
|
||||
record_case,
|
||||
record_cases,
|
||||
recorded_fixtures,
|
||||
)
|
||||
from tests.test_litellm._json_fs_cache import JsonFileCache
|
||||
from tests.test_litellm._recorded_http import (
|
||||
HttpHeader,
|
||||
RecordedHttpStreamResponse,
|
||||
RecordedResponse,
|
||||
RecordedStreamChunk,
|
||||
)
|
||||
from tests.test_litellm.parity.replay import replay_server
|
||||
|
||||
_SSE_CHUNKS: Final = (
|
||||
b'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n',
|
||||
b'data: {"choices":[{"delta":{"content":" world"}}]}\n\n',
|
||||
b"data: [DONE]\n\n",
|
||||
)
|
||||
|
||||
|
||||
class _FixtureInput(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
identifier: str
|
||||
|
||||
def canonical_input(self) -> dict[str, object]:
|
||||
return {"identifier": self.identifier}
|
||||
|
||||
|
||||
class _ParityCase(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
litellm_input: _FixtureInput
|
||||
provider_response: RecordedResponse
|
||||
|
||||
|
||||
class _ControlledUpstream(ThreadingHTTPServer):
|
||||
|
|
@ -50,6 +90,21 @@ class _ControlledUpstreamHandler(BaseHTTPRequestHandler):
|
|||
assert isinstance(upstream, _ControlledUpstream)
|
||||
length: Final = int(self.headers.get("content-length") or "0")
|
||||
self.rfile.read(length)
|
||||
if self.path == "/v1/chat/completions":
|
||||
with upstream.lock:
|
||||
upstream.request_count += 1
|
||||
self.send_response(200)
|
||||
self.send_header("content-type", "text/event-stream")
|
||||
self.send_header("transfer-encoding", "chunked")
|
||||
self.end_headers()
|
||||
for chunk in _SSE_CHUNKS:
|
||||
self.wfile.write(f"{len(chunk):X}\r\n".encode("ascii"))
|
||||
self.wfile.write(chunk)
|
||||
self.wfile.write(b"\r\n")
|
||||
self.wfile.flush()
|
||||
self.wfile.write(b"0\r\n\r\n")
|
||||
self.wfile.flush()
|
||||
return
|
||||
upstream.start_request()
|
||||
try:
|
||||
body: Final = b"{}"
|
||||
|
|
@ -78,27 +133,86 @@ def _controlled_upstream() -> Generator[_ControlledUpstream]:
|
|||
thread.join(timeout=5)
|
||||
|
||||
|
||||
def _case(identifier: str) -> MistralOcrSdkInput:
|
||||
return MistralOcrSdkInput.model_validate(
|
||||
{
|
||||
"model": "mistral/mistral-ocr-latest",
|
||||
"document": MistralImageUrlDocument(type="image_url", image_url="https://example.com/image.png"),
|
||||
"id": identifier,
|
||||
}
|
||||
)
|
||||
def _case(identifier: str) -> _FixtureInput:
|
||||
return _FixtureInput(identifier=identifier)
|
||||
|
||||
|
||||
def _sdk_call(**kwargs: object) -> object:
|
||||
api_base: Final = cast(str, kwargs["api_base"])
|
||||
def _sdk_call(api_base: str, case_input: _FixtureInput) -> object:
|
||||
return httpx.post(f"{api_base}/v1/ocr", content=b"{}", timeout=5)
|
||||
|
||||
|
||||
def _stream_sdk_call(api_base: str, case_input: _FixtureInput) -> object:
|
||||
return httpx.post(f"{api_base}/v1/chat/completions", content=b"{}", timeout=5)
|
||||
|
||||
|
||||
def test_generate_case_inputs_is_deterministic() -> None:
|
||||
strategy: Final = st.builds(_FixtureInput, identifier=st.integers().map(str))
|
||||
|
||||
assert generate_case_inputs(strategy, examples=4) == generate_case_inputs(strategy, examples=4)
|
||||
|
||||
|
||||
def test_record_cases_deduplicates_and_limits_concurrency(tmp_path: Path) -> None:
|
||||
case_inputs: Final = (_case("one"), _case("two"), _case("one"), _case("three"))
|
||||
with _controlled_upstream() as upstream:
|
||||
spec: Final = ProviderSpec(model="mistral/mistral-ocr-latest", upstream_base=upstream.url, api_key="test-key")
|
||||
results: Final = record_cases(spec, tmp_path, case_inputs, _sdk_call, max_concurrency=2)
|
||||
spec: Final = ProviderSpec(upstream_base=upstream.url)
|
||||
results: Final = record_cases(spec, tmp_path, case_inputs, _sdk_call, _ParityCase, max_concurrency=2)
|
||||
|
||||
assert len(results) == 3
|
||||
assert upstream.request_count == 3
|
||||
assert upstream.max_active_requests == 2
|
||||
assert len(recorded_fixtures(tmp_path, _ParityCase)) == 3
|
||||
for fixture_path in tmp_path.glob("*.json"):
|
||||
contents = fixture_path.read_text(encoding="utf-8")
|
||||
assert f'"schema_version": {FIXTURE_SCHEMA_VERSION}' in contents
|
||||
assert '"recorded_at":' in contents
|
||||
|
||||
|
||||
def test_record_case_rejects_stale_fixture_before_provider_call(tmp_path: Path) -> None:
|
||||
case_input: Final = _case("stale")
|
||||
cache: Final = JsonFileCache(tmp_path)
|
||||
fixture_path: Final = cache.put(fixture_cache_key(case_input), {"schema_version": 0})
|
||||
|
||||
with pytest.raises(ValueError, match=f"{fixture_path} has schema_version 0, expected {FIXTURE_SCHEMA_VERSION}"):
|
||||
record_case(
|
||||
ProviderSpec(upstream_base="http://127.0.0.1:1"),
|
||||
tmp_path,
|
||||
case_input,
|
||||
_sdk_call,
|
||||
_ParityCase,
|
||||
)
|
||||
|
||||
|
||||
def test_streaming_response_records_and_replays_chunks(tmp_path: Path) -> None:
|
||||
with _controlled_upstream() as upstream:
|
||||
result: Final = record_case(
|
||||
ProviderSpec(upstream_base=upstream.url),
|
||||
tmp_path,
|
||||
_case("stream"),
|
||||
_stream_sdk_call,
|
||||
_ParityCase,
|
||||
)
|
||||
|
||||
response: Final = result.case.provider_response
|
||||
assert isinstance(response, RecordedHttpStreamResponse)
|
||||
assert tuple(chunk.data_bytes() for chunk in response.chunks) == _SSE_CHUNKS
|
||||
|
||||
with replay_server() as provider:
|
||||
provider.enqueue_response(response)
|
||||
with httpx.stream("POST", f"{provider.url}/v1/chat/completions", json={}) as replayed:
|
||||
replayed_chunks: Final = tuple(replayed.iter_raw())
|
||||
provider.take_request()
|
||||
|
||||
assert replayed_chunks == _SSE_CHUNKS
|
||||
|
||||
|
||||
def test_stream_response_model_rejects_buffered_body() -> None:
|
||||
with pytest.raises(ValueError):
|
||||
RecordedHttpStreamResponse.model_validate(
|
||||
{
|
||||
"kind": "http_stream",
|
||||
"status_code": 200,
|
||||
"headers": [HttpHeader(name="content-type", value="text/event-stream")],
|
||||
"chunks": [RecordedStreamChunk.from_bytes(b"data: [DONE]\n\n")],
|
||||
"body_b64": "",
|
||||
}
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue