mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-17 23:51:30 +00:00
test(ocr): add recorded fixture parity harness
This commit is contained in:
parent
362fd08dc6
commit
0ff81cfb67
16 changed files with 990 additions and 243 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -1,6 +1,7 @@
|
|||
.python-version
|
||||
.venv
|
||||
tests/e2e/.fixtures/
|
||||
tests/test_litellm/ocr/.fixtures/
|
||||
.venv-typecheck
|
||||
.venv_policy_test
|
||||
.venv-mutmut
|
||||
|
|
|
|||
|
|
@ -174,6 +174,7 @@ litellm-proxy = "litellm.proxy.client.cli:cli"
|
|||
[dependency-groups]
|
||||
dev = [
|
||||
"diff-cover==9.7.2",
|
||||
"hypothesis==6.165.10",
|
||||
"basedpyright==1.39.7",
|
||||
"keyring==25.7.0",
|
||||
"pytest==9.0.3",
|
||||
|
|
|
|||
43
tests/test_litellm/_json_fs_cache.py
Normal file
43
tests/test_litellm/_json_fs_cache.py
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
JSON_OBJECT: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
def canonical_json(value: Mapping[str, object]) -> str:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class JsonFileCache:
|
||||
root: Path
|
||||
|
||||
def path_for(self, key: Mapping[str, object]) -> Path:
|
||||
digest: Final = hashlib.sha256(canonical_json(key).encode("utf-8")).hexdigest()
|
||||
return self.root / f"{digest}.json"
|
||||
|
||||
def get(self, key: Mapping[str, object]) -> dict[str, object] | None:
|
||||
path: Final = self.path_for(key)
|
||||
if not path.is_file():
|
||||
return None
|
||||
return JSON_OBJECT.validate_json(path.read_text(encoding="utf-8"))
|
||||
|
||||
def put(self, key: Mapping[str, object], value: Mapping[str, object]) -> Path:
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
path: Final = self.path_for(key)
|
||||
path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
return path
|
||||
|
||||
def values(self) -> tuple[dict[str, object], ...]:
|
||||
if not self.root.is_dir():
|
||||
return ()
|
||||
paths: Final = tuple(sorted(self.root.rglob("*.json")))
|
||||
return tuple(JSON_OBJECT.validate_json(path.read_text(encoding="utf-8")) for path in paths)
|
||||
51
tests/test_litellm/ocr/conftest.py
Normal file
51
tests/test_litellm/ocr/conftest.py
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.test_litellm._json_fs_cache import JsonFileCache, canonical_json
|
||||
from tests.test_litellm.ocr.fixture_models import OcrFixture
|
||||
|
||||
FIXTURE_DIR_ENV: Final = "LITELLM_OCR_FIXTURE_DIR"
|
||||
pytest_plugins: Final = ("tests.test_litellm.parity.pytest_plugin",)
|
||||
|
||||
|
||||
def _fixture_directory() -> Path:
|
||||
configured: Final = os.environ.get(FIXTURE_DIR_ENV)
|
||||
return Path(configured).expanduser() if configured else Path(__file__).with_name(".fixtures")
|
||||
|
||||
|
||||
def _recorded_fixtures() -> tuple[OcrFixture, ...]:
|
||||
raw_fixtures: Final = JsonFileCache(_fixture_directory()).values()
|
||||
return tuple(OcrFixture.model_validate(raw_fixture) for raw_fixture in raw_fixtures)
|
||||
|
||||
|
||||
def _fixture_id(fixture: OcrFixture) -> str:
|
||||
raw_model: Final = fixture.request.sdk_kwargs.get("model")
|
||||
model: Final = raw_model if isinstance(raw_model, str) else "unknown-model"
|
||||
request_json: Final = canonical_json(fixture.request.provider_request.model_dump(mode="json"))
|
||||
digest: Final = hashlib.sha256(request_json.encode("utf-8")).hexdigest()[:8]
|
||||
return f"{fixture.request.provider}-{model.rsplit('/', 1)[-1]}-{digest}"
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
|
||||
if "ocr_fixture" not in metafunc.fixturenames:
|
||||
return
|
||||
fixtures: Final = _recorded_fixtures()
|
||||
if not fixtures:
|
||||
metafunc.parametrize(
|
||||
"ocr_fixture",
|
||||
(
|
||||
pytest.param(
|
||||
None,
|
||||
marks=pytest.mark.skip(reason=f"no recorded OCR fixtures in {_fixture_directory()}"),
|
||||
id="no-recorded-fixtures",
|
||||
),
|
||||
),
|
||||
)
|
||||
return
|
||||
metafunc.parametrize("ocr_fixture", fixtures, ids=tuple(_fixture_id(fixture) for fixture in fixtures))
|
||||
34
tests/test_litellm/ocr/fixture_models.py
Normal file
34
tests/test_litellm/ocr/fixture_models.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
|
||||
class ProviderWireRequest(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
method: str
|
||||
path: str
|
||||
body: dict[str, object]
|
||||
|
||||
|
||||
class OcrFixtureRequest(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
provider: str
|
||||
sdk_kwargs: dict[str, object]
|
||||
provider_request: ProviderWireRequest
|
||||
|
||||
|
||||
class OcrFixtureResponse(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
status_code: int
|
||||
headers: dict[str, str]
|
||||
body: dict[str, object]
|
||||
|
||||
|
||||
class OcrFixture(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
request: OcrFixtureRequest
|
||||
response: OcrFixtureResponse
|
||||
348
tests/test_litellm/ocr/generate_fixtures.py
Normal file
348
tests/test_litellm/ocr/generate_fixtures.py
Normal file
|
|
@ -0,0 +1,348 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import threading
|
||||
from collections.abc import Callable, Generator
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Final, cast
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
from dotenv import load_dotenv
|
||||
from hypothesis import given, settings
|
||||
from hypothesis import strategies as st
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from tests.test_litellm._json_fs_cache import JsonFileCache
|
||||
from tests.test_litellm.ocr.fixture_models import (
|
||||
OcrFixture,
|
||||
OcrFixtureRequest,
|
||||
OcrFixtureResponse,
|
||||
ProviderWireRequest,
|
||||
)
|
||||
|
||||
FIXTURE_DIR_ENV: Final = "LITELLM_OCR_FIXTURE_DIR"
|
||||
JSON_OBJECT: Final = TypeAdapter(dict[str, object])
|
||||
PROVIDER_NAMES: Final = ("mistral", "azure_ai", "vertex_ai")
|
||||
LOGGER: Final = logging.getLogger(__name__)
|
||||
_TEXT: Final = st.from_regex(r"[A-Za-z0-9 ]{1,24}", fullmatch=True)
|
||||
_OPTIONS: Final = st.fixed_dictionaries(
|
||||
{
|
||||
"pages": st.just([0]),
|
||||
"include_image_base64": st.booleans(),
|
||||
"image_limit": st.integers(min_value=1, max_value=4),
|
||||
"image_min_size": st.integers(min_value=0, max_value=64),
|
||||
"extract_header": st.booleans(),
|
||||
"extract_footer": st.booleans(),
|
||||
"table_format": st.sampled_from(("markdown", "html")),
|
||||
"confidence_scores_granularity": st.sampled_from(("word", "page")),
|
||||
"include_blocks": st.booleans(),
|
||||
"id": _TEXT,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProviderSpec:
|
||||
name: str
|
||||
model: str
|
||||
upstream_base: str
|
||||
api_key: str | None
|
||||
upstream_model: str | None = None
|
||||
vertex_project: str | None = None
|
||||
vertex_location: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RecorderResult:
|
||||
fixture: OcrFixture
|
||||
cache_hit: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GeneratorArgs:
|
||||
providers: tuple[str, ...]
|
||||
examples: int
|
||||
fixture_dir: Path | None
|
||||
|
||||
|
||||
class _RecordingProvider(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
spec: ProviderSpec,
|
||||
sdk_kwargs: dict[str, object],
|
||||
cache: JsonFileCache,
|
||||
) -> None:
|
||||
super().__init__(("127.0.0.1", 0), _RecordingHandler)
|
||||
self.spec: Final = spec
|
||||
self.sdk_kwargs: Final = sdk_kwargs
|
||||
self.cache: Final = cache
|
||||
self.results: queue.Queue[RecorderResult] = queue.Queue()
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
return f"http://127.0.0.1:{self.server_address[1]}"
|
||||
|
||||
def take_result(self) -> RecorderResult:
|
||||
try:
|
||||
return self.results.get(timeout=5)
|
||||
except queue.Empty as error:
|
||||
raise RuntimeError("successful OCR call did not produce a recorder result") from error
|
||||
|
||||
|
||||
class _RecordingHandler(BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
|
||||
def do_POST(self) -> None:
|
||||
provider: Final = self.server
|
||||
assert isinstance(provider, _RecordingProvider)
|
||||
length: Final = int(self.headers.get("content-length") or "0")
|
||||
body: Final = JSON_OBJECT.validate_json(self.rfile.read(length))
|
||||
fixture_request: Final = OcrFixtureRequest(
|
||||
provider=provider.spec.name,
|
||||
sdk_kwargs=provider.sdk_kwargs,
|
||||
provider_request=ProviderWireRequest(method=self.command, path=self.path, body=body),
|
||||
)
|
||||
cache_key: Final = _fixture_cache_key(provider.spec.name, fixture_request.provider_request)
|
||||
cached_value: Final = provider.cache.get(cache_key)
|
||||
if cached_value is not None:
|
||||
cached_fixture: Final = OcrFixture.model_validate(cached_value)
|
||||
provider.results.put(RecorderResult(fixture=cached_fixture, cache_hit=True))
|
||||
self._send_fixture_response(cached_fixture.response)
|
||||
return
|
||||
|
||||
upstream_url: Final = f"{provider.spec.upstream_base.rstrip('/')}{self.path}"
|
||||
forwarded_headers: Final = {
|
||||
name: value
|
||||
for name, value in self.headers.items()
|
||||
if name.lower() not in {"host", "content-length", "accept-encoding", "x-parity-case"}
|
||||
}
|
||||
upstream_body: Final = (
|
||||
{**body, "model": provider.spec.upstream_model} if provider.spec.upstream_model is not None else body
|
||||
)
|
||||
try:
|
||||
upstream_response: Final = httpx.post(
|
||||
upstream_url,
|
||||
headers=forwarded_headers,
|
||||
content=json.dumps(upstream_body, separators=(",", ":")),
|
||||
timeout=120,
|
||||
)
|
||||
except httpx.HTTPError as error:
|
||||
error_body: Final = json.dumps({"error": str(error)}).encode()
|
||||
self._send_response(502, {"content-type": "application/json"}, error_body)
|
||||
return
|
||||
|
||||
raw_content_type: Final = cast(object, upstream_response.headers.get("content-type", "application/json"))
|
||||
content_type: Final = raw_content_type if isinstance(raw_content_type, str) else "application/json"
|
||||
response_headers: Final = {"content-type": content_type.split(";", 1)[0]}
|
||||
if not upstream_response.is_success:
|
||||
self._send_response(upstream_response.status_code, response_headers, upstream_response.content)
|
||||
return
|
||||
|
||||
upstream_response_body: Final = JSON_OBJECT.validate_json(upstream_response.content)
|
||||
fixture_response: Final = OcrFixtureResponse(
|
||||
status_code=upstream_response.status_code,
|
||||
headers=response_headers,
|
||||
body=upstream_response_body,
|
||||
)
|
||||
recorded_fixture: Final = OcrFixture(request=fixture_request, response=fixture_response)
|
||||
provider.results.put(RecorderResult(fixture=recorded_fixture, cache_hit=False))
|
||||
self._send_fixture_response(fixture_response)
|
||||
|
||||
def _send_fixture_response(self, response: OcrFixtureResponse) -> None:
|
||||
response_body: Final = json.dumps(response.body, separators=(",", ":")).encode()
|
||||
self._send_response(response.status_code, response.headers, response_body)
|
||||
|
||||
def _send_response(self, status_code: int, headers: dict[str, str], body: bytes) -> None:
|
||||
self.send_response(status_code)
|
||||
for name, value in headers.items():
|
||||
self.send_header(name, value)
|
||||
self.send_header("content-length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, format: str, *args: object) -> None:
|
||||
return
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _recording_provider(
|
||||
spec: ProviderSpec,
|
||||
sdk_kwargs: dict[str, object],
|
||||
cache: JsonFileCache,
|
||||
) -> Generator[_RecordingProvider]:
|
||||
server: Final = _RecordingProvider(spec=spec, sdk_kwargs=sdk_kwargs, cache=cache)
|
||||
thread: Final = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
yield server
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join(timeout=5)
|
||||
|
||||
|
||||
def _mistral_upstream_base() -> str:
|
||||
configured: Final = os.environ.get("MISTRAL_API_BASE", "https://api.mistral.ai").rstrip("/")
|
||||
return configured.removesuffix("/v1")
|
||||
|
||||
|
||||
def _fixture_cache_key(provider: str, request: ProviderWireRequest) -> dict[str, object]:
|
||||
return {"provider": provider, "request": request.model_dump(mode="json")}
|
||||
|
||||
|
||||
def _provider_specs(selected: tuple[str, ...]) -> tuple[ProviderSpec, ...]:
|
||||
specs: Final[dict[str, ProviderSpec | None]] = {
|
||||
"mistral": (
|
||||
ProviderSpec(
|
||||
name="mistral",
|
||||
model=os.environ.get("MISTRAL_OCR_MODEL", "mistral/mistral-ocr-latest"),
|
||||
upstream_base=_mistral_upstream_base(),
|
||||
api_key=os.environ.get("MISTRAL_API_KEY") or os.environ.get("LITELLM_API_KEY"),
|
||||
upstream_model=os.environ.get("MISTRAL_OCR_UPSTREAM_MODEL"),
|
||||
)
|
||||
if os.environ.get("MISTRAL_API_KEY") or os.environ.get("LITELLM_API_KEY")
|
||||
else None
|
||||
),
|
||||
"azure_ai": (
|
||||
ProviderSpec(
|
||||
name="azure_ai",
|
||||
model=os.environ.get("AZURE_AI_OCR_MODEL", "azure_ai/mistral-document-ai-2512"),
|
||||
upstream_base=os.environ["AZURE_AI_API_BASE"],
|
||||
api_key=os.environ["AZURE_AI_API_KEY"],
|
||||
)
|
||||
if os.environ.get("AZURE_AI_API_BASE") and os.environ.get("AZURE_AI_API_KEY")
|
||||
else None
|
||||
),
|
||||
"vertex_ai": _vertex_spec(),
|
||||
}
|
||||
missing: Final = tuple(name for name in selected if specs[name] is None)
|
||||
if missing:
|
||||
LOGGER.warning("Skipping providers without credentials: %s", ", ".join(missing))
|
||||
return tuple(spec for name in selected if (spec := specs[name]) is not None)
|
||||
|
||||
|
||||
def _vertex_spec() -> ProviderSpec | None:
|
||||
project: Final = os.environ.get("VERTEXAI_PROJECT")
|
||||
credentials_available: Final = bool(os.environ.get("VERTEX_AI_API_KEY") or os.environ.get("VERTEXAI_CREDENTIALS"))
|
||||
if project is None or not credentials_available:
|
||||
return None
|
||||
location: Final = os.environ.get("VERTEXAI_LOCATION", os.environ.get("VERTEX_LOCATION", "us-central1"))
|
||||
upstream_base: Final = os.environ.get("VERTEX_AI_API_BASE", f"https://{location}-aiplatform.googleapis.com")
|
||||
return ProviderSpec(
|
||||
name="vertex_ai",
|
||||
model=os.environ.get("VERTEX_AI_OCR_MODEL", "vertex_ai/mistral-ocr-2505"),
|
||||
upstream_base=upstream_base,
|
||||
api_key=os.environ.get("VERTEX_AI_API_KEY"),
|
||||
vertex_project=project,
|
||||
vertex_location=location,
|
||||
)
|
||||
|
||||
|
||||
def _image_data_uri(text: str, font_size: int) -> str:
|
||||
url: Final = f"https://dummyjson.com/image/800x300/ffffff/000000?text={quote(text)}&fontSize={font_size}"
|
||||
response: Final = httpx.get(url, timeout=30, follow_redirects=True)
|
||||
response.raise_for_status()
|
||||
raw_content_type: Final = cast(object, response.headers.get("content-type", "image/png"))
|
||||
content_type: Final = raw_content_type if isinstance(raw_content_type, str) else "image/png"
|
||||
media_type: Final = content_type.split(";", 1)[0]
|
||||
encoded: Final = base64.b64encode(response.content).decode("ascii")
|
||||
return f"data:{media_type};base64,{encoded}"
|
||||
|
||||
|
||||
def _sdk_kwargs(
|
||||
spec: ProviderSpec,
|
||||
image_data_uri: str,
|
||||
options: dict[str, object],
|
||||
) -> dict[str, object]:
|
||||
provider_kwargs: Final = (
|
||||
{"vertex_project": spec.vertex_project, "vertex_location": spec.vertex_location}
|
||||
if spec.name == "vertex_ai"
|
||||
else {}
|
||||
)
|
||||
return {
|
||||
"model": spec.model,
|
||||
"document": {"type": "image_url", "image_url": image_data_uri},
|
||||
**options,
|
||||
**provider_kwargs,
|
||||
}
|
||||
|
||||
|
||||
def _record_case(spec: ProviderSpec, root: Path, sdk_kwargs: dict[str, object]) -> RecorderResult:
|
||||
cache: Final = JsonFileCache(root / spec.name)
|
||||
with _recording_provider(spec=spec, sdk_kwargs=sdk_kwargs, cache=cache) as recorder:
|
||||
ocr_call: Final = cast(Callable[..., object], litellm.ocr)
|
||||
ocr_call(api_base=recorder.url, api_key=spec.api_key, **sdk_kwargs)
|
||||
result: Final = recorder.take_result()
|
||||
|
||||
if not result.cache_hit:
|
||||
cache.put(
|
||||
_fixture_cache_key(spec.name, result.fixture.request.provider_request),
|
||||
result.fixture.model_dump(mode="json"),
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _generate(specs: tuple[ProviderSpec, ...], root: Path, examples: int) -> None:
|
||||
@settings(max_examples=examples, deadline=None, derandomize=True)
|
||||
@given(text=_TEXT, font_size=st.integers(min_value=12, max_value=36), options=_OPTIONS)
|
||||
def generate_case(text: str, font_size: int, options: dict[str, object]) -> None:
|
||||
image_data_uri: Final = _image_data_uri(text, font_size)
|
||||
for spec in specs:
|
||||
_generate_provider_case(spec, root, image_data_uri, options)
|
||||
|
||||
generate_case()
|
||||
|
||||
|
||||
def _generate_provider_case(
|
||||
spec: ProviderSpec,
|
||||
root: Path,
|
||||
image_data_uri: str,
|
||||
options: dict[str, object],
|
||||
) -> None:
|
||||
sdk_kwargs: Final = _sdk_kwargs(spec, image_data_uri, options)
|
||||
result: Final = _record_case(spec, root, sdk_kwargs)
|
||||
state: Final = "cached" if result.cache_hit else "recorded"
|
||||
LOGGER.info("%s %s %s", state, spec.name, result.fixture.request.provider_request.path)
|
||||
|
||||
|
||||
def _parse_args() -> GeneratorArgs:
|
||||
parser: Final = argparse.ArgumentParser()
|
||||
parser.add_argument("--provider", action="append", choices=PROVIDER_NAMES)
|
||||
parser.add_argument("--examples", type=int, default=4)
|
||||
parser.add_argument("--fixture-dir", type=Path)
|
||||
namespace: Final = parser.parse_args()
|
||||
providers: Final = cast(list[str] | None, namespace.provider)
|
||||
return GeneratorArgs(
|
||||
providers=tuple(providers) if providers else PROVIDER_NAMES,
|
||||
examples=cast(int, namespace.examples),
|
||||
fixture_dir=cast(Path | None, namespace.fixture_dir),
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
load_dotenv()
|
||||
args: Final = _parse_args()
|
||||
root: Final = (
|
||||
args.fixture_dir or Path(os.environ.get(FIXTURE_DIR_ENV, Path(__file__).with_name(".fixtures"))).expanduser()
|
||||
)
|
||||
specs: Final = _provider_specs(args.providers)
|
||||
if not specs:
|
||||
raise SystemExit("No selected provider has the required credentials")
|
||||
_generate(specs, root, args.examples)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,46 +1,33 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import queue
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from collections.abc import Callable, Coroutine, Generator
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from collections.abc import Callable, Coroutine
|
||||
from enum import Enum
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Final, Protocol, cast
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
|
||||
MODEL: Final = "mistral/mistral-ocr-latest"
|
||||
from tests.test_litellm.ocr.fixture_models import OcrFixture, OcrFixtureResponse, ProviderWireRequest
|
||||
from tests.test_litellm.parity.compare import assert_parity
|
||||
from tests.test_litellm.parity.models import (
|
||||
CapturedRequest,
|
||||
ExceptionReport,
|
||||
NativeEvidence,
|
||||
ParityTrace,
|
||||
ReplayResponse,
|
||||
SDKOutput,
|
||||
SDKReport,
|
||||
)
|
||||
from tests.test_litellm.parity.replay import replay_json_response
|
||||
from tests.test_litellm.parity.runner import PythonScriptRunner, run_execution
|
||||
|
||||
API_KEY: Final = "test-key"
|
||||
PYTHON_HTTP_SENTINEL: Final = "python-ocr-parity-fallback"
|
||||
JSON_OBJECT: Final = TypeAdapter(dict[str, object])
|
||||
GIL_STATS: Final = TypeAdapter(dict[str, int])
|
||||
|
||||
STATIC_OCR_RESPONSE: Final[dict[str, object]] = {
|
||||
"pages": [
|
||||
{
|
||||
"index": 0,
|
||||
"markdown": "# Static OCR\n\nProvider fixture text.",
|
||||
"dimensions": {"dpi": 200, "height": 1000, "width": 800},
|
||||
"blocks": [{"type": "title", "content": "Static OCR"}],
|
||||
"header": "Fixture header",
|
||||
"footer": "Fixture footer",
|
||||
}
|
||||
],
|
||||
"model": "mistral-ocr-static",
|
||||
"document_annotation": {"language": "en"},
|
||||
"usage_info": {"pages_processed": 1, "credits": 0.25, "provider_extra": "preserved"},
|
||||
"top_level_extra": "dropped by both implementations",
|
||||
}
|
||||
|
||||
|
||||
class SDKRoute(str, Enum):
|
||||
OCR = "ocr"
|
||||
|
|
@ -54,50 +41,6 @@ class SDKInput(BaseModel):
|
|||
kwargs: dict[str, object]
|
||||
|
||||
|
||||
class ExceptionReport(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
class_name: str
|
||||
status_code: int | None
|
||||
message: str
|
||||
|
||||
|
||||
class SDKReport(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
rust_enabled: bool
|
||||
native_callable_loaded: bool
|
||||
native_handled_case: bool
|
||||
response_type: str | None
|
||||
response_json: dict[str, object] | None
|
||||
exception_json: ExceptionReport | None
|
||||
|
||||
|
||||
class ProviderRequest(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
method: str
|
||||
path: str
|
||||
authorization: str | None
|
||||
content_type: str | None
|
||||
parity_case: str | None
|
||||
body: dict[str, object]
|
||||
user_agent: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Execution:
|
||||
report: SDKReport
|
||||
request: ProviderRequest
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CallOutcome:
|
||||
response_type: str | None
|
||||
response_json: dict[str, object] | None
|
||||
exception_json: ExceptionReport | None
|
||||
|
||||
|
||||
class _NativeBridge(Protocol):
|
||||
ocr: object
|
||||
aocr: object
|
||||
|
|
@ -105,76 +48,6 @@ class _NativeBridge(Protocol):
|
|||
def gil_stats(self) -> object: ...
|
||||
|
||||
|
||||
class _StaticOcrProvider(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(("127.0.0.1", 0), _StaticOcrHandler)
|
||||
self.requests: queue.Queue[ProviderRequest] = queue.Queue()
|
||||
self.response_body: bytes = json.dumps(STATIC_OCR_RESPONSE, sort_keys=True, separators=(",", ":")).encode()
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
return f"http://127.0.0.1:{self.server_address[1]}"
|
||||
|
||||
def take_single_request(self) -> ProviderRequest:
|
||||
try:
|
||||
first: Final = self.requests.get(timeout=5)
|
||||
except queue.Empty as error:
|
||||
raise AssertionError("expected one provider request, received none") from error
|
||||
try:
|
||||
extra: Final = self.requests.get_nowait()
|
||||
except queue.Empty:
|
||||
return first
|
||||
raise AssertionError(f"expected one provider request, received an extra request: {extra.model_dump_json()}")
|
||||
|
||||
|
||||
class _StaticOcrHandler(BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
|
||||
def do_POST(self) -> None:
|
||||
provider: Final = self.server
|
||||
assert isinstance(provider, _StaticOcrProvider)
|
||||
length: Final = int(self.headers.get("content-length") or "0")
|
||||
body: Final = JSON_OBJECT.validate_json(self.rfile.read(length))
|
||||
content_type_header: Final = self.headers.get("content-type")
|
||||
content_type: Final = content_type_header.split(";", 1)[0].lower() if content_type_header else None
|
||||
provider.requests.put(
|
||||
ProviderRequest(
|
||||
method=self.command,
|
||||
path=self.path,
|
||||
authorization=self.headers.get("authorization"),
|
||||
content_type=content_type,
|
||||
parity_case=self.headers.get("x-parity-case"),
|
||||
body=body,
|
||||
user_agent=self.headers.get("user-agent"),
|
||||
)
|
||||
)
|
||||
status: Final = 200 if self.path == "/v1/ocr" else 404
|
||||
response_body: Final = provider.response_body if status == 200 else b'{"error":"unexpected path"}'
|
||||
self.send_response(status)
|
||||
self.send_header("content-type", "application/json")
|
||||
self.send_header("content-length", str(len(response_body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(response_body)
|
||||
|
||||
def log_message(self, format: str, *args: object) -> None:
|
||||
return
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _static_provider() -> Generator[_StaticOcrProvider]:
|
||||
server: Final = _StaticOcrProvider()
|
||||
thread: Final = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
yield server
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join(timeout=5)
|
||||
|
||||
|
||||
def _qualified_name(value: object) -> str:
|
||||
value_type: Final = type(value)
|
||||
return f"{value_type.__module__}.{value_type.__qualname__}"
|
||||
|
|
@ -191,19 +64,15 @@ def _gil_release_count(native_bridge: _NativeBridge) -> int:
|
|||
return stats["releases"]
|
||||
|
||||
|
||||
def _response_outcome(response: BaseModel) -> CallOutcome:
|
||||
response_json: Final = JSON_OBJECT.validate_python(response.model_dump(mode="json"))
|
||||
return CallOutcome(
|
||||
def _response_trace(response: BaseModel) -> ParityTrace:
|
||||
output: Final = SDKOutput(
|
||||
response_type=_qualified_name(response),
|
||||
response_json=response_json,
|
||||
exception_json=None,
|
||||
response_json=response.model_dump(mode="json"),
|
||||
)
|
||||
return ParityTrace(outputs=(output,), exception=None)
|
||||
|
||||
|
||||
def _capture_sdk_call(
|
||||
sdk_input: SDKInput,
|
||||
mock_url: str,
|
||||
) -> CallOutcome:
|
||||
def _capture_sdk_call(sdk_input: SDKInput, mock_url: str) -> ParityTrace:
|
||||
import litellm
|
||||
from litellm.llms.base_llm.ocr.transformation import OCRResponse
|
||||
|
||||
|
|
@ -217,26 +86,21 @@ def _capture_sdk_call(
|
|||
try:
|
||||
if sdk_input.route is SDKRoute.OCR:
|
||||
sync_route: Final = cast(Callable[..., OCRResponse], litellm.ocr)
|
||||
return _response_outcome(sync_route(**call_kwargs))
|
||||
|
||||
return _response_trace(sync_route(**call_kwargs))
|
||||
async_route: Final = cast(Callable[..., Coroutine[object, object, OCRResponse]], litellm.aocr)
|
||||
return _response_outcome(asyncio.run(async_route(**call_kwargs)))
|
||||
return _response_trace(asyncio.run(async_route(**call_kwargs)))
|
||||
except Exception as error:
|
||||
return CallOutcome(
|
||||
response_type=None,
|
||||
response_json=None,
|
||||
exception_json=_exception_report(error),
|
||||
)
|
||||
return ParityTrace(outputs=(), exception=_exception_report(error))
|
||||
|
||||
|
||||
def _native_sdk_report(outcome: CallOutcome, native_handled_case: bool) -> SDKReport:
|
||||
def _native_sdk_report(trace: ParityTrace, native_handled_case: bool) -> SDKReport:
|
||||
return SDKReport(
|
||||
rust_enabled=True,
|
||||
native_callable_loaded=True,
|
||||
native_handled_case=native_handled_case,
|
||||
response_type=outcome.response_type,
|
||||
response_json=outcome.response_json,
|
||||
exception_json=outcome.exception_json,
|
||||
trace=trace,
|
||||
native=NativeEvidence(
|
||||
rust_enabled=True,
|
||||
native_callable_loaded=True,
|
||||
native_handled_case=native_handled_case,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -246,14 +110,13 @@ def _execute_sdk_case(sdk_input: SDKInput, mock_url: str) -> SDKReport:
|
|||
|
||||
rust_enabled: Final = rust_ocr_enabled()
|
||||
if not rust_enabled:
|
||||
outcome: Final = _capture_sdk_call(sdk_input, mock_url)
|
||||
return SDKReport(
|
||||
rust_enabled=False,
|
||||
native_callable_loaded=False,
|
||||
native_handled_case=False,
|
||||
response_type=outcome.response_type,
|
||||
response_json=outcome.response_json,
|
||||
exception_json=outcome.exception_json,
|
||||
trace=_capture_sdk_call(sdk_input, mock_url),
|
||||
native=NativeEvidence(
|
||||
rust_enabled=False,
|
||||
native_callable_loaded=False,
|
||||
native_handled_case=False,
|
||||
),
|
||||
)
|
||||
|
||||
raw_native_bridge: Final = get_native_bridge()
|
||||
|
|
@ -267,92 +130,57 @@ def _execute_sdk_case(sdk_input: SDKInput, mock_url: str) -> SDKReport:
|
|||
raise RuntimeError(f"native {sdk_input.route.value} callable is unavailable")
|
||||
|
||||
if sdk_input.route is SDKRoute.AOCR:
|
||||
async_outcome: Final = _capture_sdk_call(sdk_input, mock_url)
|
||||
return _native_sdk_report(async_outcome, native_handled_case=True)
|
||||
async_trace: Final = _capture_sdk_call(sdk_input, mock_url)
|
||||
return _native_sdk_report(async_trace, native_handled_case=bool(async_trace.outputs))
|
||||
|
||||
before_gil_releases: Final = _gil_release_count(native_bridge)
|
||||
sync_outcome: Final = _capture_sdk_call(sdk_input, mock_url)
|
||||
trace: Final = _capture_sdk_call(sdk_input, mock_url)
|
||||
after_gil_releases: Final = _gil_release_count(native_bridge)
|
||||
native_handled_case: Final = after_gil_releases == before_gil_releases + 1
|
||||
return _native_sdk_report(sync_outcome, native_handled_case)
|
||||
return _native_sdk_report(trace, native_handled_case=after_gil_releases == before_gil_releases + 1)
|
||||
|
||||
|
||||
def _run_execution(
|
||||
case_file: Path,
|
||||
report_file: Path,
|
||||
provider: _StaticOcrProvider,
|
||||
rust_enabled: bool,
|
||||
) -> Execution:
|
||||
env: Final = {
|
||||
**os.environ,
|
||||
"LITELLM_USE_RUST_OCR": "1" if rust_enabled else "0",
|
||||
"LITELLM_USER_AGENT": PYTHON_HTTP_SENTINEL,
|
||||
}
|
||||
completed: Final = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(Path(__file__).resolve()),
|
||||
str(case_file),
|
||||
provider.url,
|
||||
str(report_file),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
timeout=60,
|
||||
check=False,
|
||||
)
|
||||
assert completed.returncode == 0, (
|
||||
f"SDK subprocess failed with exit code {completed.returncode}\n"
|
||||
f"stdout:\n{completed.stdout}\n"
|
||||
f"stderr:\n{completed.stderr}"
|
||||
)
|
||||
report: Final = SDKReport.model_validate_json(report_file.read_text(encoding="utf-8"))
|
||||
return Execution(report=report, request=provider.take_single_request())
|
||||
def _replay_response(response: OcrFixtureResponse) -> ReplayResponse:
|
||||
return ReplayResponse(status_code=response.status_code, headers=response.headers, body=response.body)
|
||||
|
||||
|
||||
def _provider_wire_request(request: CapturedRequest) -> ProviderWireRequest:
|
||||
return ProviderWireRequest(method=request.method, path=request.path, body=request.body)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", tuple(SDKRoute), ids=tuple(route.value for route in SDKRoute))
|
||||
def test_static_mistral_ocr_sdk_parity(route: SDKRoute, tmp_path: Path) -> None:
|
||||
sdk_input: Final = SDKInput(
|
||||
route=route,
|
||||
kwargs={
|
||||
"model": MODEL,
|
||||
"document": {"type": "document_url", "document_url": "https://example.com/static.pdf"},
|
||||
"pages": [0, 1],
|
||||
"include_image_base64": True,
|
||||
"include_blocks": True,
|
||||
"table_format": "html",
|
||||
"id": "static-ocr-parity",
|
||||
},
|
||||
)
|
||||
case_file: Final = tmp_path / f"{route.value}-sdk-input.json"
|
||||
def test_recorded_ocr_sdk_parity(ocr_fixture: OcrFixture, route: SDKRoute, tmp_path: Path) -> None:
|
||||
sdk_input: Final = SDKInput(route=route, kwargs=ocr_fixture.request.sdk_kwargs)
|
||||
case_file: Final = tmp_path / f"{ocr_fixture.request.provider}-{route.value}-sdk-input.json"
|
||||
case_file.write_text(sdk_input.model_dump_json(indent=2), encoding="utf-8")
|
||||
with _static_provider() as python_mock, _static_provider() as rust_mock:
|
||||
python: Final = _run_execution(
|
||||
expected_request: Final = ocr_fixture.request.provider_request
|
||||
runner: Final = PythonScriptRunner(
|
||||
entrypoint=Path(__file__),
|
||||
rust_env_var="LITELLM_USE_RUST_OCR",
|
||||
python_user_agent=PYTHON_HTTP_SENTINEL,
|
||||
)
|
||||
|
||||
with (
|
||||
replay_json_response(expected_request.path, _replay_response(ocr_fixture.response)) as python_provider,
|
||||
replay_json_response(expected_request.path, _replay_response(ocr_fixture.response)) as rust_provider,
|
||||
):
|
||||
python: Final = run_execution(
|
||||
runner,
|
||||
case_file,
|
||||
tmp_path / f"{route.value}-python-report.json",
|
||||
python_mock,
|
||||
False,
|
||||
python_provider,
|
||||
rust_enabled=False,
|
||||
)
|
||||
rust: Final = _run_execution(
|
||||
rust: Final = run_execution(
|
||||
runner,
|
||||
case_file,
|
||||
tmp_path / f"{route.value}-rust-report.json",
|
||||
rust_mock,
|
||||
True,
|
||||
rust_provider,
|
||||
rust_enabled=True,
|
||||
)
|
||||
|
||||
assert python.report.rust_enabled is False
|
||||
assert rust.report.rust_enabled is True
|
||||
assert rust.report.native_callable_loaded is True
|
||||
assert rust.report.native_handled_case is True
|
||||
assert python.request.user_agent == PYTHON_HTTP_SENTINEL
|
||||
assert rust.request.user_agent != PYTHON_HTTP_SENTINEL
|
||||
assert rust.request.model_dump(exclude={"user_agent"}) == python.request.model_dump(exclude={"user_agent"})
|
||||
assert rust.report.response_type == python.report.response_type
|
||||
assert rust.report.response_json == python.report.response_json
|
||||
assert rust.report.exception_json == python.report.exception_json
|
||||
assert python.report.exception_json is None
|
||||
assert python.report.response_json is not None
|
||||
assert_parity(python, rust, PYTHON_HTTP_SENTINEL)
|
||||
assert tuple(_provider_wire_request(request) for request in python.requests) == (expected_request,)
|
||||
assert tuple(_provider_wire_request(request) for request in rust.requests) == (expected_request,)
|
||||
|
||||
|
||||
def _child_main() -> None:
|
||||
|
|
|
|||
3
tests/test_litellm/parity/__init__.py
Normal file
3
tests/test_litellm/parity/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
import pytest
|
||||
|
||||
pytest.register_assert_rewrite("tests.test_litellm.parity.compare")
|
||||
79
tests/test_litellm/parity/compare.py
Normal file
79
tests/test_litellm/parity/compare.py
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final, cast
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from tests.test_litellm.parity.models import CapturedRequest, Execution, ParityTrace
|
||||
|
||||
|
||||
def _stable_request(request: CapturedRequest) -> dict[str, object]:
|
||||
return request.model_dump(mode="json", exclude={"user_agent"})
|
||||
|
||||
|
||||
def assert_parity(python: Execution, rust: Execution, python_user_agent: str) -> None:
|
||||
assert python.report.native.rust_enabled is False
|
||||
assert rust.report.native.rust_enabled is True
|
||||
assert rust.report.native.native_callable_loaded is True
|
||||
assert rust.report.native.native_handled_case is True
|
||||
assert tuple(request.user_agent for request in python.requests) == (python_user_agent,) * len(python.requests)
|
||||
assert all(request.user_agent != python_user_agent for request in rust.requests)
|
||||
assert tuple(_stable_request(request) for request in rust.requests) == tuple(
|
||||
_stable_request(request) for request in python.requests
|
||||
)
|
||||
assert rust.report.trace == python.report.trace
|
||||
assert python.report.trace.exception is None
|
||||
assert python.report.trace.outputs
|
||||
|
||||
|
||||
def _short(value: object) -> str:
|
||||
rendered: Final = repr(value)
|
||||
return rendered if len(rendered) <= 240 else f"{rendered[:237]}..."
|
||||
|
||||
|
||||
def _diff(left: object, right: object, path: str = "$") -> tuple[str, ...]:
|
||||
if isinstance(left, BaseModel) and isinstance(right, BaseModel):
|
||||
return _diff(left.model_dump(mode="json"), right.model_dump(mode="json"), path)
|
||||
if isinstance(left, Mapping) and isinstance(right, Mapping):
|
||||
left_mapping: Final = cast(Mapping[str, object], left)
|
||||
right_mapping: Final = cast(Mapping[str, object], right)
|
||||
left_keys: Final = frozenset(left_mapping)
|
||||
right_keys: Final = frozenset(right_mapping)
|
||||
missing: Final = tuple(f"{path}.{key}: missing from left" for key in sorted(right_keys - left_keys))
|
||||
extra: Final = tuple(f"{path}.{key}: missing from right" for key in sorted(left_keys - right_keys))
|
||||
shared: Final = tuple(
|
||||
line
|
||||
for key in sorted(left_keys & right_keys)
|
||||
for line in _diff(left_mapping[key], right_mapping[key], f"{path}.{key}")
|
||||
)
|
||||
return (*missing, *extra, *shared)
|
||||
if (
|
||||
isinstance(left, Sequence)
|
||||
and not isinstance(left, (str, bytes))
|
||||
and isinstance(right, Sequence)
|
||||
and not isinstance(right, (str, bytes))
|
||||
):
|
||||
left_sequence: Final = cast(Sequence[object], left)
|
||||
right_sequence: Final = cast(Sequence[object], right)
|
||||
length_diff: Final = (
|
||||
(f"{path}: lengths differ ({len(left_sequence)} != {len(right_sequence)})",)
|
||||
if len(left_sequence) != len(right_sequence)
|
||||
else ()
|
||||
)
|
||||
item_diff: Final = tuple(
|
||||
line
|
||||
for index, (left_item, right_item) in enumerate(zip(left_sequence, right_sequence))
|
||||
for line in _diff(left_item, right_item, f"{path}[{index}]")
|
||||
)
|
||||
return (*length_diff, *item_diff)
|
||||
left_value: Final = cast(object, left)
|
||||
right_value: Final = cast(object, right)
|
||||
return () if left_value == right_value else (f"{path}: {_short(left_value)} != {_short(right_value)}",)
|
||||
|
||||
|
||||
def parity_comparison(left: object, right: object) -> list[str] | None:
|
||||
if not isinstance(left, (ParityTrace, CapturedRequest)) or type(left) is not type(right):
|
||||
return None
|
||||
differences: Final = _diff(left, right)
|
||||
return [f"Comparing {type(left).__name__} values:", *(f" {line}" for line in differences)]
|
||||
67
tests/test_litellm/parity/models.py
Normal file
67
tests/test_litellm/parity/models.py
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
|
||||
class ExceptionReport(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
class_name: str
|
||||
status_code: int | None
|
||||
message: str
|
||||
|
||||
|
||||
class SDKOutput(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
response_type: str
|
||||
response_json: dict[str, object]
|
||||
|
||||
|
||||
class ParityTrace(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
outputs: tuple[SDKOutput, ...]
|
||||
exception: ExceptionReport | None
|
||||
|
||||
|
||||
class NativeEvidence(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
rust_enabled: bool
|
||||
native_callable_loaded: bool
|
||||
native_handled_case: bool
|
||||
|
||||
|
||||
class SDKReport(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
trace: ParityTrace
|
||||
native: NativeEvidence
|
||||
|
||||
|
||||
class CapturedRequest(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
method: str
|
||||
path: str
|
||||
authorization: str | None
|
||||
content_type: str | None
|
||||
parity_case: str | None
|
||||
body: dict[str, object]
|
||||
user_agent: str | None
|
||||
|
||||
|
||||
class Execution(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
report: SDKReport
|
||||
requests: tuple[CapturedRequest, ...]
|
||||
|
||||
|
||||
class ReplayResponse(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
status_code: int
|
||||
headers: dict[str, str]
|
||||
body: dict[str, object]
|
||||
11
tests/test_litellm/parity/pytest_plugin.py
Normal file
11
tests/test_litellm/parity/pytest_plugin.py
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.test_litellm.parity.compare import parity_comparison
|
||||
|
||||
|
||||
def pytest_assertrepr_compare(config: pytest.Config, op: str, left: object, right: object) -> list[str] | None:
|
||||
if op != "==":
|
||||
return None
|
||||
return parity_comparison(left, right)
|
||||
89
tests/test_litellm/parity/replay.py
Normal file
89
tests/test_litellm/parity/replay.py
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import queue
|
||||
import threading
|
||||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from tests.test_litellm.parity.models import CapturedRequest, ReplayResponse
|
||||
|
||||
JSON_OBJECT: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
class JsonReplayServer(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
|
||||
def __init__(self, expected_path: str, response: ReplayResponse) -> None:
|
||||
super().__init__(("127.0.0.1", 0), _JsonReplayHandler)
|
||||
self.expected_path: Final = expected_path
|
||||
self.response: Final = response
|
||||
self.response_body: Final = json.dumps(response.body, sort_keys=True, separators=(",", ":")).encode()
|
||||
self.requests: queue.Queue[CapturedRequest] = queue.Queue()
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
return f"http://127.0.0.1:{self.server_address[1]}"
|
||||
|
||||
def take_requests(self) -> tuple[CapturedRequest, ...]:
|
||||
try:
|
||||
first: Final = self.requests.get(timeout=5)
|
||||
except queue.Empty as error:
|
||||
raise AssertionError("expected at least one provider request, received none") from error
|
||||
remaining_count: Final = self.requests.qsize()
|
||||
remaining: Final = tuple(self.requests.get_nowait() for _ in range(remaining_count))
|
||||
return (first, *remaining)
|
||||
|
||||
|
||||
class _JsonReplayHandler(BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
|
||||
def do_POST(self) -> None:
|
||||
provider: Final = self.server
|
||||
assert isinstance(provider, JsonReplayServer)
|
||||
length: Final = int(self.headers.get("content-length") or "0")
|
||||
body: Final = JSON_OBJECT.validate_json(self.rfile.read(length))
|
||||
content_type_header: Final = self.headers.get("content-type")
|
||||
content_type: Final = content_type_header.split(";", 1)[0].lower() if content_type_header else None
|
||||
provider.requests.put(
|
||||
CapturedRequest(
|
||||
method=self.command,
|
||||
path=self.path,
|
||||
authorization=self.headers.get("authorization"),
|
||||
content_type=content_type,
|
||||
parity_case=self.headers.get("x-parity-case"),
|
||||
body=body,
|
||||
user_agent=self.headers.get("user-agent"),
|
||||
)
|
||||
)
|
||||
matched: Final = self.path == provider.expected_path
|
||||
status_code: Final = provider.response.status_code if matched else 404
|
||||
response_body: Final = provider.response_body if matched else b'{"error":"unexpected path"}'
|
||||
response_headers: Final = provider.response.headers if matched else {"content-type": "application/json"}
|
||||
self.send_response(status_code)
|
||||
for name, value in response_headers.items():
|
||||
if name.lower() not in {"content-length", "transfer-encoding", "content-encoding"}:
|
||||
self.send_header(name, value)
|
||||
self.send_header("content-length", str(len(response_body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(response_body)
|
||||
|
||||
def log_message(self, format: str, *args: object) -> None:
|
||||
return
|
||||
|
||||
|
||||
@contextmanager
|
||||
def replay_json_response(expected_path: str, response: ReplayResponse) -> Generator[JsonReplayServer]:
|
||||
server: Final = JsonReplayServer(expected_path=expected_path, response=response)
|
||||
thread: Final = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
yield server
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join(timeout=5)
|
||||
56
tests/test_litellm/parity/runner.py
Normal file
56
tests/test_litellm/parity/runner.py
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from tests.test_litellm.parity.models import Execution, SDKReport
|
||||
from tests.test_litellm.parity.replay import JsonReplayServer
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PythonScriptRunner:
|
||||
entrypoint: Path
|
||||
rust_env_var: str
|
||||
python_user_agent: str
|
||||
|
||||
def command(self, case_file: Path, provider_url: str, report_file: Path) -> tuple[str, ...]:
|
||||
return (
|
||||
sys.executable,
|
||||
str(self.entrypoint.resolve()),
|
||||
str(case_file),
|
||||
provider_url,
|
||||
str(report_file),
|
||||
)
|
||||
|
||||
|
||||
def run_execution(
|
||||
runner: PythonScriptRunner,
|
||||
case_file: Path,
|
||||
report_file: Path,
|
||||
provider: JsonReplayServer,
|
||||
rust_enabled: bool,
|
||||
) -> Execution:
|
||||
env: Final = {
|
||||
**os.environ,
|
||||
runner.rust_env_var: "1" if rust_enabled else "0",
|
||||
"LITELLM_USER_AGENT": runner.python_user_agent,
|
||||
}
|
||||
completed: Final = subprocess.run(
|
||||
runner.command(case_file, provider.url, report_file),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
timeout=60,
|
||||
check=False,
|
||||
)
|
||||
assert completed.returncode == 0, (
|
||||
f"SDK subprocess failed with exit code {completed.returncode}\n"
|
||||
f"stdout:\n{completed.stdout}\n"
|
||||
f"stderr:\n{completed.stderr}"
|
||||
)
|
||||
report: Final = SDKReport.model_validate_json(report_file.read_text(encoding="utf-8"))
|
||||
return Execution(report=report, requests=provider.take_requests())
|
||||
25
tests/test_litellm/parity/test_parity.py
Normal file
25
tests/test_litellm/parity/test_parity.py
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
|
||||
from tests.test_litellm.parity.compare import parity_comparison
|
||||
from tests.test_litellm.parity.models import ParityTrace, SDKOutput
|
||||
|
||||
|
||||
def test_parity_comparison_reports_first_nested_output_difference() -> None:
|
||||
python: Final = ParityTrace(
|
||||
outputs=(SDKOutput(response_type="Chunk", response_json={"choices": [{"delta": {"content": "a"}}]}),),
|
||||
exception=None,
|
||||
)
|
||||
rust: Final = ParityTrace(
|
||||
outputs=(SDKOutput(response_type="Chunk", response_json={"choices": [{"delta": {"content": "b"}}]}),),
|
||||
exception=None,
|
||||
)
|
||||
|
||||
explanation: Final = parity_comparison(rust, python)
|
||||
|
||||
assert explanation is not None
|
||||
assert explanation == [
|
||||
"Comparing ParityTrace values:",
|
||||
" $.outputs[0].response_json.choices[0].delta.content: 'b' != 'a'",
|
||||
]
|
||||
17
tests/test_litellm/test_json_fs_cache.py
Normal file
17
tests/test_litellm/test_json_fs_cache.py
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from tests.test_litellm._json_fs_cache import JsonFileCache
|
||||
|
||||
|
||||
def test_json_file_cache_is_content_addressed_and_recursive(tmp_path: Path) -> None:
|
||||
key: Final = {"method": "POST", "body": {"model": "test-model", "pages": [0]}}
|
||||
reordered_key: Final = {"body": {"pages": [0], "model": "test-model"}, "method": "POST"}
|
||||
value: Final = {"request": key, "response": {"status_code": 200}}
|
||||
cache: Final = JsonFileCache(tmp_path / "provider")
|
||||
|
||||
stored_path: Final = cache.put(key, value)
|
||||
|
||||
assert stored_path.name == cache.path_for(reordered_key).name
|
||||
assert cache.get(reordered_key) == value
|
||||
assert JsonFileCache(tmp_path).values() == (value,)
|
||||
94
uv.lock
generated
94
uv.lock
generated
|
|
@ -3373,6 +3373,98 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hypothesis"
|
||||
version = "6.165.10"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "exceptiongroup", marker = "python_full_version < '3.11'" },
|
||||
{ name = "sortedcontainers" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/5c/e2/0fad246d2b6330e1f78479bfc566b5c22be82aee8a865cde9a08f648487d/hypothesis-6.165.10.tar.gz", hash = "sha256:68b45e09834cd80523cb1eb274463073c7a9af4e4ef7cff34d9615f355572d32", size = 503703, upload-time = "2026-08-16T22:56:15.404Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/05/c1/9a9538e6d185baf5cc7f15bc3b76e08efbb3de4b3c782f234356449c0dd7/hypothesis-6.165.10-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f839d29d0cc12048cf073d88ca4fdf94d420bc2b8afd69641ff6d496422ccd4f", size = 783243, upload-time = "2026-08-16T22:55:44.058Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a1/30/b70d9d79e871a75cbdeccd9067f20ecdb9eb2a1dfa03c630be3ad13b8b30/hypothesis-6.165.10-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:e10858f57ed0e74baa04393845f469fe8ad502c16ece4499bef7700c575611bd", size = 778815, upload-time = "2026-08-16T22:55:46.948Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/db/52/6f0a9b7aab24b0635e2238f3fbddea5b54b17879ac813df42a3cc3384c5c/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76a7be86d986223b9f1bdb7e7cbcdb048649901fdb956c598ef73bdab1786cd5", size = 1108009, upload-time = "2026-08-16T22:54:53.082Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f6/06/8d0d4e11ff02350d09ec9f9e90af354158e59e16a8907ba5199a4ff2d7e8/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:717aea574e0e5edba2868aa66b1caae335d8f1ad3fb29f01dd6502953fa823a1", size = 1136596, upload-time = "2026-08-16T22:54:54.443Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/59/dd/01a1e440f2e38dc1ccf5d597af5b8a0bee5f21b674c99c123b5554de9690/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4334058033e0214475f019e15492a50f3854fe8728cf51fe25c6191a2c3f8e52", size = 1135234, upload-time = "2026-08-16T22:55:08.911Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7d/18/8a26c24d3d9db20265f39df341ab265858c094e209571e3179cf237935f4/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2abb50cf1cf77d721de0a24c3f99d9c4ffdeb2cbd1e12aebb5a7a93e2b6b6d1f", size = 1157528, upload-time = "2026-08-16T22:56:02.159Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ea/8e/ce3c829b1937402d7944420ca26a05a0c8563e894dcff03d34ffa279d306/hypothesis-6.165.10-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:3de69aa8b924b400291a3cc42aaf78e6ab65c905a3e7e1a5dc39d95ef1b428cb", size = 1112870, upload-time = "2026-08-16T22:54:55.919Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f2/1b/4c4926d6c9a2b5d7cc090cc1e91219d6796102aa2a2c4b8f961c939e60b5/hypothesis-6.165.10-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5841331c504e02d7c334591681cb8587cdd59dee7e149db6d3db8e3f9e9f02eb", size = 1149683, upload-time = "2026-08-16T22:55:30.567Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cb/f9/df24eb28412f82465e2b7707f0ff1ec274d580bce389d4d9156617dc7bba/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2d0e0f8263d34dd8fa3b39eaa9a50bba56a8470b3dd9ebf6672d10840abe063e", size = 1283402, upload-time = "2026-08-16T22:54:18.054Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4d/07/c2b2a761300cf60b90ccebba4328175331e67d34f4fbd39429a7ddcdce49/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:0c4e6869817c3cfdf5a2b4d348497b95159bdecb3365be732c9b8570e36a4eef", size = 1409948, upload-time = "2026-08-16T22:54:22.343Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f4/ec/1c2bf1acdd0e273d81f833f85caf0ae5423db68a783554992fca36e6c541/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:9f07ae36c3b093e13687a894e79fe69e98a94c0b67fef656c575247682218143", size = 1265023, upload-time = "2026-08-16T22:54:41.402Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3d/a8/7f984908b7391160c7801b84e51ca8e4ba88c89e8d8811aa1aa7c03de73c/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:aff1f584c9538e8979cd180b1d70bf99bc16be19d4666414f49e5942b21a4f2c", size = 1282698, upload-time = "2026-08-16T22:56:06.998Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/48/78/3a5d91c2d0250521736c42dfa2402b75049bc5fe2fb716c10bc84bb91ed1/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1f2c4db25fb8ec1a16a8dba580666337b8ffb1887c4cf1750cc954313897cef7", size = 1324816, upload-time = "2026-08-16T22:54:46.675Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6f/99/27450763853a034bca1574d3e0a315164b33ff49c3862df6872dda45e25e/hypothesis-6.165.10-cp310-abi3-win32.whl", hash = "sha256:b33dc30170a7402e03c180f2c5ef69dc077152f35b91621e9cebcde9c7d71746", size = 669039, upload-time = "2026-08-16T22:55:11.962Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2c/fc/ff2988b72b5705ad9ca500444bf3f43e3c2f41edfa034bbfeb23b215791a/hypothesis-6.165.10-cp310-abi3-win_amd64.whl", hash = "sha256:e9f924aa610c0618445e1e8738c822c3190ce2a2699a0cb48ec3a351a96761f2", size = 675213, upload-time = "2026-08-16T22:55:01.697Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c5/8b/821810d36f78d9d9421cd2c5d9d36983b45bb3575c3086276cc5c76f9f73/hypothesis-6.165.10-cp310-abi3-win_arm64.whl", hash = "sha256:1d305448e9bd8e2f4f3cea0eafd809efdaab4e998a0019bc615650c8463e42f1", size = 673537, upload-time = "2026-08-16T22:54:47.898Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/26/61/5e89268ce03317fb9f82449a1b3efd9e599dee090288fd0cf7586c532fb1/hypothesis-6.165.10-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:73e6df02a6a62f8045b511c272f894d08e56d174504c793c9effcbc6778051a8", size = 783959, upload-time = "2026-08-16T22:55:29.078Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e1/e9/f4e0832e81bb53b70cf1712e28c867db64245b32595b594217452e7dbd8d/hypothesis-6.165.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8b20f44773a9ab84400465e318712d8c2ca16418d35b9f80aa27fdf2d690ad10", size = 779684, upload-time = "2026-08-16T22:54:57.698Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/77/de/ea072d3359d5678771bed407f80439e8ac7ca905d1031b0372f61bf5746e/hypothesis-6.165.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb8c7d05ea27a093a92b250904095d71d924b6b44e5795a415c1b20c265f0c65", size = 1108540, upload-time = "2026-08-16T22:55:03.282Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/28/56/e7c395cdaa3d6c28b944c1c3c516dee50d2b7b3aeafa31874b57009ca51f/hypothesis-6.165.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f4dafd6d6ababfa3b14dd6e5f0378cb7c7d291895a31a40abcbb7cc74f396131", size = 1158089, upload-time = "2026-08-16T22:54:36.205Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b1/49/1c6d2c465b9c5fc3213f1be89be95ba53819ca0130248c484129ccfefb71/hypothesis-6.165.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:fa74636a49fc8077413ce8db3e85f1c4aff880788bb55bda56253118e036fe5b", size = 1284125, upload-time = "2026-08-16T22:54:37.727Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7d/bc/7caf5ac3d0173bd57bd2a5ab854ca49a3664a4309257be1452f81025cc24/hypothesis-6.165.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2b112768cfb67f2b683e53e58c1a33d27811aacf60c942b8eb74635e469a73f6", size = 1325082, upload-time = "2026-08-16T22:54:38.952Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/70/99/9d844330f570d6a4f127a683eab1e78c8263e6e72b16189f3534fa6bf6de/hypothesis-6.165.10-cp310-cp310-win_amd64.whl", hash = "sha256:56cb8c9055e50545fe6e3e5a560ec25a724673b2e4051f3c24d44e3ebc35dd72", size = 675082, upload-time = "2026-08-16T22:54:29.872Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ed/c2/b9546ace11f241c9c02d389f258cb80c14447a8c885771c9f1f0bc1d85ca/hypothesis-6.165.10-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:592107a0faf6c9c3a63a8dbf13dfb1cbda1cf599b0bc11c953221b00204b9ce1", size = 783716, upload-time = "2026-08-16T22:55:36.624Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/37/10/27c2fdd574fd798caf5e91eb51f7834b098f5d840ce733efb3fba79ef86e/hypothesis-6.165.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f9180c362bde06fd05380298ded4e234fbc0d6ede0a864835bfd91c1e24283d5", size = 779507, upload-time = "2026-08-16T22:55:07.633Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5e/b6/70bc23695f3783c4b0486b6cad47b08a20f791db4a3c1b25250add9659fa/hypothesis-6.165.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d623801ae3dcd97b77b983400ef3d48bf976648e4efff19929175322eaae074d", size = 1108406, upload-time = "2026-08-16T22:55:39.653Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/71/4c/32e200bd7a352af4b7f4e3729aaa4cd002cb5fe8c4c6aef5599d0019f152/hypothesis-6.165.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20f6236cfb90b7817bb1a6a087589ca4aa46d73170f0dd62963952ed5dadc589", size = 1157850, upload-time = "2026-08-16T22:55:24.394Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/03/a5/8efc2a9a484822efc0d0da466f50094e0f2c068187faaf33831fc905873e/hypothesis-6.165.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ad0764730e8e3421601c2cc7e1f054a9206c60ea0917165d8d9193dc453f34f1", size = 1283704, upload-time = "2026-08-16T22:54:27.279Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/46/2a/90cc8d7463929c04786f29600de45f3227c12fa9bed1d5b7ce319b05e1c9/hypothesis-6.165.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:10d9a650a4666b0914831f769703d36140ed8039fd19bf9b71f615b8541eccf2", size = 1325077, upload-time = "2026-08-16T22:55:16.561Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/82/ac/bc16faba4b42883e3d290bfaceff51e258b63fbbdf789bf9fe88df1ce537/hypothesis-6.165.10-cp311-cp311-win_amd64.whl", hash = "sha256:5671d2b2bf83bd4b6f02e55b32d432506eff5358c82f39b460a849ce19a2666e", size = 674920, upload-time = "2026-08-16T22:55:42.613Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e9/45/cde4f78afe2b9e29caecf38319eedc1deb76aebcacbdd128e03cbb2511c3/hypothesis-6.165.10-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:637445c1593a2a9d1024fda50082f07bb56baedda78d90a25f64b8111727ef94", size = 784835, upload-time = "2026-08-16T22:54:45.429Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7f/81/847f30b81cbfd07607296b3ce43067cf4f80799bd9244167f587de9c8081/hypothesis-6.165.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:713f4ce4e82c26b53031f139de959bc9e8b54d3995aa824b89bbdf8229df2a45", size = 776419, upload-time = "2026-08-16T22:55:33.633Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/04/66/4c71c5be7a49d84b8c3a9278c1807c4c81181ab5474beb27df9d4c40dc0e/hypothesis-6.165.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9ff356e97e3ab09db07c8b675efa67340103874a0bae7465acb83dad7a35f7f", size = 1106830, upload-time = "2026-08-16T22:55:10.389Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e3/c4/e2cbd2810e79f7a452a8ea9f6c6438ee718ce938d8cc12252cf0b36a81d3/hypothesis-6.165.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a380bc99aa3b035e6a95a2201bf792d4082a04ca75babcc21849c2d0914bb28", size = 1156952, upload-time = "2026-08-16T22:55:53.35Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a8/8b/794ced36864825492ac3712d5acab5a257b4601e6a9dc2ccdd3937198f87/hypothesis-6.165.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e9acb2c4d9cb532c3fedea74159f7b923c8c036328c9239b4049e7aa073bdd81", size = 1280780, upload-time = "2026-08-16T22:54:34.983Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5d/2d/550525442cdbcc2daf1f9bdd8ba35bcbde63db7c7a22f2ef137fbb49df2f/hypothesis-6.165.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8660572b2d424bf5369ea8990985225f70bd1615b76ecd9c25588a3b9307009f", size = 1324130, upload-time = "2026-08-16T22:55:48.659Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/74/59/6caf69dd5fe03499ada94c9cec016bffcc164511c6b93fe680f01209b9ff/hypothesis-6.165.10-cp312-cp312-win_amd64.whl", hash = "sha256:3376f2594763aef14faa519b0fb27cae7ce9eeaab4c69efa07777499110306c9", size = 672337, upload-time = "2026-08-16T22:54:49.11Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b1/fb/c82c5bd92864ffcf319772fedc8c9bf2dbe4ca14baa0fee6e49e67b5ba1c/hypothesis-6.165.10-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:9d77c3be7b429875036ad0f0597c6e5cc6bb17894a4da005e3807de64d2673ad", size = 784726, upload-time = "2026-08-16T22:54:32.371Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0e/b9/3d7acd08506da85557e65147b7f3fca8c47684e33be90bee0acb523920db/hypothesis-6.165.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:490c56b830772b0eca3b4b2cecb3741a1ed26b1d7206a279e1525dbf0aa95ee4", size = 776375, upload-time = "2026-08-16T22:55:13.303Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/38/6b/922e8b3f9a706dd89d440b9545d2c6231c65e74da1c1fee3ff36c251b9c4/hypothesis-6.165.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed68e27b8a61e57a3ccdc7c5a14499e00b54dfe223087204d5d40b3b5ef58b6d", size = 1106763, upload-time = "2026-08-16T22:55:06.129Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/01/39/f5b9a5d390d4edd1ad472334493ac442963ebeb4daaa74ff4bdac6ef292f/hypothesis-6.165.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6caadcd1afb62630ff5c5ff353626eaa616553a5971295ad6dc2b19ca8a39620", size = 1156778, upload-time = "2026-08-16T22:54:33.824Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b5/5f/5fbe1be4326337fd6acefe2d18ed44007ee1dc1f98fe5b3c0eb22942364d/hypothesis-6.165.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9145fe43ebb22e66672967c3fab411793b226ed776e4fe282271bca6ad3c0bb", size = 1280756, upload-time = "2026-08-16T22:55:54.834Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/25/c0/cf6f9e1ef632a1a75694eed0db3a02e6fc75c367a363e94acee52f043c64/hypothesis-6.165.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:79900a9920a0b1d3a626c03a90ac6bf7042e78d46906a565b86a0dbe926f1d96", size = 1323889, upload-time = "2026-08-16T22:55:56.567Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cc/cc/662b94880f260b0a88de1fdcf60fc9984f6e2a796da549542adc10a7bc83/hypothesis-6.165.10-cp313-cp313-win_amd64.whl", hash = "sha256:c01dd04044c472e47193b54f68e84e08d6ebf4f29551885aa959b015f7cd9747", size = 672346, upload-time = "2026-08-16T22:56:03.792Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3f/77/55e020c9c576532ff7d20bf8b1dfa052ecbd5ada1949b02f76c44c966f7e/hypothesis-6.165.10-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:9ccac776b2ca93b324806facd526ccb45da0fd035001c899a35b02c44431e209", size = 784833, upload-time = "2026-08-16T22:55:21.255Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4f/f2/01da2adf829cf549eaddcabb8e8072077fb3d26da4275f4c1e89b2c0af74/hypothesis-6.165.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e5f95f7b622e4171096d92175dda0a560f0955ade9b8a3a07bdcf151f7359611", size = 776545, upload-time = "2026-08-16T22:56:10.159Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cf/8e/58d4f842895220b793c53fc94a6489705b3665bb4d0ae4d338ce03fdf9fb/hypothesis-6.165.10-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f76d1562643693b8a40066f1f96af795b93fd9bcfc9690a1af2ff4c5867ee29e", size = 1107271, upload-time = "2026-08-16T22:54:50.266Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8f/b8/206468912d2153306bb8a41afdfc59e45b7a73a0495bbe4b9cb4f0e79c1d/hypothesis-6.165.10-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60cab3ab4ea468d31a33739ffd7e94ec3e37dea891d65a6582ecc8a477175191", size = 1156915, upload-time = "2026-08-16T22:54:25.89Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fb/d3/bf5a22929b70a4cfd3edf69c5642b029b27ddb5cfda48fa295d384b01abb/hypothesis-6.165.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:22cf19388f0ff6ced8eb3e49c903d14938e4ed909d93bf28383eef451511e424", size = 1281205, upload-time = "2026-08-16T22:54:44.083Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/07/a2/d7b2ba444d36fc84d4779f4431e74dd9b023dc63bcf282199f6e48ad39f4/hypothesis-6.165.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:057d0232f1224dcd0b7698902551a4341a7399f90670b036db6c4376715fe889", size = 1324243, upload-time = "2026-08-16T22:55:41.123Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d1/95/afe6b531fd01928c6f63d394ee413fa2338d088b2b44efcc23596b54477e/hypothesis-6.165.10-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:ab0f2e9d7d7d4db257f7cf53de3706c2baf124269571f20ffc2bcd6781f03063", size = 616382, upload-time = "2026-08-16T22:55:18.449Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/48/86/9b4fb75f520a028edec50ffc904a94d724180395d71feb6d7a0ce7bb6f00/hypothesis-6.165.10-cp314-cp314-win_amd64.whl", hash = "sha256:d1ea02fa8ab3d33eb1125eade81f7136341eb429152c6dbe2ae6f8bc33b3fbdd", size = 672145, upload-time = "2026-08-16T22:54:24.831Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f9/ba/f7bbaae0c789bab7ddb764d2056ee1a463cc95a8acbccc90d4184e48b242/hypothesis-6.165.10-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:ed1a5891e59472884a03cb9875483e8fc131c80a275c60967f8afc5458a0c8ff", size = 783287, upload-time = "2026-08-16T22:54:23.751Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3a/83/01ef80772b4abd335c49405576dc503cede94fb5da30ba2643a119013aea/hypothesis-6.165.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:09772e328a26e50486ac572be34f9887f9aa185efe7ebb16bde4e8f6038db1f4", size = 774991, upload-time = "2026-08-16T22:55:25.987Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a3/0b/f47506241f9d5a5a2efe4c65b6bf4830e9d9576e5d3779007a260699e608/hypothesis-6.165.10-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cf3b612542ba174c9da4000b59a4f4c81e8d66f87509be85d3a1b71b5c36413", size = 1105499, upload-time = "2026-08-16T22:54:51.864Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/84/fe/abb3909b7089835112fbe75bf00d817d733b3a8032759783db0a24ff1e56/hypothesis-6.165.10-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f69ec5be85ef508e206153bed8eafd03f7995dc464356c8bbb279a1e2b7d56f3", size = 1155685, upload-time = "2026-08-16T22:54:30.94Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/73/2f/1964738921640184067121ae77414522fc3f0463fc26c6e25a4f3b8e42ca/hypothesis-6.165.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:dd207497bb985918409a1bb5db85d1875f74e1269487332113b73d1ee7c77647", size = 1279177, upload-time = "2026-08-16T22:54:40.179Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/34/c5/312af8ae038d3af9cf3f7f1021c1abfe31c0d9035e4cf63519e0a7dc983e/hypothesis-6.165.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:00de0abdcf8c05c9d0eab735a3c49a276376b55151e6fcb903c2b39a90e5e5c3", size = 1322921, upload-time = "2026-08-16T22:54:42.7Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9e/e7/b0a2fde7570c090a1b914026266a421c751ef10138fffe37fe0ef9e675c0/hypothesis-6.165.10-cp314-cp314t-win_amd64.whl", hash = "sha256:cc2da5aa4edf14743fa9257e5ba3513963999f01211635702479d8e92b8207c8", size = 672147, upload-time = "2026-08-16T22:55:27.527Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/47/fd/985aa564d6ffd06483d45a62b40d319df0a703cd8bc1d041de17d102fbaa/hypothesis-6.165.10-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:eeab73050ea58c13dd56e329f594c1dfe32ebd7bb169bbdf4f8ceefbc31ec6b5", size = 782882, upload-time = "2026-08-16T22:55:37.93Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f8/2c/6cc11151e450f72353a490940cd0db704680d07b78dc75dcc9f480e0d0e1/hypothesis-6.165.10-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:4c68e983d0007d014bb01ad4bcbba78bc432c73a1755ff36d5102ceefa18299a", size = 774584, upload-time = "2026-08-16T22:55:51.822Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/10/39/ef26fa79c1738dfe9cdb1a3584fb6717d26429ca6c9d011cc4fdf08130c2/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7730d8197086f65d8969a991d6728a1d420a51b19fea06535c896cb43a1e05d0", size = 1104876, upload-time = "2026-08-16T22:54:58.937Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4e/f4/3fcc84e7637f42bf00d987093b9418083ac8db81b87392608a60f4b7c5fd/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7a7980a898a3e6ebe4de1896a0507e3d519edb53fb9b4bda478c9fbeb6514558", size = 1133353, upload-time = "2026-08-16T22:54:28.635Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/35/59/21c5c14179c38f8d0de3560e7f1825c083311b3013b63f817d7dc78dfcbd/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b5820d009aedb7ae9cfd32f98b1ab0c0bbd6268379c4fab042218b6b655c63f8", size = 1132300, upload-time = "2026-08-16T22:56:08.539Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/14/af/fbb56059961e416b2de7b9dc5352db2e8572bd5ea46892957e4c1e5548ab/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37a7ac3d34220800e1107871cc391bca1b00439875925d7d821878b8b791f245", size = 1155175, upload-time = "2026-08-16T22:55:19.824Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0f/53/77fb0c2dad445858555429c4e06cf94a59ae8d2407dd6426b5af97c84828/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:dafa7c9dbe3d802f9bcdf261b29c8a70700fb22839947f06e471f62c46b6257f", size = 1109881, upload-time = "2026-08-16T22:55:32.029Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a8/7b/d187f673ff30e6ada640953636f978ffe64a6332f756b64163c2277f8d0c/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:90915635b9648071129b0f72c0673cf8eac9eb84cfd445c5bedef30c714b1ec2", size = 1144963, upload-time = "2026-08-16T22:56:13.428Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e0/60/31d504e364134d60af23e5f6365db0da3cf4a51b3ed3d4836e5a2cff12cf/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:e1bbeb7c506b07ee0422cf9b2f7212fefa4240957f03526d38d27bc6743a0a48", size = 1278684, upload-time = "2026-08-16T22:55:22.971Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ef/e6/89d26834a08c02f8da149e541dd40d7a96f68d9722f43146e69a77436ed7/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:2b36aaffc88625a44f91074c5bbedfdefb9b376c38d1b3c342edcd2e4c8ed16c", size = 1407202, upload-time = "2026-08-16T22:55:14.949Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/dc/61/20d1e72246867ea195440092e8bb422c7ddc2f271b87b5b65679d5532719/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:18a3ea838ddea183388f8788750afa8494d79abb5358823be9782585f34445d3", size = 1261395, upload-time = "2026-08-16T22:56:05.448Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2a/9b/ebab6c3c2b90a16abb4119198178652d12aff83cc8ec2cfde5276c69fb1e/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:2a2567b3a03a4a5a7c575c191cfcce321a967df3727803817e75bffbbeaecabe", size = 1279213, upload-time = "2026-08-16T22:55:35.066Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/23/78/69b219b524231d36eb20c792e1f01e7cb037e02bd0af1c29f77ed9a969c0/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:8001925fa3dde51cb574e4c9de4c7efe77c4e4d64bd2fd2ef61d5651f9d04f3d", size = 1322367, upload-time = "2026-08-16T22:54:21.279Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/55/63/ad5cc153dcc72ae5e7905fb9b3585f3e48ce892a2d6366f90163e867a69d/hypothesis-6.165.10-cp315-abi3.abi3t-win32.whl", hash = "sha256:c6559380469295c4009215fe1cab561301591a3bee2e2fb3f4f96d2273a3affc", size = 666038, upload-time = "2026-08-16T22:56:11.797Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/80/32/b62307b73fbc99f0a4381d6f9456df76fbcbb7a27ef7256e26f0376f48ea/hypothesis-6.165.10-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:30797f20ca45e57f526d2df872f63ba453cb4e1091ad542184a7a951af8da79d", size = 671941, upload-time = "2026-08-16T22:55:00.235Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c2/dd/e0f98add0548ef73ea7afac45da1fb8efc854d7f9931db568754d0f963f3/hypothesis-6.165.10-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:c53e9b1c36350df9965ec44d6c0d4e0bbbb38f720dd2b0e1256dc6524d411015", size = 669931, upload-time = "2026-08-16T22:55:50.205Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0b/6a/880d6eeed5c451fb40a66733dadec4a5d498628a4a7f6a8a5f633f4c6dcb/hypothesis-6.165.10-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:34ee6402df6f31274d89119f1561b5f7489c97866afc5b7a3ed3a13d7e762802", size = 784644, upload-time = "2026-08-16T22:54:20.127Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/27/e0/9e942bd3c3cf5ea0d5c0fd0905893bbfb6cefb7284c70fcc8033f8fdec38/hypothesis-6.165.10-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:277f41801e88dad2eba082f91a75632b7584ff64044ba2cf9dadf511b0d19cd0", size = 780515, upload-time = "2026-08-16T22:55:04.676Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/19/32/f11a618415dc5fa9cdde41fea56c489f0814759527ae1ecd11a75a4558b9/hypothesis-6.165.10-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:72df95fb1db41755b155c5f02106e0036a339250555c8d351d488704fd112cf9", size = 1109374, upload-time = "2026-08-16T22:56:00.241Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5e/6f/db49b719842297c2b71e0d81e5b8967d31215fb7389421abcb465ce7ed3f/hypothesis-6.165.10-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e20a02775eb3cf0ffb4f0219b6d7c1f240336663d4e5d7028675ec247c790c4", size = 1159092, upload-time = "2026-08-16T22:55:58.57Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b2/2a/bf0bae84ba1cb3923d295973f1fe38ee867eaf90119e0d559116083be300/hypothesis-6.165.10-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:1ec53f08732e3cfd0342cbbd75dbd1b193c8f19390660466e536a748bb81f757", size = 676045, upload-time = "2026-08-16T22:55:45.514Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "3.15"
|
||||
|
|
@ -4430,6 +4522,7 @@ dev = [
|
|||
{ name = "diff-cover" },
|
||||
{ name = "fakeredis" },
|
||||
{ name = "fastapi-offline" },
|
||||
{ name = "hypothesis" },
|
||||
{ name = "keyring" },
|
||||
{ name = "langfuse" },
|
||||
{ name = "openapi-core" },
|
||||
|
|
@ -4615,6 +4708,7 @@ dev = [
|
|||
{ name = "diff-cover", specifier = "==9.7.2" },
|
||||
{ name = "fakeredis", specifier = "==2.34.1" },
|
||||
{ name = "fastapi-offline", specifier = "==1.7.6" },
|
||||
{ name = "hypothesis", specifier = "==6.165.10" },
|
||||
{ name = "keyring", specifier = "==25.7.0" },
|
||||
{ name = "langfuse", specifier = "==2.59.7" },
|
||||
{ name = "openapi-core", specifier = "==0.22.0" },
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue