diff --git a/.gitignore b/.gitignore index deb0acae56e..4259be35d08 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .python-version .venv tests/e2e/.fixtures/ +tests/test_litellm/ocr/.fixtures/ .venv-typecheck .venv_policy_test .venv-mutmut diff --git a/pyproject.toml b/pyproject.toml index 2866e27e84c..576f18c3597 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -174,6 +174,7 @@ litellm-proxy = "litellm.proxy.client.cli:cli" [dependency-groups] dev = [ "diff-cover==9.7.2", + "hypothesis==6.165.10", "basedpyright==1.39.7", "keyring==25.7.0", "pytest==9.0.3", diff --git a/tests/test_litellm/_json_fs_cache.py b/tests/test_litellm/_json_fs_cache.py new file mode 100644 index 00000000000..61ae77b915a --- /dev/null +++ b/tests/test_litellm/_json_fs_cache.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +from pydantic import TypeAdapter + +JSON_OBJECT: Final = TypeAdapter(dict[str, object]) + + +def canonical_json(value: Mapping[str, object]) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + +@dataclass(frozen=True, slots=True) +class JsonFileCache: + root: Path + + def path_for(self, key: Mapping[str, object]) -> Path: + digest: Final = hashlib.sha256(canonical_json(key).encode("utf-8")).hexdigest() + return self.root / f"{digest}.json" + + def get(self, key: Mapping[str, object]) -> dict[str, object] | None: + path: Final = self.path_for(key) + if not path.is_file(): + return None + return JSON_OBJECT.validate_json(path.read_text(encoding="utf-8")) + + def put(self, key: Mapping[str, object], value: Mapping[str, object]) -> Path: + self.root.mkdir(parents=True, exist_ok=True) + path: Final = self.path_for(key) + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return path + + def values(self) -> tuple[dict[str, object], ...]: + if not self.root.is_dir(): + return () + paths: Final = tuple(sorted(self.root.rglob("*.json"))) + return tuple(JSON_OBJECT.validate_json(path.read_text(encoding="utf-8")) for path in paths) diff --git a/tests/test_litellm/ocr/conftest.py b/tests/test_litellm/ocr/conftest.py new file mode 100644 index 00000000000..6e0b033e28d --- /dev/null +++ b/tests/test_litellm/ocr/conftest.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import hashlib +import os +from pathlib import Path +from typing import Final + +import pytest + +from tests.test_litellm._json_fs_cache import JsonFileCache, canonical_json +from tests.test_litellm.ocr.fixture_models import OcrFixture + +FIXTURE_DIR_ENV: Final = "LITELLM_OCR_FIXTURE_DIR" +pytest_plugins: Final = ("tests.test_litellm.parity.pytest_plugin",) + + +def _fixture_directory() -> Path: + configured: Final = os.environ.get(FIXTURE_DIR_ENV) + return Path(configured).expanduser() if configured else Path(__file__).with_name(".fixtures") + + +def _recorded_fixtures() -> tuple[OcrFixture, ...]: + raw_fixtures: Final = JsonFileCache(_fixture_directory()).values() + return tuple(OcrFixture.model_validate(raw_fixture) for raw_fixture in raw_fixtures) + + +def _fixture_id(fixture: OcrFixture) -> str: + raw_model: Final = fixture.request.sdk_kwargs.get("model") + model: Final = raw_model if isinstance(raw_model, str) else "unknown-model" + request_json: Final = canonical_json(fixture.request.provider_request.model_dump(mode="json")) + digest: Final = hashlib.sha256(request_json.encode("utf-8")).hexdigest()[:8] + return f"{fixture.request.provider}-{model.rsplit('/', 1)[-1]}-{digest}" + + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + if "ocr_fixture" not in metafunc.fixturenames: + return + fixtures: Final = _recorded_fixtures() + if not fixtures: + metafunc.parametrize( + "ocr_fixture", + ( + pytest.param( + None, + marks=pytest.mark.skip(reason=f"no recorded OCR fixtures in {_fixture_directory()}"), + id="no-recorded-fixtures", + ), + ), + ) + return + metafunc.parametrize("ocr_fixture", fixtures, ids=tuple(_fixture_id(fixture) for fixture in fixtures)) diff --git a/tests/test_litellm/ocr/fixture_models.py b/tests/test_litellm/ocr/fixture_models.py new file mode 100644 index 00000000000..bd2c73ffc8e --- /dev/null +++ b/tests/test_litellm/ocr/fixture_models.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + + +class ProviderWireRequest(BaseModel): + model_config = ConfigDict(frozen=True) + + method: str + path: str + body: dict[str, object] + + +class OcrFixtureRequest(BaseModel): + model_config = ConfigDict(frozen=True) + + provider: str + sdk_kwargs: dict[str, object] + provider_request: ProviderWireRequest + + +class OcrFixtureResponse(BaseModel): + model_config = ConfigDict(frozen=True) + + status_code: int + headers: dict[str, str] + body: dict[str, object] + + +class OcrFixture(BaseModel): + model_config = ConfigDict(frozen=True) + + request: OcrFixtureRequest + response: OcrFixtureResponse diff --git a/tests/test_litellm/ocr/generate_fixtures.py b/tests/test_litellm/ocr/generate_fixtures.py new file mode 100644 index 00000000000..8783cf74ac6 --- /dev/null +++ b/tests/test_litellm/ocr/generate_fixtures.py @@ -0,0 +1,348 @@ +from __future__ import annotations + +import argparse +import base64 +import json +import logging +import os +import queue +import threading +from collections.abc import Callable, Generator +from contextlib import contextmanager +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Final, cast +from urllib.parse import quote + +import httpx +from dotenv import load_dotenv +from hypothesis import given, settings +from hypothesis import strategies as st +from pydantic import TypeAdapter + +import litellm +from tests.test_litellm._json_fs_cache import JsonFileCache +from tests.test_litellm.ocr.fixture_models import ( + OcrFixture, + OcrFixtureRequest, + OcrFixtureResponse, + ProviderWireRequest, +) + +FIXTURE_DIR_ENV: Final = "LITELLM_OCR_FIXTURE_DIR" +JSON_OBJECT: Final = TypeAdapter(dict[str, object]) +PROVIDER_NAMES: Final = ("mistral", "azure_ai", "vertex_ai") +LOGGER: Final = logging.getLogger(__name__) +_TEXT: Final = st.from_regex(r"[A-Za-z0-9 ]{1,24}", fullmatch=True) +_OPTIONS: Final = st.fixed_dictionaries( + { + "pages": st.just([0]), + "include_image_base64": st.booleans(), + "image_limit": st.integers(min_value=1, max_value=4), + "image_min_size": st.integers(min_value=0, max_value=64), + "extract_header": st.booleans(), + "extract_footer": st.booleans(), + "table_format": st.sampled_from(("markdown", "html")), + "confidence_scores_granularity": st.sampled_from(("word", "page")), + "include_blocks": st.booleans(), + "id": _TEXT, + } +) + + +@dataclass(frozen=True, slots=True) +class ProviderSpec: + name: str + model: str + upstream_base: str + api_key: str | None + upstream_model: str | None = None + vertex_project: str | None = None + vertex_location: str | None = None + + +@dataclass(frozen=True, slots=True) +class RecorderResult: + fixture: OcrFixture + cache_hit: bool + + +@dataclass(frozen=True, slots=True) +class GeneratorArgs: + providers: tuple[str, ...] + examples: int + fixture_dir: Path | None + + +class _RecordingProvider(ThreadingHTTPServer): + daemon_threads = True + + def __init__( + self, + spec: ProviderSpec, + sdk_kwargs: dict[str, object], + cache: JsonFileCache, + ) -> None: + super().__init__(("127.0.0.1", 0), _RecordingHandler) + self.spec: Final = spec + self.sdk_kwargs: Final = sdk_kwargs + self.cache: Final = cache + self.results: queue.Queue[RecorderResult] = queue.Queue() + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.server_address[1]}" + + def take_result(self) -> RecorderResult: + try: + return self.results.get(timeout=5) + except queue.Empty as error: + raise RuntimeError("successful OCR call did not produce a recorder result") from error + + +class _RecordingHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: + provider: Final = self.server + assert isinstance(provider, _RecordingProvider) + length: Final = int(self.headers.get("content-length") or "0") + body: Final = JSON_OBJECT.validate_json(self.rfile.read(length)) + fixture_request: Final = OcrFixtureRequest( + provider=provider.spec.name, + sdk_kwargs=provider.sdk_kwargs, + provider_request=ProviderWireRequest(method=self.command, path=self.path, body=body), + ) + cache_key: Final = _fixture_cache_key(provider.spec.name, fixture_request.provider_request) + cached_value: Final = provider.cache.get(cache_key) + if cached_value is not None: + cached_fixture: Final = OcrFixture.model_validate(cached_value) + provider.results.put(RecorderResult(fixture=cached_fixture, cache_hit=True)) + self._send_fixture_response(cached_fixture.response) + return + + upstream_url: Final = f"{provider.spec.upstream_base.rstrip('/')}{self.path}" + forwarded_headers: Final = { + name: value + for name, value in self.headers.items() + if name.lower() not in {"host", "content-length", "accept-encoding", "x-parity-case"} + } + upstream_body: Final = ( + {**body, "model": provider.spec.upstream_model} if provider.spec.upstream_model is not None else body + ) + try: + upstream_response: Final = httpx.post( + upstream_url, + headers=forwarded_headers, + content=json.dumps(upstream_body, separators=(",", ":")), + timeout=120, + ) + except httpx.HTTPError as error: + error_body: Final = json.dumps({"error": str(error)}).encode() + self._send_response(502, {"content-type": "application/json"}, error_body) + return + + raw_content_type: Final = cast(object, upstream_response.headers.get("content-type", "application/json")) + content_type: Final = raw_content_type if isinstance(raw_content_type, str) else "application/json" + response_headers: Final = {"content-type": content_type.split(";", 1)[0]} + if not upstream_response.is_success: + self._send_response(upstream_response.status_code, response_headers, upstream_response.content) + return + + upstream_response_body: Final = JSON_OBJECT.validate_json(upstream_response.content) + fixture_response: Final = OcrFixtureResponse( + status_code=upstream_response.status_code, + headers=response_headers, + body=upstream_response_body, + ) + recorded_fixture: Final = OcrFixture(request=fixture_request, response=fixture_response) + provider.results.put(RecorderResult(fixture=recorded_fixture, cache_hit=False)) + self._send_fixture_response(fixture_response) + + def _send_fixture_response(self, response: OcrFixtureResponse) -> None: + response_body: Final = json.dumps(response.body, separators=(",", ":")).encode() + self._send_response(response.status_code, response.headers, response_body) + + def _send_response(self, status_code: int, headers: dict[str, str], body: bytes) -> None: + self.send_response(status_code) + for name, value in headers.items(): + self.send_header(name, value) + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + return + + +@contextmanager +def _recording_provider( + spec: ProviderSpec, + sdk_kwargs: dict[str, object], + cache: JsonFileCache, +) -> Generator[_RecordingProvider]: + server: Final = _RecordingProvider(spec=spec, sdk_kwargs=sdk_kwargs, cache=cache) + thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +def _mistral_upstream_base() -> str: + configured: Final = os.environ.get("MISTRAL_API_BASE", "https://api.mistral.ai").rstrip("/") + return configured.removesuffix("/v1") + + +def _fixture_cache_key(provider: str, request: ProviderWireRequest) -> dict[str, object]: + return {"provider": provider, "request": request.model_dump(mode="json")} + + +def _provider_specs(selected: tuple[str, ...]) -> tuple[ProviderSpec, ...]: + specs: Final[dict[str, ProviderSpec | None]] = { + "mistral": ( + ProviderSpec( + name="mistral", + model=os.environ.get("MISTRAL_OCR_MODEL", "mistral/mistral-ocr-latest"), + upstream_base=_mistral_upstream_base(), + api_key=os.environ.get("MISTRAL_API_KEY") or os.environ.get("LITELLM_API_KEY"), + upstream_model=os.environ.get("MISTRAL_OCR_UPSTREAM_MODEL"), + ) + if os.environ.get("MISTRAL_API_KEY") or os.environ.get("LITELLM_API_KEY") + else None + ), + "azure_ai": ( + ProviderSpec( + name="azure_ai", + model=os.environ.get("AZURE_AI_OCR_MODEL", "azure_ai/mistral-document-ai-2512"), + upstream_base=os.environ["AZURE_AI_API_BASE"], + api_key=os.environ["AZURE_AI_API_KEY"], + ) + if os.environ.get("AZURE_AI_API_BASE") and os.environ.get("AZURE_AI_API_KEY") + else None + ), + "vertex_ai": _vertex_spec(), + } + missing: Final = tuple(name for name in selected if specs[name] is None) + if missing: + LOGGER.warning("Skipping providers without credentials: %s", ", ".join(missing)) + return tuple(spec for name in selected if (spec := specs[name]) is not None) + + +def _vertex_spec() -> ProviderSpec | None: + project: Final = os.environ.get("VERTEXAI_PROJECT") + credentials_available: Final = bool(os.environ.get("VERTEX_AI_API_KEY") or os.environ.get("VERTEXAI_CREDENTIALS")) + if project is None or not credentials_available: + return None + location: Final = os.environ.get("VERTEXAI_LOCATION", os.environ.get("VERTEX_LOCATION", "us-central1")) + upstream_base: Final = os.environ.get("VERTEX_AI_API_BASE", f"https://{location}-aiplatform.googleapis.com") + return ProviderSpec( + name="vertex_ai", + model=os.environ.get("VERTEX_AI_OCR_MODEL", "vertex_ai/mistral-ocr-2505"), + upstream_base=upstream_base, + api_key=os.environ.get("VERTEX_AI_API_KEY"), + vertex_project=project, + vertex_location=location, + ) + + +def _image_data_uri(text: str, font_size: int) -> str: + url: Final = f"https://dummyjson.com/image/800x300/ffffff/000000?text={quote(text)}&fontSize={font_size}" + response: Final = httpx.get(url, timeout=30, follow_redirects=True) + response.raise_for_status() + raw_content_type: Final = cast(object, response.headers.get("content-type", "image/png")) + content_type: Final = raw_content_type if isinstance(raw_content_type, str) else "image/png" + media_type: Final = content_type.split(";", 1)[0] + encoded: Final = base64.b64encode(response.content).decode("ascii") + return f"data:{media_type};base64,{encoded}" + + +def _sdk_kwargs( + spec: ProviderSpec, + image_data_uri: str, + options: dict[str, object], +) -> dict[str, object]: + provider_kwargs: Final = ( + {"vertex_project": spec.vertex_project, "vertex_location": spec.vertex_location} + if spec.name == "vertex_ai" + else {} + ) + return { + "model": spec.model, + "document": {"type": "image_url", "image_url": image_data_uri}, + **options, + **provider_kwargs, + } + + +def _record_case(spec: ProviderSpec, root: Path, sdk_kwargs: dict[str, object]) -> RecorderResult: + cache: Final = JsonFileCache(root / spec.name) + with _recording_provider(spec=spec, sdk_kwargs=sdk_kwargs, cache=cache) as recorder: + ocr_call: Final = cast(Callable[..., object], litellm.ocr) + ocr_call(api_base=recorder.url, api_key=spec.api_key, **sdk_kwargs) + result: Final = recorder.take_result() + + if not result.cache_hit: + cache.put( + _fixture_cache_key(spec.name, result.fixture.request.provider_request), + result.fixture.model_dump(mode="json"), + ) + return result + + +def _generate(specs: tuple[ProviderSpec, ...], root: Path, examples: int) -> None: + @settings(max_examples=examples, deadline=None, derandomize=True) + @given(text=_TEXT, font_size=st.integers(min_value=12, max_value=36), options=_OPTIONS) + def generate_case(text: str, font_size: int, options: dict[str, object]) -> None: + image_data_uri: Final = _image_data_uri(text, font_size) + for spec in specs: + _generate_provider_case(spec, root, image_data_uri, options) + + generate_case() + + +def _generate_provider_case( + spec: ProviderSpec, + root: Path, + image_data_uri: str, + options: dict[str, object], +) -> None: + sdk_kwargs: Final = _sdk_kwargs(spec, image_data_uri, options) + result: Final = _record_case(spec, root, sdk_kwargs) + state: Final = "cached" if result.cache_hit else "recorded" + LOGGER.info("%s %s %s", state, spec.name, result.fixture.request.provider_request.path) + + +def _parse_args() -> GeneratorArgs: + parser: Final = argparse.ArgumentParser() + parser.add_argument("--provider", action="append", choices=PROVIDER_NAMES) + parser.add_argument("--examples", type=int, default=4) + parser.add_argument("--fixture-dir", type=Path) + namespace: Final = parser.parse_args() + providers: Final = cast(list[str] | None, namespace.provider) + return GeneratorArgs( + providers=tuple(providers) if providers else PROVIDER_NAMES, + examples=cast(int, namespace.examples), + fixture_dir=cast(Path | None, namespace.fixture_dir), + ) + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(message)s") + load_dotenv() + args: Final = _parse_args() + root: Final = ( + args.fixture_dir or Path(os.environ.get(FIXTURE_DIR_ENV, Path(__file__).with_name(".fixtures"))).expanduser() + ) + specs: Final = _provider_specs(args.providers) + if not specs: + raise SystemExit("No selected provider has the required credentials") + _generate(specs, root, args.examples) + + +if __name__ == "__main__": + main() diff --git a/tests/test_litellm/ocr/test_sdk_parity.py b/tests/test_litellm/ocr/test_sdk_parity.py index f543122c125..d587ce99281 100644 --- a/tests/test_litellm/ocr/test_sdk_parity.py +++ b/tests/test_litellm/ocr/test_sdk_parity.py @@ -1,46 +1,33 @@ from __future__ import annotations import asyncio -import json -import os -import queue -import subprocess import sys -import threading -from collections.abc import Callable, Coroutine, Generator -from contextlib import contextmanager -from dataclasses import dataclass +from collections.abc import Callable, Coroutine from enum import Enum -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from typing import Final, Protocol, cast import pytest from pydantic import BaseModel, ConfigDict, TypeAdapter -MODEL: Final = "mistral/mistral-ocr-latest" +from tests.test_litellm.ocr.fixture_models import OcrFixture, OcrFixtureResponse, ProviderWireRequest +from tests.test_litellm.parity.compare import assert_parity +from tests.test_litellm.parity.models import ( + CapturedRequest, + ExceptionReport, + NativeEvidence, + ParityTrace, + ReplayResponse, + SDKOutput, + SDKReport, +) +from tests.test_litellm.parity.replay import replay_json_response +from tests.test_litellm.parity.runner import PythonScriptRunner, run_execution + API_KEY: Final = "test-key" PYTHON_HTTP_SENTINEL: Final = "python-ocr-parity-fallback" -JSON_OBJECT: Final = TypeAdapter(dict[str, object]) GIL_STATS: Final = TypeAdapter(dict[str, int]) -STATIC_OCR_RESPONSE: Final[dict[str, object]] = { - "pages": [ - { - "index": 0, - "markdown": "# Static OCR\n\nProvider fixture text.", - "dimensions": {"dpi": 200, "height": 1000, "width": 800}, - "blocks": [{"type": "title", "content": "Static OCR"}], - "header": "Fixture header", - "footer": "Fixture footer", - } - ], - "model": "mistral-ocr-static", - "document_annotation": {"language": "en"}, - "usage_info": {"pages_processed": 1, "credits": 0.25, "provider_extra": "preserved"}, - "top_level_extra": "dropped by both implementations", -} - class SDKRoute(str, Enum): OCR = "ocr" @@ -54,50 +41,6 @@ class SDKInput(BaseModel): kwargs: dict[str, object] -class ExceptionReport(BaseModel): - model_config = ConfigDict(frozen=True) - - class_name: str - status_code: int | None - message: str - - -class SDKReport(BaseModel): - model_config = ConfigDict(frozen=True) - - rust_enabled: bool - native_callable_loaded: bool - native_handled_case: bool - response_type: str | None - response_json: dict[str, object] | None - exception_json: ExceptionReport | None - - -class ProviderRequest(BaseModel): - model_config = ConfigDict(frozen=True) - - method: str - path: str - authorization: str | None - content_type: str | None - parity_case: str | None - body: dict[str, object] - user_agent: str | None - - -@dataclass(frozen=True, slots=True) -class Execution: - report: SDKReport - request: ProviderRequest - - -@dataclass(frozen=True, slots=True) -class CallOutcome: - response_type: str | None - response_json: dict[str, object] | None - exception_json: ExceptionReport | None - - class _NativeBridge(Protocol): ocr: object aocr: object @@ -105,76 +48,6 @@ class _NativeBridge(Protocol): def gil_stats(self) -> object: ... -class _StaticOcrProvider(ThreadingHTTPServer): - daemon_threads = True - - def __init__(self) -> None: - super().__init__(("127.0.0.1", 0), _StaticOcrHandler) - self.requests: queue.Queue[ProviderRequest] = queue.Queue() - self.response_body: bytes = json.dumps(STATIC_OCR_RESPONSE, sort_keys=True, separators=(",", ":")).encode() - - @property - def url(self) -> str: - return f"http://127.0.0.1:{self.server_address[1]}" - - def take_single_request(self) -> ProviderRequest: - try: - first: Final = self.requests.get(timeout=5) - except queue.Empty as error: - raise AssertionError("expected one provider request, received none") from error - try: - extra: Final = self.requests.get_nowait() - except queue.Empty: - return first - raise AssertionError(f"expected one provider request, received an extra request: {extra.model_dump_json()}") - - -class _StaticOcrHandler(BaseHTTPRequestHandler): - protocol_version = "HTTP/1.1" - - def do_POST(self) -> None: - provider: Final = self.server - assert isinstance(provider, _StaticOcrProvider) - length: Final = int(self.headers.get("content-length") or "0") - body: Final = JSON_OBJECT.validate_json(self.rfile.read(length)) - content_type_header: Final = self.headers.get("content-type") - content_type: Final = content_type_header.split(";", 1)[0].lower() if content_type_header else None - provider.requests.put( - ProviderRequest( - method=self.command, - path=self.path, - authorization=self.headers.get("authorization"), - content_type=content_type, - parity_case=self.headers.get("x-parity-case"), - body=body, - user_agent=self.headers.get("user-agent"), - ) - ) - status: Final = 200 if self.path == "/v1/ocr" else 404 - response_body: Final = provider.response_body if status == 200 else b'{"error":"unexpected path"}' - self.send_response(status) - self.send_header("content-type", "application/json") - self.send_header("content-length", str(len(response_body))) - self.end_headers() - self.wfile.write(response_body) - - def log_message(self, format: str, *args: object) -> None: - return - - -@contextmanager -def _static_provider() -> Generator[_StaticOcrProvider]: - server: Final = _StaticOcrProvider() - thread: Final = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - yield server - finally: - server.shutdown() - server.server_close() - thread.join(timeout=5) - - def _qualified_name(value: object) -> str: value_type: Final = type(value) return f"{value_type.__module__}.{value_type.__qualname__}" @@ -191,19 +64,15 @@ def _gil_release_count(native_bridge: _NativeBridge) -> int: return stats["releases"] -def _response_outcome(response: BaseModel) -> CallOutcome: - response_json: Final = JSON_OBJECT.validate_python(response.model_dump(mode="json")) - return CallOutcome( +def _response_trace(response: BaseModel) -> ParityTrace: + output: Final = SDKOutput( response_type=_qualified_name(response), - response_json=response_json, - exception_json=None, + response_json=response.model_dump(mode="json"), ) + return ParityTrace(outputs=(output,), exception=None) -def _capture_sdk_call( - sdk_input: SDKInput, - mock_url: str, -) -> CallOutcome: +def _capture_sdk_call(sdk_input: SDKInput, mock_url: str) -> ParityTrace: import litellm from litellm.llms.base_llm.ocr.transformation import OCRResponse @@ -217,26 +86,21 @@ def _capture_sdk_call( try: if sdk_input.route is SDKRoute.OCR: sync_route: Final = cast(Callable[..., OCRResponse], litellm.ocr) - return _response_outcome(sync_route(**call_kwargs)) - + return _response_trace(sync_route(**call_kwargs)) async_route: Final = cast(Callable[..., Coroutine[object, object, OCRResponse]], litellm.aocr) - return _response_outcome(asyncio.run(async_route(**call_kwargs))) + return _response_trace(asyncio.run(async_route(**call_kwargs))) except Exception as error: - return CallOutcome( - response_type=None, - response_json=None, - exception_json=_exception_report(error), - ) + return ParityTrace(outputs=(), exception=_exception_report(error)) -def _native_sdk_report(outcome: CallOutcome, native_handled_case: bool) -> SDKReport: +def _native_sdk_report(trace: ParityTrace, native_handled_case: bool) -> SDKReport: return SDKReport( - rust_enabled=True, - native_callable_loaded=True, - native_handled_case=native_handled_case, - response_type=outcome.response_type, - response_json=outcome.response_json, - exception_json=outcome.exception_json, + trace=trace, + native=NativeEvidence( + rust_enabled=True, + native_callable_loaded=True, + native_handled_case=native_handled_case, + ), ) @@ -246,14 +110,13 @@ def _execute_sdk_case(sdk_input: SDKInput, mock_url: str) -> SDKReport: rust_enabled: Final = rust_ocr_enabled() if not rust_enabled: - outcome: Final = _capture_sdk_call(sdk_input, mock_url) return SDKReport( - rust_enabled=False, - native_callable_loaded=False, - native_handled_case=False, - response_type=outcome.response_type, - response_json=outcome.response_json, - exception_json=outcome.exception_json, + trace=_capture_sdk_call(sdk_input, mock_url), + native=NativeEvidence( + rust_enabled=False, + native_callable_loaded=False, + native_handled_case=False, + ), ) raw_native_bridge: Final = get_native_bridge() @@ -267,92 +130,57 @@ def _execute_sdk_case(sdk_input: SDKInput, mock_url: str) -> SDKReport: raise RuntimeError(f"native {sdk_input.route.value} callable is unavailable") if sdk_input.route is SDKRoute.AOCR: - async_outcome: Final = _capture_sdk_call(sdk_input, mock_url) - return _native_sdk_report(async_outcome, native_handled_case=True) + async_trace: Final = _capture_sdk_call(sdk_input, mock_url) + return _native_sdk_report(async_trace, native_handled_case=bool(async_trace.outputs)) before_gil_releases: Final = _gil_release_count(native_bridge) - sync_outcome: Final = _capture_sdk_call(sdk_input, mock_url) + trace: Final = _capture_sdk_call(sdk_input, mock_url) after_gil_releases: Final = _gil_release_count(native_bridge) - native_handled_case: Final = after_gil_releases == before_gil_releases + 1 - return _native_sdk_report(sync_outcome, native_handled_case) + return _native_sdk_report(trace, native_handled_case=after_gil_releases == before_gil_releases + 1) -def _run_execution( - case_file: Path, - report_file: Path, - provider: _StaticOcrProvider, - rust_enabled: bool, -) -> Execution: - env: Final = { - **os.environ, - "LITELLM_USE_RUST_OCR": "1" if rust_enabled else "0", - "LITELLM_USER_AGENT": PYTHON_HTTP_SENTINEL, - } - completed: Final = subprocess.run( - [ - sys.executable, - str(Path(__file__).resolve()), - str(case_file), - provider.url, - str(report_file), - ], - capture_output=True, - text=True, - env=env, - timeout=60, - check=False, - ) - assert completed.returncode == 0, ( - f"SDK subprocess failed with exit code {completed.returncode}\n" - f"stdout:\n{completed.stdout}\n" - f"stderr:\n{completed.stderr}" - ) - report: Final = SDKReport.model_validate_json(report_file.read_text(encoding="utf-8")) - return Execution(report=report, request=provider.take_single_request()) +def _replay_response(response: OcrFixtureResponse) -> ReplayResponse: + return ReplayResponse(status_code=response.status_code, headers=response.headers, body=response.body) + + +def _provider_wire_request(request: CapturedRequest) -> ProviderWireRequest: + return ProviderWireRequest(method=request.method, path=request.path, body=request.body) @pytest.mark.parametrize("route", tuple(SDKRoute), ids=tuple(route.value for route in SDKRoute)) -def test_static_mistral_ocr_sdk_parity(route: SDKRoute, tmp_path: Path) -> None: - sdk_input: Final = SDKInput( - route=route, - kwargs={ - "model": MODEL, - "document": {"type": "document_url", "document_url": "https://example.com/static.pdf"}, - "pages": [0, 1], - "include_image_base64": True, - "include_blocks": True, - "table_format": "html", - "id": "static-ocr-parity", - }, - ) - case_file: Final = tmp_path / f"{route.value}-sdk-input.json" +def test_recorded_ocr_sdk_parity(ocr_fixture: OcrFixture, route: SDKRoute, tmp_path: Path) -> None: + sdk_input: Final = SDKInput(route=route, kwargs=ocr_fixture.request.sdk_kwargs) + case_file: Final = tmp_path / f"{ocr_fixture.request.provider}-{route.value}-sdk-input.json" case_file.write_text(sdk_input.model_dump_json(indent=2), encoding="utf-8") - with _static_provider() as python_mock, _static_provider() as rust_mock: - python: Final = _run_execution( + expected_request: Final = ocr_fixture.request.provider_request + runner: Final = PythonScriptRunner( + entrypoint=Path(__file__), + rust_env_var="LITELLM_USE_RUST_OCR", + python_user_agent=PYTHON_HTTP_SENTINEL, + ) + + with ( + replay_json_response(expected_request.path, _replay_response(ocr_fixture.response)) as python_provider, + replay_json_response(expected_request.path, _replay_response(ocr_fixture.response)) as rust_provider, + ): + python: Final = run_execution( + runner, case_file, tmp_path / f"{route.value}-python-report.json", - python_mock, - False, + python_provider, + rust_enabled=False, ) - rust: Final = _run_execution( + rust: Final = run_execution( + runner, case_file, tmp_path / f"{route.value}-rust-report.json", - rust_mock, - True, + rust_provider, + rust_enabled=True, ) - assert python.report.rust_enabled is False - assert rust.report.rust_enabled is True - assert rust.report.native_callable_loaded is True - assert rust.report.native_handled_case is True - assert python.request.user_agent == PYTHON_HTTP_SENTINEL - assert rust.request.user_agent != PYTHON_HTTP_SENTINEL - assert rust.request.model_dump(exclude={"user_agent"}) == python.request.model_dump(exclude={"user_agent"}) - assert rust.report.response_type == python.report.response_type - assert rust.report.response_json == python.report.response_json - assert rust.report.exception_json == python.report.exception_json - assert python.report.exception_json is None - assert python.report.response_json is not None + assert_parity(python, rust, PYTHON_HTTP_SENTINEL) + assert tuple(_provider_wire_request(request) for request in python.requests) == (expected_request,) + assert tuple(_provider_wire_request(request) for request in rust.requests) == (expected_request,) def _child_main() -> None: diff --git a/tests/test_litellm/parity/__init__.py b/tests/test_litellm/parity/__init__.py new file mode 100644 index 00000000000..b72fc0dc894 --- /dev/null +++ b/tests/test_litellm/parity/__init__.py @@ -0,0 +1,3 @@ +import pytest + +pytest.register_assert_rewrite("tests.test_litellm.parity.compare") diff --git a/tests/test_litellm/parity/compare.py b/tests/test_litellm/parity/compare.py new file mode 100644 index 00000000000..323524c1164 --- /dev/null +++ b/tests/test_litellm/parity/compare.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Final, cast + +from pydantic import BaseModel + +from tests.test_litellm.parity.models import CapturedRequest, Execution, ParityTrace + + +def _stable_request(request: CapturedRequest) -> dict[str, object]: + return request.model_dump(mode="json", exclude={"user_agent"}) + + +def assert_parity(python: Execution, rust: Execution, python_user_agent: str) -> None: + assert python.report.native.rust_enabled is False + assert rust.report.native.rust_enabled is True + assert rust.report.native.native_callable_loaded is True + assert rust.report.native.native_handled_case is True + assert tuple(request.user_agent for request in python.requests) == (python_user_agent,) * len(python.requests) + assert all(request.user_agent != python_user_agent for request in rust.requests) + assert tuple(_stable_request(request) for request in rust.requests) == tuple( + _stable_request(request) for request in python.requests + ) + assert rust.report.trace == python.report.trace + assert python.report.trace.exception is None + assert python.report.trace.outputs + + +def _short(value: object) -> str: + rendered: Final = repr(value) + return rendered if len(rendered) <= 240 else f"{rendered[:237]}..." + + +def _diff(left: object, right: object, path: str = "$") -> tuple[str, ...]: + if isinstance(left, BaseModel) and isinstance(right, BaseModel): + return _diff(left.model_dump(mode="json"), right.model_dump(mode="json"), path) + if isinstance(left, Mapping) and isinstance(right, Mapping): + left_mapping: Final = cast(Mapping[str, object], left) + right_mapping: Final = cast(Mapping[str, object], right) + left_keys: Final = frozenset(left_mapping) + right_keys: Final = frozenset(right_mapping) + missing: Final = tuple(f"{path}.{key}: missing from left" for key in sorted(right_keys - left_keys)) + extra: Final = tuple(f"{path}.{key}: missing from right" for key in sorted(left_keys - right_keys)) + shared: Final = tuple( + line + for key in sorted(left_keys & right_keys) + for line in _diff(left_mapping[key], right_mapping[key], f"{path}.{key}") + ) + return (*missing, *extra, *shared) + if ( + isinstance(left, Sequence) + and not isinstance(left, (str, bytes)) + and isinstance(right, Sequence) + and not isinstance(right, (str, bytes)) + ): + left_sequence: Final = cast(Sequence[object], left) + right_sequence: Final = cast(Sequence[object], right) + length_diff: Final = ( + (f"{path}: lengths differ ({len(left_sequence)} != {len(right_sequence)})",) + if len(left_sequence) != len(right_sequence) + else () + ) + item_diff: Final = tuple( + line + for index, (left_item, right_item) in enumerate(zip(left_sequence, right_sequence)) + for line in _diff(left_item, right_item, f"{path}[{index}]") + ) + return (*length_diff, *item_diff) + left_value: Final = cast(object, left) + right_value: Final = cast(object, right) + return () if left_value == right_value else (f"{path}: {_short(left_value)} != {_short(right_value)}",) + + +def parity_comparison(left: object, right: object) -> list[str] | None: + if not isinstance(left, (ParityTrace, CapturedRequest)) or type(left) is not type(right): + return None + differences: Final = _diff(left, right) + return [f"Comparing {type(left).__name__} values:", *(f" {line}" for line in differences)] diff --git a/tests/test_litellm/parity/models.py b/tests/test_litellm/parity/models.py new file mode 100644 index 00000000000..f68cc690c71 --- /dev/null +++ b/tests/test_litellm/parity/models.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + + +class ExceptionReport(BaseModel): + model_config = ConfigDict(frozen=True) + + class_name: str + status_code: int | None + message: str + + +class SDKOutput(BaseModel): + model_config = ConfigDict(frozen=True) + + response_type: str + response_json: dict[str, object] + + +class ParityTrace(BaseModel): + model_config = ConfigDict(frozen=True) + + outputs: tuple[SDKOutput, ...] + exception: ExceptionReport | None + + +class NativeEvidence(BaseModel): + model_config = ConfigDict(frozen=True) + + rust_enabled: bool + native_callable_loaded: bool + native_handled_case: bool + + +class SDKReport(BaseModel): + model_config = ConfigDict(frozen=True) + + trace: ParityTrace + native: NativeEvidence + + +class CapturedRequest(BaseModel): + model_config = ConfigDict(frozen=True) + + method: str + path: str + authorization: str | None + content_type: str | None + parity_case: str | None + body: dict[str, object] + user_agent: str | None + + +class Execution(BaseModel): + model_config = ConfigDict(frozen=True) + + report: SDKReport + requests: tuple[CapturedRequest, ...] + + +class ReplayResponse(BaseModel): + model_config = ConfigDict(frozen=True) + + status_code: int + headers: dict[str, str] + body: dict[str, object] diff --git a/tests/test_litellm/parity/pytest_plugin.py b/tests/test_litellm/parity/pytest_plugin.py new file mode 100644 index 00000000000..1d8782c4255 --- /dev/null +++ b/tests/test_litellm/parity/pytest_plugin.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +import pytest + +from tests.test_litellm.parity.compare import parity_comparison + + +def pytest_assertrepr_compare(config: pytest.Config, op: str, left: object, right: object) -> list[str] | None: + if op != "==": + return None + return parity_comparison(left, right) diff --git a/tests/test_litellm/parity/replay.py b/tests/test_litellm/parity/replay.py new file mode 100644 index 00000000000..3f2998593ac --- /dev/null +++ b/tests/test_litellm/parity/replay.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +import json +import queue +import threading +from collections.abc import Generator +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final + +from pydantic import TypeAdapter + +from tests.test_litellm.parity.models import CapturedRequest, ReplayResponse + +JSON_OBJECT: Final = TypeAdapter(dict[str, object]) + + +class JsonReplayServer(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self, expected_path: str, response: ReplayResponse) -> None: + super().__init__(("127.0.0.1", 0), _JsonReplayHandler) + self.expected_path: Final = expected_path + self.response: Final = response + self.response_body: Final = json.dumps(response.body, sort_keys=True, separators=(",", ":")).encode() + self.requests: queue.Queue[CapturedRequest] = queue.Queue() + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.server_address[1]}" + + def take_requests(self) -> tuple[CapturedRequest, ...]: + try: + first: Final = self.requests.get(timeout=5) + except queue.Empty as error: + raise AssertionError("expected at least one provider request, received none") from error + remaining_count: Final = self.requests.qsize() + remaining: Final = tuple(self.requests.get_nowait() for _ in range(remaining_count)) + return (first, *remaining) + + +class _JsonReplayHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: + provider: Final = self.server + assert isinstance(provider, JsonReplayServer) + length: Final = int(self.headers.get("content-length") or "0") + body: Final = JSON_OBJECT.validate_json(self.rfile.read(length)) + content_type_header: Final = self.headers.get("content-type") + content_type: Final = content_type_header.split(";", 1)[0].lower() if content_type_header else None + provider.requests.put( + CapturedRequest( + method=self.command, + path=self.path, + authorization=self.headers.get("authorization"), + content_type=content_type, + parity_case=self.headers.get("x-parity-case"), + body=body, + user_agent=self.headers.get("user-agent"), + ) + ) + matched: Final = self.path == provider.expected_path + status_code: Final = provider.response.status_code if matched else 404 + response_body: Final = provider.response_body if matched else b'{"error":"unexpected path"}' + response_headers: Final = provider.response.headers if matched else {"content-type": "application/json"} + self.send_response(status_code) + for name, value in response_headers.items(): + if name.lower() not in {"content-length", "transfer-encoding", "content-encoding"}: + self.send_header(name, value) + self.send_header("content-length", str(len(response_body))) + self.end_headers() + self.wfile.write(response_body) + + def log_message(self, format: str, *args: object) -> None: + return + + +@contextmanager +def replay_json_response(expected_path: str, response: ReplayResponse) -> Generator[JsonReplayServer]: + server: Final = JsonReplayServer(expected_path=expected_path, response=response) + thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) diff --git a/tests/test_litellm/parity/runner.py b/tests/test_litellm/parity/runner.py new file mode 100644 index 00000000000..a136d9901ee --- /dev/null +++ b/tests/test_litellm/parity/runner.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import os +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +from tests.test_litellm.parity.models import Execution, SDKReport +from tests.test_litellm.parity.replay import JsonReplayServer + + +@dataclass(frozen=True, slots=True) +class PythonScriptRunner: + entrypoint: Path + rust_env_var: str + python_user_agent: str + + def command(self, case_file: Path, provider_url: str, report_file: Path) -> tuple[str, ...]: + return ( + sys.executable, + str(self.entrypoint.resolve()), + str(case_file), + provider_url, + str(report_file), + ) + + +def run_execution( + runner: PythonScriptRunner, + case_file: Path, + report_file: Path, + provider: JsonReplayServer, + rust_enabled: bool, +) -> Execution: + env: Final = { + **os.environ, + runner.rust_env_var: "1" if rust_enabled else "0", + "LITELLM_USER_AGENT": runner.python_user_agent, + } + completed: Final = subprocess.run( + runner.command(case_file, provider.url, report_file), + capture_output=True, + text=True, + env=env, + timeout=60, + check=False, + ) + assert completed.returncode == 0, ( + f"SDK subprocess failed with exit code {completed.returncode}\n" + f"stdout:\n{completed.stdout}\n" + f"stderr:\n{completed.stderr}" + ) + report: Final = SDKReport.model_validate_json(report_file.read_text(encoding="utf-8")) + return Execution(report=report, requests=provider.take_requests()) diff --git a/tests/test_litellm/parity/test_parity.py b/tests/test_litellm/parity/test_parity.py new file mode 100644 index 00000000000..af7e0f95038 --- /dev/null +++ b/tests/test_litellm/parity/test_parity.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from typing import Final + +from tests.test_litellm.parity.compare import parity_comparison +from tests.test_litellm.parity.models import ParityTrace, SDKOutput + + +def test_parity_comparison_reports_first_nested_output_difference() -> None: + python: Final = ParityTrace( + outputs=(SDKOutput(response_type="Chunk", response_json={"choices": [{"delta": {"content": "a"}}]}),), + exception=None, + ) + rust: Final = ParityTrace( + outputs=(SDKOutput(response_type="Chunk", response_json={"choices": [{"delta": {"content": "b"}}]}),), + exception=None, + ) + + explanation: Final = parity_comparison(rust, python) + + assert explanation is not None + assert explanation == [ + "Comparing ParityTrace values:", + " $.outputs[0].response_json.choices[0].delta.content: 'b' != 'a'", + ] diff --git a/tests/test_litellm/test_json_fs_cache.py b/tests/test_litellm/test_json_fs_cache.py new file mode 100644 index 00000000000..f2fc7c8d628 --- /dev/null +++ b/tests/test_litellm/test_json_fs_cache.py @@ -0,0 +1,17 @@ +from pathlib import Path +from typing import Final + +from tests.test_litellm._json_fs_cache import JsonFileCache + + +def test_json_file_cache_is_content_addressed_and_recursive(tmp_path: Path) -> None: + key: Final = {"method": "POST", "body": {"model": "test-model", "pages": [0]}} + reordered_key: Final = {"body": {"pages": [0], "model": "test-model"}, "method": "POST"} + value: Final = {"request": key, "response": {"status_code": 200}} + cache: Final = JsonFileCache(tmp_path / "provider") + + stored_path: Final = cache.put(key, value) + + assert stored_path.name == cache.path_for(reordered_key).name + assert cache.get(reordered_key) == value + assert JsonFileCache(tmp_path).values() == (value,) diff --git a/uv.lock b/uv.lock index 27be919eea1..125640a112f 100644 --- a/uv.lock +++ b/uv.lock @@ -3373,6 +3373,98 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, ] +[[package]] +name = "hypothesis" +version = "6.165.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5c/e2/0fad246d2b6330e1f78479bfc566b5c22be82aee8a865cde9a08f648487d/hypothesis-6.165.10.tar.gz", hash = "sha256:68b45e09834cd80523cb1eb274463073c7a9af4e4ef7cff34d9615f355572d32", size = 503703, upload-time = "2026-08-16T22:56:15.404Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/c1/9a9538e6d185baf5cc7f15bc3b76e08efbb3de4b3c782f234356449c0dd7/hypothesis-6.165.10-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f839d29d0cc12048cf073d88ca4fdf94d420bc2b8afd69641ff6d496422ccd4f", size = 783243, upload-time = "2026-08-16T22:55:44.058Z" }, + { url = "https://files.pythonhosted.org/packages/a1/30/b70d9d79e871a75cbdeccd9067f20ecdb9eb2a1dfa03c630be3ad13b8b30/hypothesis-6.165.10-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:e10858f57ed0e74baa04393845f469fe8ad502c16ece4499bef7700c575611bd", size = 778815, upload-time = "2026-08-16T22:55:46.948Z" }, + { url = "https://files.pythonhosted.org/packages/db/52/6f0a9b7aab24b0635e2238f3fbddea5b54b17879ac813df42a3cc3384c5c/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76a7be86d986223b9f1bdb7e7cbcdb048649901fdb956c598ef73bdab1786cd5", size = 1108009, upload-time = "2026-08-16T22:54:53.082Z" }, + { url = "https://files.pythonhosted.org/packages/f6/06/8d0d4e11ff02350d09ec9f9e90af354158e59e16a8907ba5199a4ff2d7e8/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:717aea574e0e5edba2868aa66b1caae335d8f1ad3fb29f01dd6502953fa823a1", size = 1136596, upload-time = "2026-08-16T22:54:54.443Z" }, + { url = "https://files.pythonhosted.org/packages/59/dd/01a1e440f2e38dc1ccf5d597af5b8a0bee5f21b674c99c123b5554de9690/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4334058033e0214475f019e15492a50f3854fe8728cf51fe25c6191a2c3f8e52", size = 1135234, upload-time = "2026-08-16T22:55:08.911Z" }, + { url = "https://files.pythonhosted.org/packages/7d/18/8a26c24d3d9db20265f39df341ab265858c094e209571e3179cf237935f4/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2abb50cf1cf77d721de0a24c3f99d9c4ffdeb2cbd1e12aebb5a7a93e2b6b6d1f", size = 1157528, upload-time = "2026-08-16T22:56:02.159Z" }, + { url = "https://files.pythonhosted.org/packages/ea/8e/ce3c829b1937402d7944420ca26a05a0c8563e894dcff03d34ffa279d306/hypothesis-6.165.10-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:3de69aa8b924b400291a3cc42aaf78e6ab65c905a3e7e1a5dc39d95ef1b428cb", size = 1112870, upload-time = "2026-08-16T22:54:55.919Z" }, + { url = "https://files.pythonhosted.org/packages/f2/1b/4c4926d6c9a2b5d7cc090cc1e91219d6796102aa2a2c4b8f961c939e60b5/hypothesis-6.165.10-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5841331c504e02d7c334591681cb8587cdd59dee7e149db6d3db8e3f9e9f02eb", size = 1149683, upload-time = "2026-08-16T22:55:30.567Z" }, + { url = "https://files.pythonhosted.org/packages/cb/f9/df24eb28412f82465e2b7707f0ff1ec274d580bce389d4d9156617dc7bba/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2d0e0f8263d34dd8fa3b39eaa9a50bba56a8470b3dd9ebf6672d10840abe063e", size = 1283402, upload-time = "2026-08-16T22:54:18.054Z" }, + { url = "https://files.pythonhosted.org/packages/4d/07/c2b2a761300cf60b90ccebba4328175331e67d34f4fbd39429a7ddcdce49/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:0c4e6869817c3cfdf5a2b4d348497b95159bdecb3365be732c9b8570e36a4eef", size = 1409948, upload-time = "2026-08-16T22:54:22.343Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ec/1c2bf1acdd0e273d81f833f85caf0ae5423db68a783554992fca36e6c541/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:9f07ae36c3b093e13687a894e79fe69e98a94c0b67fef656c575247682218143", size = 1265023, upload-time = "2026-08-16T22:54:41.402Z" }, + { url = "https://files.pythonhosted.org/packages/3d/a8/7f984908b7391160c7801b84e51ca8e4ba88c89e8d8811aa1aa7c03de73c/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:aff1f584c9538e8979cd180b1d70bf99bc16be19d4666414f49e5942b21a4f2c", size = 1282698, upload-time = "2026-08-16T22:56:06.998Z" }, + { url = "https://files.pythonhosted.org/packages/48/78/3a5d91c2d0250521736c42dfa2402b75049bc5fe2fb716c10bc84bb91ed1/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1f2c4db25fb8ec1a16a8dba580666337b8ffb1887c4cf1750cc954313897cef7", size = 1324816, upload-time = "2026-08-16T22:54:46.675Z" }, + { url = "https://files.pythonhosted.org/packages/6f/99/27450763853a034bca1574d3e0a315164b33ff49c3862df6872dda45e25e/hypothesis-6.165.10-cp310-abi3-win32.whl", hash = "sha256:b33dc30170a7402e03c180f2c5ef69dc077152f35b91621e9cebcde9c7d71746", size = 669039, upload-time = "2026-08-16T22:55:11.962Z" }, + { url = "https://files.pythonhosted.org/packages/2c/fc/ff2988b72b5705ad9ca500444bf3f43e3c2f41edfa034bbfeb23b215791a/hypothesis-6.165.10-cp310-abi3-win_amd64.whl", hash = "sha256:e9f924aa610c0618445e1e8738c822c3190ce2a2699a0cb48ec3a351a96761f2", size = 675213, upload-time = "2026-08-16T22:55:01.697Z" }, + { url = "https://files.pythonhosted.org/packages/c5/8b/821810d36f78d9d9421cd2c5d9d36983b45bb3575c3086276cc5c76f9f73/hypothesis-6.165.10-cp310-abi3-win_arm64.whl", hash = "sha256:1d305448e9bd8e2f4f3cea0eafd809efdaab4e998a0019bc615650c8463e42f1", size = 673537, upload-time = "2026-08-16T22:54:47.898Z" }, + { url = "https://files.pythonhosted.org/packages/26/61/5e89268ce03317fb9f82449a1b3efd9e599dee090288fd0cf7586c532fb1/hypothesis-6.165.10-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:73e6df02a6a62f8045b511c272f894d08e56d174504c793c9effcbc6778051a8", size = 783959, upload-time = "2026-08-16T22:55:29.078Z" }, + { url = "https://files.pythonhosted.org/packages/e1/e9/f4e0832e81bb53b70cf1712e28c867db64245b32595b594217452e7dbd8d/hypothesis-6.165.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8b20f44773a9ab84400465e318712d8c2ca16418d35b9f80aa27fdf2d690ad10", size = 779684, upload-time = "2026-08-16T22:54:57.698Z" }, + { url = "https://files.pythonhosted.org/packages/77/de/ea072d3359d5678771bed407f80439e8ac7ca905d1031b0372f61bf5746e/hypothesis-6.165.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb8c7d05ea27a093a92b250904095d71d924b6b44e5795a415c1b20c265f0c65", size = 1108540, upload-time = "2026-08-16T22:55:03.282Z" }, + { url = "https://files.pythonhosted.org/packages/28/56/e7c395cdaa3d6c28b944c1c3c516dee50d2b7b3aeafa31874b57009ca51f/hypothesis-6.165.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f4dafd6d6ababfa3b14dd6e5f0378cb7c7d291895a31a40abcbb7cc74f396131", size = 1158089, upload-time = "2026-08-16T22:54:36.205Z" }, + { url = "https://files.pythonhosted.org/packages/b1/49/1c6d2c465b9c5fc3213f1be89be95ba53819ca0130248c484129ccfefb71/hypothesis-6.165.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:fa74636a49fc8077413ce8db3e85f1c4aff880788bb55bda56253118e036fe5b", size = 1284125, upload-time = "2026-08-16T22:54:37.727Z" }, + { url = "https://files.pythonhosted.org/packages/7d/bc/7caf5ac3d0173bd57bd2a5ab854ca49a3664a4309257be1452f81025cc24/hypothesis-6.165.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2b112768cfb67f2b683e53e58c1a33d27811aacf60c942b8eb74635e469a73f6", size = 1325082, upload-time = "2026-08-16T22:54:38.952Z" }, + { url = "https://files.pythonhosted.org/packages/70/99/9d844330f570d6a4f127a683eab1e78c8263e6e72b16189f3534fa6bf6de/hypothesis-6.165.10-cp310-cp310-win_amd64.whl", hash = "sha256:56cb8c9055e50545fe6e3e5a560ec25a724673b2e4051f3c24d44e3ebc35dd72", size = 675082, upload-time = "2026-08-16T22:54:29.872Z" }, + { url = "https://files.pythonhosted.org/packages/ed/c2/b9546ace11f241c9c02d389f258cb80c14447a8c885771c9f1f0bc1d85ca/hypothesis-6.165.10-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:592107a0faf6c9c3a63a8dbf13dfb1cbda1cf599b0bc11c953221b00204b9ce1", size = 783716, upload-time = "2026-08-16T22:55:36.624Z" }, + { url = "https://files.pythonhosted.org/packages/37/10/27c2fdd574fd798caf5e91eb51f7834b098f5d840ce733efb3fba79ef86e/hypothesis-6.165.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f9180c362bde06fd05380298ded4e234fbc0d6ede0a864835bfd91c1e24283d5", size = 779507, upload-time = "2026-08-16T22:55:07.633Z" }, + { url = "https://files.pythonhosted.org/packages/5e/b6/70bc23695f3783c4b0486b6cad47b08a20f791db4a3c1b25250add9659fa/hypothesis-6.165.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d623801ae3dcd97b77b983400ef3d48bf976648e4efff19929175322eaae074d", size = 1108406, upload-time = "2026-08-16T22:55:39.653Z" }, + { url = "https://files.pythonhosted.org/packages/71/4c/32e200bd7a352af4b7f4e3729aaa4cd002cb5fe8c4c6aef5599d0019f152/hypothesis-6.165.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20f6236cfb90b7817bb1a6a087589ca4aa46d73170f0dd62963952ed5dadc589", size = 1157850, upload-time = "2026-08-16T22:55:24.394Z" }, + { url = "https://files.pythonhosted.org/packages/03/a5/8efc2a9a484822efc0d0da466f50094e0f2c068187faaf33831fc905873e/hypothesis-6.165.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ad0764730e8e3421601c2cc7e1f054a9206c60ea0917165d8d9193dc453f34f1", size = 1283704, upload-time = "2026-08-16T22:54:27.279Z" }, + { url = "https://files.pythonhosted.org/packages/46/2a/90cc8d7463929c04786f29600de45f3227c12fa9bed1d5b7ce319b05e1c9/hypothesis-6.165.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:10d9a650a4666b0914831f769703d36140ed8039fd19bf9b71f615b8541eccf2", size = 1325077, upload-time = "2026-08-16T22:55:16.561Z" }, + { url = "https://files.pythonhosted.org/packages/82/ac/bc16faba4b42883e3d290bfaceff51e258b63fbbdf789bf9fe88df1ce537/hypothesis-6.165.10-cp311-cp311-win_amd64.whl", hash = "sha256:5671d2b2bf83bd4b6f02e55b32d432506eff5358c82f39b460a849ce19a2666e", size = 674920, upload-time = "2026-08-16T22:55:42.613Z" }, + { url = "https://files.pythonhosted.org/packages/e9/45/cde4f78afe2b9e29caecf38319eedc1deb76aebcacbdd128e03cbb2511c3/hypothesis-6.165.10-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:637445c1593a2a9d1024fda50082f07bb56baedda78d90a25f64b8111727ef94", size = 784835, upload-time = "2026-08-16T22:54:45.429Z" }, + { url = "https://files.pythonhosted.org/packages/7f/81/847f30b81cbfd07607296b3ce43067cf4f80799bd9244167f587de9c8081/hypothesis-6.165.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:713f4ce4e82c26b53031f139de959bc9e8b54d3995aa824b89bbdf8229df2a45", size = 776419, upload-time = "2026-08-16T22:55:33.633Z" }, + { url = "https://files.pythonhosted.org/packages/04/66/4c71c5be7a49d84b8c3a9278c1807c4c81181ab5474beb27df9d4c40dc0e/hypothesis-6.165.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9ff356e97e3ab09db07c8b675efa67340103874a0bae7465acb83dad7a35f7f", size = 1106830, upload-time = "2026-08-16T22:55:10.389Z" }, + { url = "https://files.pythonhosted.org/packages/e3/c4/e2cbd2810e79f7a452a8ea9f6c6438ee718ce938d8cc12252cf0b36a81d3/hypothesis-6.165.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a380bc99aa3b035e6a95a2201bf792d4082a04ca75babcc21849c2d0914bb28", size = 1156952, upload-time = "2026-08-16T22:55:53.35Z" }, + { url = "https://files.pythonhosted.org/packages/a8/8b/794ced36864825492ac3712d5acab5a257b4601e6a9dc2ccdd3937198f87/hypothesis-6.165.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e9acb2c4d9cb532c3fedea74159f7b923c8c036328c9239b4049e7aa073bdd81", size = 1280780, upload-time = "2026-08-16T22:54:34.983Z" }, + { url = "https://files.pythonhosted.org/packages/5d/2d/550525442cdbcc2daf1f9bdd8ba35bcbde63db7c7a22f2ef137fbb49df2f/hypothesis-6.165.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8660572b2d424bf5369ea8990985225f70bd1615b76ecd9c25588a3b9307009f", size = 1324130, upload-time = "2026-08-16T22:55:48.659Z" }, + { url = "https://files.pythonhosted.org/packages/74/59/6caf69dd5fe03499ada94c9cec016bffcc164511c6b93fe680f01209b9ff/hypothesis-6.165.10-cp312-cp312-win_amd64.whl", hash = "sha256:3376f2594763aef14faa519b0fb27cae7ce9eeaab4c69efa07777499110306c9", size = 672337, upload-time = "2026-08-16T22:54:49.11Z" }, + { url = "https://files.pythonhosted.org/packages/b1/fb/c82c5bd92864ffcf319772fedc8c9bf2dbe4ca14baa0fee6e49e67b5ba1c/hypothesis-6.165.10-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:9d77c3be7b429875036ad0f0597c6e5cc6bb17894a4da005e3807de64d2673ad", size = 784726, upload-time = "2026-08-16T22:54:32.371Z" }, + { url = "https://files.pythonhosted.org/packages/0e/b9/3d7acd08506da85557e65147b7f3fca8c47684e33be90bee0acb523920db/hypothesis-6.165.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:490c56b830772b0eca3b4b2cecb3741a1ed26b1d7206a279e1525dbf0aa95ee4", size = 776375, upload-time = "2026-08-16T22:55:13.303Z" }, + { url = "https://files.pythonhosted.org/packages/38/6b/922e8b3f9a706dd89d440b9545d2c6231c65e74da1c1fee3ff36c251b9c4/hypothesis-6.165.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed68e27b8a61e57a3ccdc7c5a14499e00b54dfe223087204d5d40b3b5ef58b6d", size = 1106763, upload-time = "2026-08-16T22:55:06.129Z" }, + { url = "https://files.pythonhosted.org/packages/01/39/f5b9a5d390d4edd1ad472334493ac442963ebeb4daaa74ff4bdac6ef292f/hypothesis-6.165.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6caadcd1afb62630ff5c5ff353626eaa616553a5971295ad6dc2b19ca8a39620", size = 1156778, upload-time = "2026-08-16T22:54:33.824Z" }, + { url = "https://files.pythonhosted.org/packages/b5/5f/5fbe1be4326337fd6acefe2d18ed44007ee1dc1f98fe5b3c0eb22942364d/hypothesis-6.165.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9145fe43ebb22e66672967c3fab411793b226ed776e4fe282271bca6ad3c0bb", size = 1280756, upload-time = "2026-08-16T22:55:54.834Z" }, + { url = "https://files.pythonhosted.org/packages/25/c0/cf6f9e1ef632a1a75694eed0db3a02e6fc75c367a363e94acee52f043c64/hypothesis-6.165.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:79900a9920a0b1d3a626c03a90ac6bf7042e78d46906a565b86a0dbe926f1d96", size = 1323889, upload-time = "2026-08-16T22:55:56.567Z" }, + { url = "https://files.pythonhosted.org/packages/cc/cc/662b94880f260b0a88de1fdcf60fc9984f6e2a796da549542adc10a7bc83/hypothesis-6.165.10-cp313-cp313-win_amd64.whl", hash = "sha256:c01dd04044c472e47193b54f68e84e08d6ebf4f29551885aa959b015f7cd9747", size = 672346, upload-time = "2026-08-16T22:56:03.792Z" }, + { url = "https://files.pythonhosted.org/packages/3f/77/55e020c9c576532ff7d20bf8b1dfa052ecbd5ada1949b02f76c44c966f7e/hypothesis-6.165.10-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:9ccac776b2ca93b324806facd526ccb45da0fd035001c899a35b02c44431e209", size = 784833, upload-time = "2026-08-16T22:55:21.255Z" }, + { url = "https://files.pythonhosted.org/packages/4f/f2/01da2adf829cf549eaddcabb8e8072077fb3d26da4275f4c1e89b2c0af74/hypothesis-6.165.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e5f95f7b622e4171096d92175dda0a560f0955ade9b8a3a07bdcf151f7359611", size = 776545, upload-time = "2026-08-16T22:56:10.159Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8e/58d4f842895220b793c53fc94a6489705b3665bb4d0ae4d338ce03fdf9fb/hypothesis-6.165.10-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f76d1562643693b8a40066f1f96af795b93fd9bcfc9690a1af2ff4c5867ee29e", size = 1107271, upload-time = "2026-08-16T22:54:50.266Z" }, + { url = "https://files.pythonhosted.org/packages/8f/b8/206468912d2153306bb8a41afdfc59e45b7a73a0495bbe4b9cb4f0e79c1d/hypothesis-6.165.10-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60cab3ab4ea468d31a33739ffd7e94ec3e37dea891d65a6582ecc8a477175191", size = 1156915, upload-time = "2026-08-16T22:54:25.89Z" }, + { url = "https://files.pythonhosted.org/packages/fb/d3/bf5a22929b70a4cfd3edf69c5642b029b27ddb5cfda48fa295d384b01abb/hypothesis-6.165.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:22cf19388f0ff6ced8eb3e49c903d14938e4ed909d93bf28383eef451511e424", size = 1281205, upload-time = "2026-08-16T22:54:44.083Z" }, + { url = "https://files.pythonhosted.org/packages/07/a2/d7b2ba444d36fc84d4779f4431e74dd9b023dc63bcf282199f6e48ad39f4/hypothesis-6.165.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:057d0232f1224dcd0b7698902551a4341a7399f90670b036db6c4376715fe889", size = 1324243, upload-time = "2026-08-16T22:55:41.123Z" }, + { url = "https://files.pythonhosted.org/packages/d1/95/afe6b531fd01928c6f63d394ee413fa2338d088b2b44efcc23596b54477e/hypothesis-6.165.10-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:ab0f2e9d7d7d4db257f7cf53de3706c2baf124269571f20ffc2bcd6781f03063", size = 616382, upload-time = "2026-08-16T22:55:18.449Z" }, + { url = "https://files.pythonhosted.org/packages/48/86/9b4fb75f520a028edec50ffc904a94d724180395d71feb6d7a0ce7bb6f00/hypothesis-6.165.10-cp314-cp314-win_amd64.whl", hash = "sha256:d1ea02fa8ab3d33eb1125eade81f7136341eb429152c6dbe2ae6f8bc33b3fbdd", size = 672145, upload-time = "2026-08-16T22:54:24.831Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ba/f7bbaae0c789bab7ddb764d2056ee1a463cc95a8acbccc90d4184e48b242/hypothesis-6.165.10-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:ed1a5891e59472884a03cb9875483e8fc131c80a275c60967f8afc5458a0c8ff", size = 783287, upload-time = "2026-08-16T22:54:23.751Z" }, + { url = "https://files.pythonhosted.org/packages/3a/83/01ef80772b4abd335c49405576dc503cede94fb5da30ba2643a119013aea/hypothesis-6.165.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:09772e328a26e50486ac572be34f9887f9aa185efe7ebb16bde4e8f6038db1f4", size = 774991, upload-time = "2026-08-16T22:55:25.987Z" }, + { url = "https://files.pythonhosted.org/packages/a3/0b/f47506241f9d5a5a2efe4c65b6bf4830e9d9576e5d3779007a260699e608/hypothesis-6.165.10-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cf3b612542ba174c9da4000b59a4f4c81e8d66f87509be85d3a1b71b5c36413", size = 1105499, upload-time = "2026-08-16T22:54:51.864Z" }, + { url = "https://files.pythonhosted.org/packages/84/fe/abb3909b7089835112fbe75bf00d817d733b3a8032759783db0a24ff1e56/hypothesis-6.165.10-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f69ec5be85ef508e206153bed8eafd03f7995dc464356c8bbb279a1e2b7d56f3", size = 1155685, upload-time = "2026-08-16T22:54:30.94Z" }, + { url = "https://files.pythonhosted.org/packages/73/2f/1964738921640184067121ae77414522fc3f0463fc26c6e25a4f3b8e42ca/hypothesis-6.165.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:dd207497bb985918409a1bb5db85d1875f74e1269487332113b73d1ee7c77647", size = 1279177, upload-time = "2026-08-16T22:54:40.179Z" }, + { url = "https://files.pythonhosted.org/packages/34/c5/312af8ae038d3af9cf3f7f1021c1abfe31c0d9035e4cf63519e0a7dc983e/hypothesis-6.165.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:00de0abdcf8c05c9d0eab735a3c49a276376b55151e6fcb903c2b39a90e5e5c3", size = 1322921, upload-time = "2026-08-16T22:54:42.7Z" }, + { url = "https://files.pythonhosted.org/packages/9e/e7/b0a2fde7570c090a1b914026266a421c751ef10138fffe37fe0ef9e675c0/hypothesis-6.165.10-cp314-cp314t-win_amd64.whl", hash = "sha256:cc2da5aa4edf14743fa9257e5ba3513963999f01211635702479d8e92b8207c8", size = 672147, upload-time = "2026-08-16T22:55:27.527Z" }, + { url = "https://files.pythonhosted.org/packages/47/fd/985aa564d6ffd06483d45a62b40d319df0a703cd8bc1d041de17d102fbaa/hypothesis-6.165.10-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:eeab73050ea58c13dd56e329f594c1dfe32ebd7bb169bbdf4f8ceefbc31ec6b5", size = 782882, upload-time = "2026-08-16T22:55:37.93Z" }, + { url = "https://files.pythonhosted.org/packages/f8/2c/6cc11151e450f72353a490940cd0db704680d07b78dc75dcc9f480e0d0e1/hypothesis-6.165.10-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:4c68e983d0007d014bb01ad4bcbba78bc432c73a1755ff36d5102ceefa18299a", size = 774584, upload-time = "2026-08-16T22:55:51.822Z" }, + { url = "https://files.pythonhosted.org/packages/10/39/ef26fa79c1738dfe9cdb1a3584fb6717d26429ca6c9d011cc4fdf08130c2/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7730d8197086f65d8969a991d6728a1d420a51b19fea06535c896cb43a1e05d0", size = 1104876, upload-time = "2026-08-16T22:54:58.937Z" }, + { url = "https://files.pythonhosted.org/packages/4e/f4/3fcc84e7637f42bf00d987093b9418083ac8db81b87392608a60f4b7c5fd/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7a7980a898a3e6ebe4de1896a0507e3d519edb53fb9b4bda478c9fbeb6514558", size = 1133353, upload-time = "2026-08-16T22:54:28.635Z" }, + { url = "https://files.pythonhosted.org/packages/35/59/21c5c14179c38f8d0de3560e7f1825c083311b3013b63f817d7dc78dfcbd/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b5820d009aedb7ae9cfd32f98b1ab0c0bbd6268379c4fab042218b6b655c63f8", size = 1132300, upload-time = "2026-08-16T22:56:08.539Z" }, + { url = "https://files.pythonhosted.org/packages/14/af/fbb56059961e416b2de7b9dc5352db2e8572bd5ea46892957e4c1e5548ab/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37a7ac3d34220800e1107871cc391bca1b00439875925d7d821878b8b791f245", size = 1155175, upload-time = "2026-08-16T22:55:19.824Z" }, + { url = "https://files.pythonhosted.org/packages/0f/53/77fb0c2dad445858555429c4e06cf94a59ae8d2407dd6426b5af97c84828/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:dafa7c9dbe3d802f9bcdf261b29c8a70700fb22839947f06e471f62c46b6257f", size = 1109881, upload-time = "2026-08-16T22:55:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/a8/7b/d187f673ff30e6ada640953636f978ffe64a6332f756b64163c2277f8d0c/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:90915635b9648071129b0f72c0673cf8eac9eb84cfd445c5bedef30c714b1ec2", size = 1144963, upload-time = "2026-08-16T22:56:13.428Z" }, + { url = "https://files.pythonhosted.org/packages/e0/60/31d504e364134d60af23e5f6365db0da3cf4a51b3ed3d4836e5a2cff12cf/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:e1bbeb7c506b07ee0422cf9b2f7212fefa4240957f03526d38d27bc6743a0a48", size = 1278684, upload-time = "2026-08-16T22:55:22.971Z" }, + { url = "https://files.pythonhosted.org/packages/ef/e6/89d26834a08c02f8da149e541dd40d7a96f68d9722f43146e69a77436ed7/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:2b36aaffc88625a44f91074c5bbedfdefb9b376c38d1b3c342edcd2e4c8ed16c", size = 1407202, upload-time = "2026-08-16T22:55:14.949Z" }, + { url = "https://files.pythonhosted.org/packages/dc/61/20d1e72246867ea195440092e8bb422c7ddc2f271b87b5b65679d5532719/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:18a3ea838ddea183388f8788750afa8494d79abb5358823be9782585f34445d3", size = 1261395, upload-time = "2026-08-16T22:56:05.448Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9b/ebab6c3c2b90a16abb4119198178652d12aff83cc8ec2cfde5276c69fb1e/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:2a2567b3a03a4a5a7c575c191cfcce321a967df3727803817e75bffbbeaecabe", size = 1279213, upload-time = "2026-08-16T22:55:35.066Z" }, + { url = "https://files.pythonhosted.org/packages/23/78/69b219b524231d36eb20c792e1f01e7cb037e02bd0af1c29f77ed9a969c0/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:8001925fa3dde51cb574e4c9de4c7efe77c4e4d64bd2fd2ef61d5651f9d04f3d", size = 1322367, upload-time = "2026-08-16T22:54:21.279Z" }, + { url = "https://files.pythonhosted.org/packages/55/63/ad5cc153dcc72ae5e7905fb9b3585f3e48ce892a2d6366f90163e867a69d/hypothesis-6.165.10-cp315-abi3.abi3t-win32.whl", hash = "sha256:c6559380469295c4009215fe1cab561301591a3bee2e2fb3f4f96d2273a3affc", size = 666038, upload-time = "2026-08-16T22:56:11.797Z" }, + { url = "https://files.pythonhosted.org/packages/80/32/b62307b73fbc99f0a4381d6f9456df76fbcbb7a27ef7256e26f0376f48ea/hypothesis-6.165.10-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:30797f20ca45e57f526d2df872f63ba453cb4e1091ad542184a7a951af8da79d", size = 671941, upload-time = "2026-08-16T22:55:00.235Z" }, + { url = "https://files.pythonhosted.org/packages/c2/dd/e0f98add0548ef73ea7afac45da1fb8efc854d7f9931db568754d0f963f3/hypothesis-6.165.10-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:c53e9b1c36350df9965ec44d6c0d4e0bbbb38f720dd2b0e1256dc6524d411015", size = 669931, upload-time = "2026-08-16T22:55:50.205Z" }, + { url = "https://files.pythonhosted.org/packages/0b/6a/880d6eeed5c451fb40a66733dadec4a5d498628a4a7f6a8a5f633f4c6dcb/hypothesis-6.165.10-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:34ee6402df6f31274d89119f1561b5f7489c97866afc5b7a3ed3a13d7e762802", size = 784644, upload-time = "2026-08-16T22:54:20.127Z" }, + { url = "https://files.pythonhosted.org/packages/27/e0/9e942bd3c3cf5ea0d5c0fd0905893bbfb6cefb7284c70fcc8033f8fdec38/hypothesis-6.165.10-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:277f41801e88dad2eba082f91a75632b7584ff64044ba2cf9dadf511b0d19cd0", size = 780515, upload-time = "2026-08-16T22:55:04.676Z" }, + { url = "https://files.pythonhosted.org/packages/19/32/f11a618415dc5fa9cdde41fea56c489f0814759527ae1ecd11a75a4558b9/hypothesis-6.165.10-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:72df95fb1db41755b155c5f02106e0036a339250555c8d351d488704fd112cf9", size = 1109374, upload-time = "2026-08-16T22:56:00.241Z" }, + { url = "https://files.pythonhosted.org/packages/5e/6f/db49b719842297c2b71e0d81e5b8967d31215fb7389421abcb465ce7ed3f/hypothesis-6.165.10-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e20a02775eb3cf0ffb4f0219b6d7c1f240336663d4e5d7028675ec247c790c4", size = 1159092, upload-time = "2026-08-16T22:55:58.57Z" }, + { url = "https://files.pythonhosted.org/packages/b2/2a/bf0bae84ba1cb3923d295973f1fe38ee867eaf90119e0d559116083be300/hypothesis-6.165.10-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:1ec53f08732e3cfd0342cbbd75dbd1b193c8f19390660466e536a748bb81f757", size = 676045, upload-time = "2026-08-16T22:55:45.514Z" }, +] + [[package]] name = "idna" version = "3.15" @@ -4430,6 +4522,7 @@ dev = [ { name = "diff-cover" }, { name = "fakeredis" }, { name = "fastapi-offline" }, + { name = "hypothesis" }, { name = "keyring" }, { name = "langfuse" }, { name = "openapi-core" }, @@ -4615,6 +4708,7 @@ dev = [ { name = "diff-cover", specifier = "==9.7.2" }, { name = "fakeredis", specifier = "==2.34.1" }, { name = "fastapi-offline", specifier = "==1.7.6" }, + { name = "hypothesis", specifier = "==6.165.10" }, { name = "keyring", specifier = "==25.7.0" }, { name = "langfuse", specifier = "==2.59.7" }, { name = "openapi-core", specifier = "==0.22.0" },