diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 3306b970229..1b646eb6fbe 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -60,9 +60,6 @@ fn chat_completions_response_to_py( fn core_error_to_pyerr(err: CoreError) -> PyErr { match err { - CoreError::MissingField("document_url") => { - PyValueError::new_err("Document URL is required") - } CoreError::Auth(message) => PyValueError::new_err(message), CoreError::InvalidProvider(_) | CoreError::InvalidRequest(_) @@ -74,6 +71,9 @@ fn core_error_to_pyerr(err: CoreError) -> PyErr { fn ocr_error_to_pyerr(err: CoreError) -> PyErr { match err { + CoreError::MissingField("document_url" | "image_url") => { + PyValueError::new_err("Document URL is required") + } CoreError::Http { status, body } => RustUpstreamError::new_err((status, body)), other => core_error_to_pyerr(other), } diff --git a/litellm/llms/vertex_ai/ocr/deepseek_transformation.py b/litellm/llms/vertex_ai/ocr/deepseek_transformation.py index 2603552152d..b57a87c3325 100644 --- a/litellm/llms/vertex_ai/ocr/deepseek_transformation.py +++ b/litellm/llms/vertex_ai/ocr/deepseek_transformation.py @@ -177,8 +177,9 @@ class VertexAIDeepSeekOCRConfig(BaseOCRConfig): content_item = {"type": "image_url", "image_url": document_url} # Build DeepSeek OCR request + provider_model: Final = model if model.startswith("deepseek-ai/") else f"deepseek-ai/{model}" data: Final = { - "model": "deepseek-ai/" + model, + "model": provider_model, "messages": [{"role": "user", "content": [content_item]}], } diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index b918f013700..58b7259480e 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -52,12 +52,13 @@ class _PreparedOCRRequest: litellm_logging_obj: LiteLLMLoggingObj -@dataclass +@dataclass(frozen=True, slots=True) class _PreparedRustOCRCall: api_key: str | None api_base: str | None headers: dict[str, object] optional_params: dict[str, object] + request_url: str _RUST_OCR_PROVIDERS: Final = { @@ -191,6 +192,10 @@ def _prepare_ocr_request( def _rust_ocr_supported(prepared_request: _PreparedOCRRequest) -> bool: if prepared_request.optional_params.get(OCR_REQUEST_FORMAT_PARAM) == "native": return False + if prepared_request.extra_headers is not None and any( + not isinstance(value, str) for value in prepared_request.extra_headers.values() + ): + return False return prepared_request.custom_llm_provider in _RUST_OCR_PROVIDERS @@ -275,6 +280,7 @@ def _prepare_rust_ocr_call( api_base=rust_api_base, headers=cast(dict[str, object], resolved_headers), optional_params=rust_optional_params, + request_url=resolved_complete_url, ) @@ -297,6 +303,7 @@ def _run_rust_ocr( extra_headers=prepared.headers, optional_params=prepared.optional_params, timeout=prepared_request.effective_timeout, + request_url=prepared.request_url, ) if rust_response is None: return None @@ -322,6 +329,7 @@ async def _run_rust_aocr( extra_headers=prepared.headers, optional_params=prepared.optional_params, timeout=prepared_request.effective_timeout, + request_url=prepared.request_url, ) if rust_response is None: return None diff --git a/litellm/rust_bridge/chat_completions.py b/litellm/rust_bridge/chat_completions.py index acda3086051..fe9ce15775f 100644 --- a/litellm/rust_bridge/chat_completions.py +++ b/litellm/rust_bridge/chat_completions.py @@ -281,18 +281,6 @@ def rust_chat_completions_accepts( return True -def _rust_bridge_exceptions() -> tuple[type[BaseException], type[BaseException]] | None: - """`(declined, upstream_failed)` from the native module, or None when absent.""" - native_bridge: Final = get_native_bridge() - if native_bridge is None: - return None - declined: Final = getattr(native_bridge, "RustBridgeDeclined", None) - upstream: Final = getattr(native_bridge, "RustUpstreamError", None) - if declined is None or upstream is None: - return None - return declined, upstream - - def _reraise_or_decline( rust_error: BaseException, *, @@ -306,25 +294,25 @@ def _reraise_or_decline( second attempt bills for it twice. Those surface as an `APIError` carrying the upstream status, which LiteLLM's exception mapping already understands. """ - exceptions: Final = _rust_bridge_exceptions() + from litellm.rust_bridge.errors import native_bridge_exceptions, rust_upstream_failure + + native_bridge: Final = get_native_bridge() + exceptions: Final = native_bridge_exceptions(native_bridge) if exceptions is None: verbose_logger.debug( "Rust chat completions bridge raised %s; falling back to Python path", type(rust_error).__name__, ) return - declined, upstream_failed = exceptions - if isinstance(rust_error, upstream_failed): - args: Final = rust_error.args - status: Final = args[0] if args else 0 - message: Final = args[1] if len(args) > 1 else "" + upstream_failure: Final = rust_upstream_failure(rust_error, native_bridge) + if upstream_failure is not None: raise APIError( - status_code=int(status) or 500, - message=f"litellm rust chat completions: {message}", + status_code=upstream_failure.status_code, + message=f"litellm rust chat completions: {upstream_failure.message}", llm_provider=custom_llm_provider or "", model=model, ) - if not isinstance(rust_error, declined): + if not isinstance(rust_error, exceptions.declined): raise rust_error verbose_logger.debug( "Rust chat completions declined before calling the provider (%s); using the Python path", diff --git a/litellm/rust_bridge/errors.py b/litellm/rust_bridge/errors.py new file mode 100644 index 00000000000..82d7339bc84 --- /dev/null +++ b/litellm/rust_bridge/errors.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Final + + +@dataclass(frozen=True, slots=True) +class NativeBridgeExceptions: + declined: type[BaseException] + upstream: type[BaseException] + + +@dataclass(frozen=True, slots=True) +class RustUpstreamFailure: + status_code: int + message: str + + +def native_bridge_exceptions(native_bridge: object | None) -> NativeBridgeExceptions | None: + if native_bridge is None: + return None + declined: Final = getattr(native_bridge, "RustBridgeDeclined", None) + upstream: Final = getattr(native_bridge, "RustUpstreamError", None) + if not isinstance(declined, type) or not isinstance(upstream, type): + return None + return NativeBridgeExceptions(declined=declined, upstream=upstream) + + +def rust_upstream_failure( + error: BaseException, + native_bridge: object | None, +) -> RustUpstreamFailure | None: + exceptions: Final = native_bridge_exceptions(native_bridge) + if exceptions is None or not isinstance(error, exceptions.upstream): + return None + status: Final = error.args[0] if error.args else 0 + message: Final = error.args[1] if len(error.args) > 1 else "" + return RustUpstreamFailure(status_code=int(status) or 500, message=str(message)) diff --git a/litellm/rust_bridge/ocr.py b/litellm/rust_bridge/ocr.py index ef880b1dce5..52e92782b44 100644 --- a/litellm/rust_bridge/ocr.py +++ b/litellm/rust_bridge/ocr.py @@ -66,25 +66,21 @@ _rust_aocr_impl: RustAocr | None = None class _OcrProviderError(Exception): - def __init__(self, status_code: int, message: str, api_base: str | None) -> None: + def __init__(self, status_code: int, message: str, request_url: str | None) -> None: super().__init__(message) self.status_code: Final = status_code - self.response: Final = httpx.Response( - status_code=status_code, - request=httpx.Request("POST", api_base or "https://api.mistral.ai/v1/ocr"), - ) + request: Final = httpx.Request("POST", request_url) if request_url is not None else None + self.response: Final = httpx.Response(status_code=status_code, request=request) -def _raise_provider_error(error: BaseException, api_base: str | None) -> None: +def _raise_provider_error(error: BaseException, request_url: str | None) -> None: from litellm.rust_bridge import get_native_bridge + from litellm.rust_bridge.errors import rust_upstream_failure - native_bridge: Final = get_native_bridge() - upstream_error: Final = getattr(native_bridge, "RustUpstreamError", None) if native_bridge is not None else None - if upstream_error is None or not isinstance(error, upstream_error): + failure: Final = rust_upstream_failure(error, get_native_bridge()) + if failure is None: raise error - status: Final = error.args[0] if error.args else 0 - message: Final = error.args[1] if len(error.args) > 1 else "" - raise _OcrProviderError(int(status) or 500, str(message), api_base) from error + raise _OcrProviderError(failure.status_code, failure.message, request_url) from error def use_litellm_rust( @@ -170,6 +166,7 @@ def ocr( extra_headers: dict[str, object] | None, optional_params: dict[str, object], timeout: float | httpx.Timeout | None, + request_url: str | None = None, ) -> dict[str, object] | None: rust_ocr: Final = load_rust_ocr() if rust_ocr is None: @@ -186,7 +183,7 @@ def ocr( timeout_seconds=_timeout_to_seconds(timeout), ) except Exception as error: - _raise_provider_error(error, api_base) + _raise_provider_error(error, request_url) async def aocr( @@ -199,6 +196,7 @@ async def aocr( extra_headers: dict[str, object] | None, optional_params: dict[str, object], timeout: float | httpx.Timeout | None, + request_url: str | None = None, ) -> dict[str, object] | None: rust_aocr: Final = load_rust_aocr() if rust_aocr is None: @@ -215,4 +213,4 @@ async def aocr( timeout_seconds=_timeout_to_seconds(timeout), ) except Exception as error: - _raise_provider_error(error, api_base) + _raise_provider_error(error, request_url) diff --git a/tests/e2e/provider_edge.py b/tests/e2e/provider_edge.py index ceb695ffcd6..e70b50257b1 100644 --- a/tests/e2e/provider_edge.py +++ b/tests/e2e/provider_edge.py @@ -94,6 +94,11 @@ from fixture_mode import ( current_test_key, parse_fixture_mode, ) +from tests.provider_record_replay.http import ( + dropped_request_headers, + dropped_response_headers, + is_streaming_response, +) EDGE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType( { @@ -104,29 +109,6 @@ EDGE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType( REPLAY_MISS_STATUS: Final = 599 -_HOP_BY_HOP_HEADERS: Final[frozenset[str]] = frozenset( - { - "connection", - "keep-alive", - "proxy-authenticate", - "proxy-authorization", - "te", - "trailers", - "transfer-encoding", - "upgrade", - } -) -_REQUEST_DROPPED_HEADERS: Final[frozenset[str]] = _HOP_BY_HOP_HEADERS | { - "host", - "content-length", - "accept-encoding", -} -_RESPONSE_DROPPED_HEADERS: Final[frozenset[str]] = _HOP_BY_HOP_HEADERS | { - "content-encoding", - "content-length", - "set-cookie", -} - _JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) @@ -566,9 +548,8 @@ def _filtered_response_headers(headers: Mapping[str, str]) -> dict[str, str]: """What the edge stores and serves: the provider's headers minus hop-by-hop and volatile entries. Framing headers are in that set, so a stored header can never contradict the framing the edge chooses when it serves the response.""" - return { - name: value for name, value in headers.items() if name not in _RESPONSE_DROPPED_HEADERS - } + excluded: Final = dropped_response_headers(headers.items()) + return {name: value for name, value in headers.items() if name.lower() not in excluded} def _network_error_response(message: str) -> RecordedHttpResponse: @@ -610,7 +591,7 @@ def _is_streamed(headers: Mapping[str, str]) -> bool: move nearly every recording to the streamed shape for no gain. The content type is the header that says "consume this as it arrives", and it is already how the harness defines streaming everywhere else.""" - return "text/event-stream" in _header_value(headers, "content-type").lower() + return is_streaming_response(_header_value(headers, "content-type")) def _upstream_url(upstream_base: str, upstream_path: str, query: str) -> str: @@ -698,8 +679,9 @@ def _handle_record( timeout: float, ) -> EdgeOutcome: test_key: Final = current_test_key() + excluded: Final = dropped_request_headers(headers.items()) forwarded: Final = { - name: value for name, value in headers.items() if name.lower() not in _REQUEST_DROPPED_HEADERS + name: value for name, value in headers.items() if name.lower() not in excluded } head: Final = forward_stream(method, url, headers=forwarded, body=body, timeout=timeout) match head: diff --git a/tests/provider_record_replay/__init__.py b/tests/provider_record_replay/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/tests/provider_record_replay/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/provider_record_replay/http.py b/tests/provider_record_replay/http.py new file mode 100644 index 00000000000..c164e3c6549 --- /dev/null +++ b/tests/provider_record_replay/http.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from collections.abc import Iterable +from typing import Final + +HOP_BY_HOP_HEADERS: Final[frozenset[str]] = frozenset( + { + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "trailers", + "transfer-encoding", + "upgrade", + } +) + +REQUEST_DROPPED_HEADERS: Final[frozenset[str]] = HOP_BY_HOP_HEADERS | { + "host", + "content-length", + "accept-encoding", +} + +RESPONSE_DROPPED_HEADERS: Final[frozenset[str]] = HOP_BY_HOP_HEADERS | { + "content-encoding", + "content-length", + "set-cookie", +} + + +def connection_header_names(headers: Iterable[tuple[str, str]]) -> frozenset[str]: + return frozenset( + token.strip().lower() + for name, value in headers + if name.lower() == "connection" + for token in value.split(",") + if token.strip() + ) + + +def dropped_request_headers(headers: Iterable[tuple[str, str]]) -> frozenset[str]: + materialized: Final = tuple(headers) + return REQUEST_DROPPED_HEADERS | connection_header_names(materialized) + + +def dropped_response_headers(headers: Iterable[tuple[str, str]]) -> frozenset[str]: + materialized: Final = tuple(headers) + return RESPONSE_DROPPED_HEADERS | connection_header_names(materialized) + + +def is_streaming_response(content_type: str) -> bool: + return "text/event-stream" in content_type.lower() diff --git a/tests/route_parity/README.md b/tests/route_parity/README.md index bf396a9e82c..c072b5ae575 100644 --- a/tests/route_parity/README.md +++ b/tests/route_parity/README.md @@ -1,11 +1,11 @@ -# Python/Rust parity testing in the Python SDK interface +# Implementation parity testing through the SDK interface -> Given the same SDK call and identical provider behavior, does the PyO3 implementation behave same as Python? +> Given the same SDK call and identical provider behavior, do two implementations expose the same SDK contract? ## What the harness compares - A fixture contains a LiteLLM SDK input and a recorded upstream provider response -- The same LiteLLM input is transformed by isolated Python and Rust workers +- The same LiteLLM input is transformed by isolated baseline and candidate implementations - The resulting provider requests must match in method, path, headers, and body, excluding runtime-specific HTTP metadata - The recorded provider response is then replayed unchanged to both workers - The harness compares the values returned through the Python SDK interface @@ -17,7 +17,7 @@ ## Process isolation -- SDK object and stream parity runs Python and Rust sequentially in the same process so tests can retain the returned objects +- SDK object and stream parity runs both implementations sequentially in the same process so tests can retain returned objects - Every test saves and restores the original bridge state - A small subprocess smoke test verifies environment-based startup configuration and detects fallback to the Python HTTP implementation @@ -28,59 +28,17 @@ - Property-based tests define strategies for valid inputs and properties that must hold for every generated example - Hypothesis generates combinations from those strategies and normally shrinks a failing example to a smaller reproducible case - In this harness, Hypothesis is used only during fixture generation to expand the LiteLLM input corpus -- The current OCR strategy varies supported optional parameters while keeping inputs valid +- Each API owns the strategies that vary its supported inputs - Fixture generation is deterministic, and each generated input is recorded with the raw provider response it received - The parity tests use committed fixtures and do not call the provider or generate new Hypothesis examples - Provider responses are replayed unchanged, so the parity test does not fuzz or validate provider behavior - Because Hypothesis does not run the parity assertion directly, parity failures are not automatically shrunk -## Recording fixtures +## API-owned fixtures -The recording command runs four explicit stages: - -1. Generate deterministic SDK inputs for every configured OCR target -2. Build target-scoped, deduplicated recording jobs -3. Record upstream responses through one globally bounded worker pool -4. Persist each fixture and report whether it was recorded, cached, or failed - -Run it with: - -```shell -uv run python -m tests.test_litellm.ocr.fixtures.record --examples 4 --concurrency 4 -``` - -`--concurrency` caps all provider calls across all targets. Each completed job is reported immediately, and the final -summary reports recorded, cached, and failed totals. Independent jobs finish after a failure, then the command exits -nonzero if any job failed - -## OCR input boundaries - -OCR strategies generate only public `litellm.ocr()` and `litellm.aocr()` inputs. Every case contains the normalized -`model`, Mistral-shaped `document`, optional `custom_llm_provider`, and LiteLLM keyword arguments. The fixture-only -`boundary` tag selects the valid keyword-argument set and is removed before calling the SDK. Strategies never build -provider wire payloads. - -Each boundary has a required corpus containing a baseline and one case for every supported top-level LiteLLM OCR -parameter for every active registered model that uses that transformation. Models whose registry deprecation date has -passed are excluded. `--examples` controls additional Hypothesis-generated cases; it does not replace the required -corpus. - -The explicit boundaries are Mistral, Azure-hosted Mistral, Vertex-hosted Mistral, Azure Document Intelligence, -Vertex DeepSeek, Reducto v3, and Reducto legacy. Provider credentials and endpoints only control target discovery, so -a machine records the boundaries it has configured and skips the rest. Azure-hosted Mistral enumerates its active -registered models rather than requiring a separately configured deployment model. Reducto fixtures record both upload -and parse responses. Their parity cases are non-strict expected failures until the Rust OCR bridge supports Reducto, so -both expected failures and unexpected passes keep CI green during the rollout. - -The committed corpus does not need to contain live recordings for every configured target. In particular, Azure and -Vertex generation paths are covered by unit tests without requiring their credentials in CI. Recordings can be added -later without changing the fixture schema or runner. - -Invalid OCR inputs do not use recorded provider responses. The parity suite checks unsupported providers and models, -malformed documents, invalid request formats, invalid Azure Document Intelligence parameters, and invalid headers in -both sync and async SDK calls. These cases must return the same public exception fields without sending a provider -request. A malformed Azure Document Intelligence document reaches the Rust bridge so its native validation error is -also compared against Python +The shared package owns recording, replay, persistence, execution, comparison, and route-neutral media constructors. +Each API package owns its input models, explicit strategies, provider targets, route-specific assets, fixture directory, +and regeneration command. See the API package documentation for its configured contracts and recording command ## References diff --git a/tests/route_parity/compare.py b/tests/route_parity/compare.py index 0e469e6eb6d..6c0b81735be 100644 --- a/tests/route_parity/compare.py +++ b/tests/route_parity/compare.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import Final, TypeVar +from collections.abc import Mapping, Sequence +from typing import Final, TypeVar, cast from pydantic import BaseModel @@ -9,26 +10,26 @@ from tests.route_parity.models import CapturedRequest, Execution ModelT = TypeVar("ModelT", bound=BaseModel) -def validate_harness(python: Execution, accelerated: Execution, python_user_agent: str) -> None: - for request in python.requests: - if request.user_agent != python_user_agent: +def validate_harness(baseline: Execution, candidate: Execution, baseline_user_agent: str) -> None: + for request in baseline.requests: + if request.user_agent != baseline_user_agent: raise AssertionError( - f"Python provider request did not carry fallback sentinel user-agent {python_user_agent!r}: " + f"baseline provider request did not carry sentinel user-agent {baseline_user_agent!r}: " f"{request.user_agent!r}" ) - for request in accelerated.requests: - if request.user_agent == python_user_agent: - raise AssertionError("accelerated route fell back to the Python HTTP implementation") + for request in candidate.requests: + if request.user_agent == baseline_user_agent: + raise AssertionError("candidate route fell back to the baseline HTTP implementation") def _request_after_transformation(request: CapturedRequest) -> CapturedRequest: return request.model_copy(update={"user_agent": None}) -def assert_request_parity(python: tuple[CapturedRequest, ...], accelerated: tuple[CapturedRequest, ...]) -> None: - python_requests: Final = tuple(_request_after_transformation(request) for request in python) - accelerated_requests: Final = tuple(_request_after_transformation(request) for request in accelerated) - assert python_requests == accelerated_requests +def assert_request_parity(baseline: tuple[CapturedRequest, ...], candidate: tuple[CapturedRequest, ...]) -> None: + baseline_requests: Final = tuple(_request_after_transformation(request) for request in baseline) + candidate_requests: Final = tuple(_request_after_transformation(request) for request in candidate) + assert baseline_requests == candidate_requests def public_model_copy(model: ModelT) -> ModelT: @@ -37,12 +38,39 @@ def public_model_copy(model: ModelT) -> ModelT: return copied -def assert_model_parity(python: BaseModel, accelerated: BaseModel) -> None: - assert type(python) is type(accelerated) - assert public_model_copy(python) == public_model_copy(accelerated) +def assert_model_parity(baseline: BaseModel, candidate: BaseModel) -> None: + assert type(baseline) is type(candidate) + _assert_value_parity( + public_model_copy(baseline).model_dump(mode="python"), + public_model_copy(candidate).model_dump(mode="python"), + path="$", + ) -def assert_parity(python: Execution, accelerated: Execution, python_user_agent: str) -> None: - validate_harness(python, accelerated, python_user_agent) - assert_request_parity(python.requests, accelerated.requests) - assert python.report == accelerated.report +def _assert_value_parity(baseline: object, candidate: object, *, path: str) -> None: + if isinstance(baseline, Mapping) and isinstance(candidate, Mapping): + baseline_mapping: Final = cast(Mapping[object, object], baseline) + candidate_mapping: Final = cast(Mapping[object, object], candidate) + assert baseline_mapping.keys() == candidate_mapping.keys(), f"mapping keys differ at {path}" + for key in baseline_mapping: + _assert_value_parity(baseline_mapping[key], candidate_mapping[key], path=f"{path}.{key}") + return + if ( + isinstance(baseline, Sequence) + and not isinstance(baseline, (str, bytes)) + and isinstance(candidate, Sequence) + and not isinstance(candidate, (str, bytes)) + ): + baseline_sequence: Final = cast(Sequence[object], baseline) + candidate_sequence: Final = cast(Sequence[object], candidate) + assert len(baseline_sequence) == len(candidate_sequence), f"sequence lengths differ at {path}" + for index, (baseline_item, candidate_item) in enumerate(zip(baseline_sequence, candidate_sequence)): + _assert_value_parity(baseline_item, candidate_item, path=f"{path}[{index}]") + return + assert baseline == candidate, f"value mismatch at {path}: {baseline!r} != {candidate!r}" + + +def assert_parity(baseline: Execution, candidate: Execution, baseline_user_agent: str) -> None: + validate_harness(baseline, candidate, baseline_user_agent) + assert_request_parity(baseline.requests, candidate.requests) + assert baseline.report == candidate.report diff --git a/tests/route_parity/fixture_models.py b/tests/route_parity/fixture_models.py index 61ec0484611..f1ed9dc3fe9 100644 --- a/tests/route_parity/fixture_models.py +++ b/tests/route_parity/fixture_models.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import Final, Generic, Literal, TypeVar, cast +from typing import ClassVar, Final, Generic, Literal, TypeVar, cast from pydantic import BaseModel, ConfigDict, Field, JsonValue, model_validator @@ -15,15 +15,22 @@ class FixtureModel(BaseModel): class SdkInputBase(FixtureModel): - boundary: str = "default" + fixture_only_fields: ClassVar[tuple[str, ...]] = () def as_sdk_kwargs(self) -> dict[str, object]: - dumped: Final = cast(dict[str, object], self.model_dump(mode="python", exclude_unset=True)) - return {key: value for key, value in dumped.items() if key != "boundary"} + return cast( + dict[str, object], + self.model_dump( + mode="python", + exclude_unset=True, + exclude=set(self.fixture_only_fields), + ), + ) def canonical_input(self) -> dict[str, object]: dumped: Final = cast(dict[str, object], self.model_dump(mode="json", exclude_unset=True)) - return {"boundary": self.boundary, **{key: value for key, value in dumped.items() if key != "boundary"}} + fixture_fields: Final = {field: getattr(self, field) for field in self.fixture_only_fields} + return {**fixture_fields, **dumped} class JsonSchemaDefinition(FixtureModel): @@ -50,29 +57,8 @@ class ParityCase(FixtureModel, Generic[InputT]): def load_legacy_single_response(cls, value: object) -> object: if not isinstance(value, Mapping): return value - migrated: Final = dict(value) + migrated: Final = dict(cast(Mapping[str, object], value)) provider_response: Final = migrated.pop("provider_response", None) if "provider_responses" not in migrated and provider_response is not None: migrated["provider_responses"] = (provider_response,) - litellm_input: Final = migrated.get("litellm_input") - if isinstance(litellm_input, Mapping) and "boundary" not in litellm_input: - model: Final = litellm_input.get("model") - if isinstance(model, str): - migrated["litellm_input"] = {"boundary": _legacy_boundary(model), **litellm_input} return migrated - - -def _legacy_boundary(model: str) -> str: - if model.startswith("azure_ai/doc-intelligence/"): - return "azure_document_intelligence" - if model.startswith("azure_ai/"): - return "azure_mistral" - if model.startswith("vertex_ai/deepseek"): - return "vertex_deepseek" - if model.startswith("vertex_ai/"): - return "vertex_mistral" - if model.endswith("parse-v3"): - return "reducto_v3" - if model.endswith("parse-legacy"): - return "reducto_legacy" - return "mistral" diff --git a/tests/route_parity/fixtures/cli.py b/tests/route_parity/fixtures/cli.py new file mode 100644 index 00000000000..b6a2db1c395 --- /dev/null +++ b/tests/route_parity/fixtures/cli.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import argparse +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Final, cast + + +@dataclass(frozen=True, slots=True) +class RecordingArgs: + concurrency: int + examples: int + fixture_dir: Path | None + + +def _positive_int(value: str) -> int: + parsed: Final = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("must be at least 1") + return parsed + + +def parse_recording_args(argv: Sequence[str] | None = None) -> RecordingArgs: + parser: Final = argparse.ArgumentParser() + parser.add_argument("--concurrency", type=_positive_int, default=4) + parser.add_argument("--examples", type=_positive_int, default=4) + parser.add_argument("--fixture-dir", type=Path) + namespace: Final = parser.parse_args(argv) + return RecordingArgs( + concurrency=cast(int, namespace.concurrency), + examples=cast(int, namespace.examples), + fixture_dir=cast(Path | None, namespace.fixture_dir), + ) diff --git a/tests/route_parity/fixtures/media.py b/tests/route_parity/fixtures/media.py index 2bb3cd99601..d1ea2901f5d 100644 --- a/tests/route_parity/fixtures/media.py +++ b/tests/route_parity/fixtures/media.py @@ -19,9 +19,9 @@ def dummy_image_url(text: str, font_size: int, width: int = 800, height: int = 3 _GLYPHS: Final = { + "D": ("11110", "10001", "10001", "10001", "10001", "10001", "11110"), "O": ("01110", "10001", "10001", "10001", "10001", "10001", "01110"), "C": ("01111", "10000", "10000", "10000", "10000", "10000", "01111"), - "R": ("11110", "10001", "10001", "11110", "10100", "10010", "10001"), "1": ("00100", "01100", "00100", "00100", "00100", "00100", "01110"), "2": ("01110", "10001", "00001", "00010", "00100", "01000", "11111"), "3": ("11110", "00001", "00001", "01110", "00001", "00001", "11110"), @@ -34,7 +34,7 @@ def structured_image_bytes() -> bytes: draw: Final = ImageDraw.Draw(image) scale: Final = 8 cursor_x = 24 - for character in "OCR 123": + for character in "DOC 123": if character == " ": cursor_x += scale * 3 continue @@ -108,7 +108,7 @@ def _draw_table_page(pdf: canvas.Canvas) -> None: ( ("Item", "Quantity", "Amount", 707), ("Document analysis", "2", "120.00", 672), - ("OCR verification", "1", "80.00", 637), + ("Document verification", "1", "80.00", 637), ), ), ( @@ -200,9 +200,9 @@ def structured_pdf_bytes() -> bytes: output: Final = BytesIO() pdf: Final = canvas.Canvas(output, pagesize=letter, pageCompression=0, invariant=1) pdf.setTitle("Quarterly Operations Report") - pdf.setAuthor("LiteLLM OCR fixture generator") - pdf.setSubject("Semantic OCR coverage for tables, figures, annotations, and metadata") - pdf.setKeywords("OCR, invoice, table, figure, annotation") + pdf.setAuthor("LiteLLM parity fixture generator") + pdf.setSubject("Semantic document coverage for tables, figures, annotations, and metadata") + pdf.setKeywords("document, invoice, table, figure, annotation") pages: Final = ( ("Invoice Summary and Line Items", _draw_table_page), ("Revenue Chart and Formula Review", _draw_chart_page), diff --git a/tests/route_parity/fixtures/pipeline.py b/tests/route_parity/fixtures/pipeline.py index 17a2791593c..39e4ee76eb9 100644 --- a/tests/route_parity/fixtures/pipeline.py +++ b/tests/route_parity/fixtures/pipeline.py @@ -1,19 +1,17 @@ from __future__ import annotations -import argparse import logging -from collections.abc import Sequence from concurrent.futures import Future, ThreadPoolExecutor, as_completed from dataclasses import dataclass, field from pathlib import Path from types import MappingProxyType -from typing import Final, Generic, Literal, Protocol, TypeVar, cast +from typing import Final, Generic, Literal, Protocol, TypeVar from hypothesis.strategies import SearchStrategy from pydantic import BaseModel from tests.route_parity.fixtures.inputs import generate_case_inputs -from tests.route_parity.fixtures.recording import ProviderSpec, record_upstream_responses +from tests.route_parity.fixtures.recording import UpstreamEndpoint, record_upstream_responses from tests.route_parity.fixtures.store import ( FixtureInput, canonical_json, @@ -30,13 +28,6 @@ InputT_contra = TypeVar("InputT_contra", bound=FixtureInput, contravariant=True) CaseT = TypeVar("CaseT", bound=BaseModel) -@dataclass(frozen=True, slots=True) -class RecordingArgs: - concurrency: int - examples: int - fixture_dir: Path | None - - class RecordingInvocation(Protocol[InputT_contra]): def execute(self, provider_url: str, case_input: InputT_contra) -> None: ... @@ -44,7 +35,7 @@ class RecordingInvocation(Protocol[InputT_contra]): @dataclass(frozen=True, slots=True) class RecordingTarget(Generic[InputT]): name: str - provider_spec: ProviderSpec + upstream: UpstreamEndpoint strategy: SearchStrategy[InputT] invocation: RecordingInvocation[InputT] = field(repr=False) required_inputs: tuple[InputT, ...] = () @@ -54,7 +45,7 @@ class RecordingTarget(Generic[InputT]): class RecordingJob(Generic[InputT]): target_name: str directory: Path - provider_spec: ProviderSpec + upstream: UpstreamEndpoint case_input: InputT invocation: RecordingInvocation[InputT] = field(repr=False) @@ -118,7 +109,7 @@ def build_recording_jobs( RecordingJob( target_name=target.name, directory=root / target.name, - provider_spec=target.provider_spec, + upstream=target.upstream, case_input=case_input, invocation=target.invocation, ) @@ -136,7 +127,7 @@ def _record_job(job: RecordingJob[InputT], case_type: type[CaseT]) -> RecordedFi path=fixture_path(job.directory, job.case_input), ) provider_responses: Final = record_upstream_responses( - job.provider_spec, + job.upstream, job.case_input, job.invocation.execute, ) @@ -199,23 +190,3 @@ def record_fixtures( len(summary.failed), ) return summary - - -def _positive_int(value: str) -> int: - parsed: Final = int(value) - if parsed < 1: - raise argparse.ArgumentTypeError("must be at least 1") - return parsed - - -def parse_recording_args(argv: Sequence[str] | None = None) -> RecordingArgs: - parser: Final = argparse.ArgumentParser() - parser.add_argument("--concurrency", type=_positive_int, default=4) - parser.add_argument("--examples", type=_positive_int, default=4) - parser.add_argument("--fixture-dir", type=Path) - namespace: Final = parser.parse_args(argv) - return RecordingArgs( - concurrency=cast(int, namespace.concurrency), - examples=cast(int, namespace.examples), - fixture_dir=cast(Path | None, namespace.fixture_dir), - ) diff --git a/tests/route_parity/fixtures/pytest_support.py b/tests/route_parity/fixtures/pytest_support.py new file mode 100644 index 00000000000..c4582ff62a4 --- /dev/null +++ b/tests/route_parity/fixtures/pytest_support.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import os +from collections.abc import Callable +from pathlib import Path +from typing import Final, TypeVar + +import pytest +from pydantic import BaseModel, ValidationError + +from tests.route_parity.fixtures.store import recorded_fixtures + +CaseT = TypeVar("CaseT", bound=BaseModel) + + +def parametrize_recorded_fixtures( + metafunc: pytest.Metafunc, + *, + fixture_name: str, + case_type: type[CaseT], + env_var: str, + default_directory: Path, + regeneration_command: str, + id_builder: Callable[[CaseT], str], + marks_builder: Callable[[CaseT], tuple[pytest.MarkDecorator, ...]] | None = None, +) -> None: + if fixture_name not in metafunc.fixturenames: + return + configured: Final = os.environ.get(env_var) + if configured == "": + raise pytest.UsageError(f"{env_var} is set but empty") + directory: Final = Path(configured).expanduser() if configured is not None else default_directory + try: + fixtures: Final = recorded_fixtures(directory, case_type) + except (ValidationError, ValueError) as error: + raise pytest.UsageError( + f"Invalid parity fixture bundle at {directory}. " + "Each fixture must use the current versioned envelope. " + f"Record fresh fixtures in an empty directory with: `{regeneration_command}`. " + f"Validation details: {error}" + ) from error + if fixtures: + metafunc.parametrize( + fixture_name, + tuple( + pytest.param( + fixture, + id=id_builder(fixture), + marks=marks_builder(fixture) if marks_builder is not None else (), + ) + for fixture in fixtures + ), + ) + return + if configured is not None: + raise pytest.UsageError(f"no recorded fixtures in {directory}") + metafunc.parametrize( + fixture_name, + ( + pytest.param( + None, + marks=pytest.mark.skip(reason=f"no recorded fixtures in {directory}"), + id="no-recorded-fixtures", + ), + ), + ) diff --git a/tests/route_parity/fixtures/recording.py b/tests/route_parity/fixtures/recording.py index ea0ee77c215..d85f53324cb 100644 --- a/tests/route_parity/fixtures/recording.py +++ b/tests/route_parity/fixtures/recording.py @@ -11,6 +11,11 @@ from urllib.parse import urlsplit, urlunsplit import httpx +from tests.provider_record_replay.http import ( + dropped_request_headers, + dropped_response_headers, + is_streaming_response, +) from tests.route_parity.recorded_http import ( HttpHeader, RecordedHttpResponse, @@ -21,39 +26,17 @@ from tests.route_parity.recorded_http import ( _PARITY_PROVIDER_HOST: Final = "parity-provider.invalid" -_HOP_BY_HOP_HEADERS: Final = frozenset( - { - "connection", - "keep-alive", - "proxy-authenticate", - "proxy-authorization", - "te", - "trailer", - "transfer-encoding", - "upgrade", - } -) - - InputT = TypeVar("InputT") @dataclass(frozen=True, slots=True) -class ProviderSpec: - upstream_base: str - - -def _excluded_headers(headers: tuple[tuple[str, str], ...]) -> frozenset[str]: - connection_values: Final = tuple(value for name, value in headers if name.lower() == "connection") - connection_headers: Final = frozenset( - token.strip().lower() for value in connection_values for token in value.split(",") if token.strip() - ) - return _HOP_BY_HOP_HEADERS | connection_headers +class UpstreamEndpoint: + base_url: str def _end_to_end_headers(headers: httpx.Headers) -> tuple[HttpHeader, ...]: decoded: Final = tuple((name.decode("ascii"), value.decode("latin-1")) for name, value in headers.raw) - excluded: Final = _excluded_headers(decoded) | {"content-encoding", "content-length"} + excluded: Final = dropped_response_headers(decoded) return tuple( HttpHeader(name=name, value=_normalized_response_header(name, value)) for name, value in decoded @@ -82,7 +65,7 @@ def local_response_header(name: str, value: str, provider_url: str) -> str: class _RecordingProvider(ThreadingHTTPServer): daemon_threads = True - def __init__(self, spec: ProviderSpec) -> None: + def __init__(self, spec: UpstreamEndpoint) -> None: super().__init__(("127.0.0.1", 0), _RecordingHandler) self.spec: Final = spec self.responses: queue.Queue[RecordedResponse] = queue.Queue() @@ -109,15 +92,24 @@ class _RecordingHandler(BaseHTTPRequestHandler): def do_GET(self) -> None: self._forward() + def do_PUT(self) -> None: + self._forward() + + def do_PATCH(self) -> None: + self._forward() + + def do_DELETE(self) -> None: + self._forward() + def _forward(self) -> None: provider: Final = self.server assert isinstance(provider, _RecordingProvider) length: Final = int(self.headers.get("content-length") or "0") request_body: Final = self.rfile.read(length) if length else b"" raw_headers: Final = tuple(self.headers.raw_items()) - excluded: Final = _excluded_headers(raw_headers) | {"host", "content-length"} + excluded: Final = dropped_request_headers(raw_headers) forwarded_headers: Final = tuple((name, value) for name, value in raw_headers if name.lower() not in excluded) - upstream_url: Final = f"{provider.spec.upstream_base.rstrip('/')}{self.path}" + upstream_url: Final = f"{provider.spec.base_url.rstrip('/')}{self.path}" try: with httpx.stream( @@ -145,7 +137,7 @@ class _RecordingHandler(BaseHTTPRequestHandler): headers: tuple[HttpHeader, ...], ) -> RecordedResponse: content_type: Final = cast(str, upstream.headers.get("content-type", "")) - if content_type.lower().startswith("text/event-stream"): + if is_streaming_response(content_type): return self._record_stream(upstream, headers) response_body: Final = b"".join(upstream.iter_bytes()) return RecordedHttpResponse.from_bytes( @@ -199,7 +191,7 @@ class _RecordingHandler(BaseHTTPRequestHandler): @contextmanager -def _recording_provider(spec: ProviderSpec) -> Generator[_RecordingProvider]: +def _recording_provider(spec: UpstreamEndpoint) -> Generator[_RecordingProvider]: server: Final = _RecordingProvider(spec) thread: Final = threading.Thread(target=server.serve_forever, daemon=True) thread.start() @@ -227,7 +219,7 @@ def _invoke_and_take_responses( def record_upstream_responses( - spec: ProviderSpec, + spec: UpstreamEndpoint, case_input: InputT, sdk_call: Callable[[str, InputT], object], ) -> tuple[RecordedResponse, ...]: diff --git a/tests/route_parity/fixtures/store.py b/tests/route_parity/fixtures/store.py index 41f488d45d6..a169d96c835 100644 --- a/tests/route_parity/fixtures/store.py +++ b/tests/route_parity/fixtures/store.py @@ -2,13 +2,11 @@ from __future__ import annotations import hashlib import json -import os -from collections.abc import Callable, Mapping +from collections.abc import Mapping from datetime import datetime, timezone from pathlib import Path from typing import Final, Protocol, TypeVar, cast -import pytest from pydantic import AwareDatetime, BaseModel, ConfigDict, TypeAdapter, ValidationError FIXTURE_SCHEMA_VERSION: Final = 1 @@ -99,56 +97,3 @@ def fixture_id(case_input: FixtureInput, prefix: str) -> str: input_json: Final = canonical_json(case_input.canonical_input()) digest: Final = hashlib.sha256(input_json.encode("utf-8")).hexdigest()[:8] return f"{prefix}-{digest}" - - -def parametrize_recorded_fixtures( - metafunc: pytest.Metafunc, - *, - fixture_name: str, - case_type: type[CaseT], - env_var: str, - default_directory: Path, - regeneration_command: str, - id_builder: Callable[[CaseT], str], - marks_builder: Callable[[CaseT], tuple[pytest.MarkDecorator, ...]] | None = None, -) -> None: - if fixture_name not in metafunc.fixturenames: - return - configured: Final = os.environ.get(env_var) - if configured == "": - raise pytest.UsageError(f"{env_var} is set but empty") - directory: Final = Path(configured).expanduser() if configured is not None else default_directory - try: - fixtures: Final = recorded_fixtures(directory, case_type) - except (ValidationError, ValueError) as error: - raise pytest.UsageError( - f"Invalid parity fixture bundle at {directory}. " - "Each fixture must use the current versioned envelope. " - f"Record fresh fixtures in an empty directory with: `{regeneration_command}`. " - f"Validation details: {error}" - ) from error - if fixtures: - metafunc.parametrize( - fixture_name, - tuple( - pytest.param( - fixture, - id=id_builder(fixture), - marks=marks_builder(fixture) if marks_builder is not None else (), - ) - for fixture in fixtures - ), - ) - return - if configured is not None: - raise pytest.UsageError(f"no recorded fixtures in {directory}") - metafunc.parametrize( - fixture_name, - ( - pytest.param( - None, - marks=pytest.mark.skip(reason=f"no recorded fixtures in {directory}"), - id="no-recorded-fixtures", - ), - ), - ) diff --git a/tests/route_parity/fixtures/test_pipeline.py b/tests/route_parity/fixtures/test_pipeline.py index 52f54f332d5..59366c9adb6 100644 --- a/tests/route_parity/fixtures/test_pipeline.py +++ b/tests/route_parity/fixtures/test_pipeline.py @@ -20,7 +20,7 @@ from tests.route_parity.fixtures.pipeline import ( build_recording_jobs, record_fixtures, ) -from tests.route_parity.fixtures.recording import ProviderSpec +from tests.route_parity.fixtures.recording import UpstreamEndpoint from tests.route_parity.fixtures.store import fixture_path from tests.route_parity.recorded_http import RecordedResponse @@ -117,7 +117,7 @@ def _target( ) -> RecordingTarget[_FixtureInput]: return RecordingTarget( name=name, - provider_spec=ProviderSpec(upstream_base=upstream_url), + upstream=UpstreamEndpoint(base_url=upstream_url), strategy=st.just(case_input), invocation=invocation, required_inputs=(case_input,), @@ -129,7 +129,7 @@ def test_build_jobs_keeps_required_inputs_before_generated_inputs_and_deduplicat generated: Final = _FixtureInput(identifier="generated") target: Final = RecordingTarget( name="ordered", - provider_spec=ProviderSpec(upstream_base="https://provider.invalid"), + upstream=UpstreamEndpoint(base_url="https://provider.invalid"), strategy=st.just(generated), invocation=_Invocation(), required_inputs=(required, required), diff --git a/tests/route_parity/fixtures/test_recording.py b/tests/route_parity/fixtures/test_recording.py index d27b14ee1de..6dc70b7d8a2 100644 --- a/tests/route_parity/fixtures/test_recording.py +++ b/tests/route_parity/fixtures/test_recording.py @@ -14,7 +14,7 @@ from hypothesis import strategies as st from pydantic import BaseModel, ConfigDict from tests.route_parity.fixtures.pipeline import RecordingTarget, record_fixtures -from tests.route_parity.fixtures.recording import ProviderSpec, record_upstream_responses +from tests.route_parity.fixtures.recording import UpstreamEndpoint, record_upstream_responses from tests.route_parity.fixtures.store import ( FIXTURE_SCHEMA_VERSION, fixture_path, @@ -97,7 +97,7 @@ class _ControlledUpstreamHandler(BaseHTTPRequestHandler): length: Final = int(self.headers.get("content-length") or "0") self.rfile.read(length) if self.path == "/upload": - self._send_json(200, b'{"file_id":"reducto://fixture.pdf"}') + self._send_json(200, b'{"file_id":"fixture://document.pdf"}') return if self.path == "/parse": self._send_json(200, b'{"result":{"chunks":[]}}') @@ -131,6 +131,7 @@ class _ControlledUpstreamHandler(BaseHTTPRequestHandler): body: Final = b"{}" self.send_response(200) self.send_header("content-type", "application/json") + self.send_header("set-cookie", "session=must-not-be-recorded") self.send_header("content-length", str(len(body))) self.end_headers() self.wfile.write(body) @@ -143,6 +144,15 @@ class _ControlledUpstreamHandler(BaseHTTPRequestHandler): return self.send_error(404) + def do_PUT(self) -> None: + self.do_POST() + + def do_PATCH(self) -> None: + self.do_POST() + + def do_DELETE(self) -> None: + self.do_POST() + def _send_json(self, status: int, body: bytes) -> None: self.send_response(status) self.send_header("content-type", "application/json") @@ -172,7 +182,7 @@ def _case(identifier: str) -> _FixtureInput: def _sdk_call(api_base: str, case_input: _FixtureInput) -> object: - return httpx.post(f"{api_base}/v1/ocr", content=b"{}", timeout=5) + return httpx.post(f"{api_base}/v1/operation", content=b"{}", timeout=5) def _stream_sdk_call(api_base: str, case_input: _FixtureInput) -> object: @@ -185,6 +195,13 @@ def _error_sdk_call(api_base: str, case_input: _FixtureInput) -> object: return response +def _method_sdk_call(method: str) -> Callable[[str, _FixtureInput], object]: + def call(api_base: str, case_input: _FixtureInput) -> object: + return httpx.request(method, f"{api_base}/method", json={"id": case_input.identifier}, timeout=5) + + return call + + def _multi_sdk_call(api_base: str, case_input: _FixtureInput) -> object: upload: Final = httpx.post(f"{api_base}/upload", json={"document": case_input.identifier}, timeout=5) upload.raise_for_status() @@ -204,18 +221,18 @@ def _polling_sdk_call(api_base: str, case_input: _FixtureInput) -> object: def test_recording_deduplicates_per_target_and_caps_global_concurrency(tmp_path: Path) -> None: shared_input: Final = _case("shared") with _controlled_upstream() as upstream: - spec: Final = ProviderSpec(upstream_base=upstream.url) + spec: Final = UpstreamEndpoint(base_url=upstream.url) targets: Final = ( RecordingTarget( name="first", - provider_spec=spec, + upstream=spec, strategy=st.just(shared_input), invocation=_Invocation(_sdk_call), required_inputs=(shared_input, shared_input), ), RecordingTarget( name="second", - provider_spec=spec, + upstream=spec, strategy=st.just(shared_input), invocation=_Invocation(_sdk_call), required_inputs=(shared_input,), @@ -244,7 +261,7 @@ def test_pipeline_rejects_stale_fixture_before_provider_call(tmp_path: Path) -> path.write_text('{"schema_version": 0}\n', encoding="utf-8") target: Final = RecordingTarget( name="stale-target", - provider_spec=ProviderSpec(upstream_base="http://127.0.0.1:1"), + upstream=UpstreamEndpoint(base_url="http://127.0.0.1:1"), strategy=st.just(case_input), invocation=_Invocation(_sdk_call), ) @@ -271,7 +288,7 @@ def test_cached_fixture_is_reported_without_provider_call(tmp_path: Path) -> Non with _controlled_upstream() as upstream: target: Final = RecordingTarget( name="cached-target", - provider_spec=ProviderSpec(upstream_base=upstream.url), + upstream=UpstreamEndpoint(base_url=upstream.url), strategy=st.just(case_input), invocation=_Invocation(_sdk_call), ) @@ -286,7 +303,7 @@ def test_cached_fixture_is_reported_without_provider_call(tmp_path: Path) -> Non def test_streaming_response_records_and_replays_chunks() -> None: with _controlled_upstream() as upstream: responses: Final = record_upstream_responses( - ProviderSpec(upstream_base=upstream.url), + UpstreamEndpoint(base_url=upstream.url), _case("stream"), _stream_sdk_call, ) @@ -308,7 +325,7 @@ def test_streaming_response_records_and_replays_chunks() -> None: def test_non_successful_provider_response_is_recorded() -> None: with _controlled_upstream() as upstream: responses: Final = record_upstream_responses( - ProviderSpec(upstream_base=upstream.url), + UpstreamEndpoint(base_url=upstream.url), _case("provider-error"), _error_sdk_call, ) @@ -317,6 +334,34 @@ def test_non_successful_provider_response_is_recorded() -> None: assert response.status_code == 429 +def test_sensitive_response_headers_are_not_recorded() -> None: + with _controlled_upstream() as upstream: + responses: Final = record_upstream_responses( + UpstreamEndpoint(base_url=upstream.url), + _case("headers"), + _sdk_call, + ) + + assert all(header.name.lower() != "set-cookie" for header in responses[0].headers) + + +@pytest.mark.parametrize("method", ("PUT", "PATCH", "DELETE")) +def test_recording_and_replay_support_mutating_http_methods(method: str) -> None: + sdk_call: Final = _method_sdk_call(method) + with _controlled_upstream() as upstream: + responses: Final = record_upstream_responses( + UpstreamEndpoint(base_url=upstream.url), + _case(method), + sdk_call, + ) + with replay_server() as provider: + provider.enqueue_response(responses[0]) + sdk_call(provider.url, _case(method)) + requests: Final = provider.take_requests(1) + + assert requests[0].method == method + + def test_stream_response_model_rejects_buffered_body() -> None: with pytest.raises(ValueError, match="Extra inputs are not permitted"): RecordedHttpStreamResponse.model_validate( @@ -336,7 +381,7 @@ def test_multiple_provider_calls_record_and_replay_in_order( ) -> None: with _controlled_upstream() as upstream: responses: Final = record_upstream_responses( - ProviderSpec(upstream_base=upstream.url), + UpstreamEndpoint(base_url=upstream.url), _case(sdk_call.__name__), sdk_call, ) diff --git a/tests/route_parity/replay.py b/tests/route_parity/replay.py index 9a185ead100..91a5e110393 100644 --- a/tests/route_parity/replay.py +++ b/tests/route_parity/replay.py @@ -65,6 +65,15 @@ class _ReplayHandler(BaseHTTPRequestHandler): def do_GET(self) -> None: self._replay() + def do_PUT(self) -> None: + self._replay() + + def do_PATCH(self) -> None: + self._replay() + + def do_DELETE(self) -> None: + self._replay() + def _replay(self) -> None: provider: Final = self.server assert isinstance(provider, ReplayServer) diff --git a/tests/route_parity/runner.py b/tests/route_parity/runner.py index f5eb5a39eb2..1b8d6242201 100644 --- a/tests/route_parity/runner.py +++ b/tests/route_parity/runner.py @@ -29,10 +29,9 @@ WORKER_RESULT_ADAPTER: Final[TypeAdapter[WorkerResult]] = TypeAdapter(WorkerResu @dataclass(frozen=True, slots=True) -class PythonScriptRunner: +class SubprocessRunner: entrypoint: Path - rust_env_var: str - python_user_agent: str + baseline_user_agent: str route_label: str def command(self, provider_url: str) -> tuple[str, ...]: @@ -44,17 +43,23 @@ class PythonScriptRunner: ) -class PythonScriptWorker: - def __init__(self, runner: PythonScriptRunner, provider: ReplayServer, rust_enabled: bool) -> None: +@dataclass(frozen=True, slots=True) +class ExecutionVariant: + name: str + environment: tuple[tuple[str, str], ...] + + +class SubprocessWorker: + def __init__(self, runner: SubprocessRunner, provider: ReplayServer, variant: ExecutionVariant) -> None: project_root: Final = str(runner.entrypoint.resolve().parents[3]) existing_pythonpath: Final = os.environ.get("PYTHONPATH") env: Final = { **os.environ, - runner.rust_env_var: "1" if rust_enabled else "0", - "LITELLM_USER_AGENT": runner.python_user_agent, + **dict(variant.environment), + "LITELLM_USER_AGENT": runner.baseline_user_agent, "PYTHONPATH": os.pathsep.join(path for path in (project_root, existing_pythonpath) if path), } - self.mode: Final = "Rust" if rust_enabled else "Python" + self.mode: Final = variant.name self.route_label: Final = runner.route_label self.provider: Final = provider self.process: Final = subprocess.Popen( @@ -146,11 +151,11 @@ class PythonScriptWorker: @contextmanager def execution_worker( - runner: PythonScriptRunner, - rust_enabled: bool, -) -> Generator[PythonScriptWorker]: + runner: SubprocessRunner, + variant: ExecutionVariant, +) -> Generator[SubprocessWorker]: with replay_server() as provider: - worker: Final = PythonScriptWorker(runner, provider, rust_enabled) + worker: Final = SubprocessWorker(runner, provider, variant) try: yield worker finally: @@ -158,7 +163,7 @@ def execution_worker( def run_execution( - worker: PythonScriptWorker, + worker: SubprocessWorker, case_file: Path, route: str, responses: tuple[RecordedResponse, ...], @@ -168,11 +173,13 @@ def run_execution( @contextmanager def execution_worker_pair( - runner: PythonScriptRunner, -) -> Generator[tuple[PythonScriptWorker, PythonScriptWorker]]: - with execution_worker(runner, rust_enabled=False) as python_worker: - with execution_worker(runner, rust_enabled=True) as accelerated_worker: - yield python_worker, accelerated_worker + runner: SubprocessRunner, + baseline: ExecutionVariant, + candidate: ExecutionVariant, +) -> Generator[tuple[SubprocessWorker, SubprocessWorker]]: + with execution_worker(runner, baseline) as baseline_worker: + with execution_worker(runner, candidate) as candidate_worker: + yield baseline_worker, candidate_worker def parity_worker_main( diff --git a/tests/route_parity/stream.py b/tests/route_parity/stream.py index 5f3352dc6d3..a85ff2dbf67 100644 --- a/tests/route_parity/stream.py +++ b/tests/route_parity/stream.py @@ -185,16 +185,16 @@ def normalize_chunk(chunk: object) -> object: def assert_stream_parity( - python: StreamOutcome, - accelerated: StreamOutcome, + baseline: StreamOutcome, + candidate: StreamOutcome, *, normalize: ChunkNormalizer = normalize_chunk, ) -> None: - assert python.wrapper_type is accelerated.wrapper_type - assert python.supports_sync_iteration is accelerated.supports_sync_iteration - assert python.supports_async_iteration is accelerated.supports_async_iteration - assert python.chunk_types == accelerated.chunk_types - assert len(python.chunks) == len(accelerated.chunks) - for python_chunk, accelerated_chunk in zip(python.chunks, accelerated.chunks, strict=True): - assert normalize(python_chunk) == normalize(accelerated_chunk) - assert python.terminal == accelerated.terminal + assert baseline.wrapper_type is candidate.wrapper_type + assert baseline.supports_sync_iteration is candidate.supports_sync_iteration + assert baseline.supports_async_iteration is candidate.supports_async_iteration + assert baseline.chunk_types == candidate.chunk_types + assert len(baseline.chunks) == len(candidate.chunks) + for baseline_chunk, candidate_chunk in zip(baseline.chunks, candidate.chunks, strict=True): + assert normalize(baseline_chunk) == normalize(candidate_chunk) + assert baseline.terminal == candidate.terminal diff --git a/tests/route_parity/test_parity.py b/tests/route_parity/test_parity.py index 6aa362fe21f..b95cccbb9e9 100644 --- a/tests/route_parity/test_parity.py +++ b/tests/route_parity/test_parity.py @@ -23,6 +23,10 @@ class _DifferentResponse(BaseModel): value: str +class _FloatResponse(BaseModel): + values: list[float] + + class _PublicError(ValueError): status_code: Final = 400 @@ -69,7 +73,7 @@ def test_parity_rejects_error_difference() -> None: error_type=None, param=None, model="test-model", - llm_provider="mistral", + llm_provider="test-provider", ), ) rust: Final = python.model_copy(update={"report": python.report.model_copy(update={"status_code": 500})}) @@ -116,3 +120,19 @@ def test_model_parity_rejects_public_value_difference() -> None: def test_model_parity_rejects_type_difference() -> None: with pytest.raises(AssertionError): assert_model_parity(_ComparableResponse(value="same"), _DifferentResponse(value="same")) + + +def test_model_parity_rejects_wire_float_rounding_difference() -> None: + with pytest.raises(AssertionError, match=r"\$\.values\[0\]"): + assert_model_parity( + _FloatResponse(values=[0.22590550796036835]), + _FloatResponse(values=[0.22590550796036837]), + ) + + +def test_model_parity_rejects_meaningful_float_difference() -> None: + with pytest.raises(AssertionError, match=r"\$\.values\[0\]"): + assert_model_parity( + _FloatResponse(values=[0.22590550796036835]), + _FloatResponse(values=[0.2259]), + ) diff --git a/tests/test_litellm/ocr/conftest.py b/tests/test_litellm/ocr/conftest.py index 73a2c12e0c3..a32963a5ab6 100644 --- a/tests/test_litellm/ocr/conftest.py +++ b/tests/test_litellm/ocr/conftest.py @@ -1,15 +1,14 @@ from __future__ import annotations -from pathlib import Path from typing import Final import pytest -from tests.route_parity.fixtures.store import fixture_id, parametrize_recorded_fixtures +from tests.route_parity.fixtures.pytest_support import parametrize_recorded_fixtures +from tests.route_parity.fixtures.store import fixture_id +from tests.test_litellm.ocr.fixtures.config import DEFAULT_FIXTURE_DIRECTORY, FIXTURE_DIR_ENV from tests.test_litellm.ocr.fixtures.models import OcrParityCase -FIXTURE_DIR_ENV: Final = "LITELLM_OCR_FIXTURE_DIR" - def ocr_fixture_id(fixture: OcrParityCase) -> str: case_input: Final = fixture.litellm_input @@ -19,26 +18,25 @@ def ocr_fixture_id(fixture: OcrParityCase) -> str: def ocr_fixture_marks(fixture: OcrParityCase) -> tuple[pytest.MarkDecorator, ...]: - if fixture.litellm_input.boundary not in {"reducto_v3", "reducto_legacy"}: + if fixture.litellm_input.contract not in {"reducto_v3", "reducto_legacy"}: return () return ( pytest.mark.xfail( - reason="Reducto does not have a Rust OCR boundary", + reason="Reducto does not have a Rust OCR contract", strict=False, ), ) def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: - default_directory: Final = Path(__file__).with_name("fixtures") / "data" parametrize_recorded_fixtures( metafunc, fixture_name="ocr_fixture", case_type=OcrParityCase, env_var=FIXTURE_DIR_ENV, - default_directory=default_directory, + default_directory=DEFAULT_FIXTURE_DIRECTORY, regeneration_command=( - f"uv run python -m tests.test_litellm.ocr.fixtures.record --fixture-dir {default_directory}" + f"uv run python -m tests.test_litellm.ocr.fixtures.record --fixture-dir {DEFAULT_FIXTURE_DIRECTORY}" ), id_builder=ocr_fixture_id, marks_builder=ocr_fixture_marks, diff --git a/tests/test_litellm/ocr/fixtures/README.md b/tests/test_litellm/ocr/fixtures/README.md new file mode 100644 index 00000000000..7071467218c --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/README.md @@ -0,0 +1,35 @@ +# OCR parity fixtures + +The recording command runs four stages: + +1. Generate deterministic SDK inputs for every configured OCR target +2. Build target-scoped, deduplicated recording jobs +3. Record upstream responses through one globally bounded worker pool +4. Persist each fixture and report whether it was recorded, cached, or failed + +Run it with: + +```shell +uv run python -m tests.test_litellm.ocr.fixtures.record --examples 4 --concurrency 4 +``` + +`--concurrency` caps provider calls across all targets. Independent jobs finish after a failure, then the command exits +nonzero if any job failed + +OCR strategies generate public `litellm.ocr()` and `litellm.aocr()` inputs. Every case contains the normalized model, +document, optional provider override, and LiteLLM keyword arguments. The fixture-only `contract` literal selects the +input schema and is removed before calling the SDK. Strategies never build provider wire payloads + +Each contract has a required corpus containing a baseline and cases for its supported top-level OCR parameters. The +contracts are Mistral, Azure-hosted Mistral, Vertex-hosted Mistral, Azure Document Intelligence, Vertex DeepSeek, +Reducto v3, and Reducto legacy. Credentials and endpoints only control target discovery, so a machine records the +contracts it has configured and skips the rest + +Reducto fixtures record upload and parse responses. Their parity cases remain non-strict expected failures until the +Rust OCR bridge supports Reducto. Azure and Vertex generation paths are unit-tested without credentials in CI, so the +committed corpus does not need live recordings for every target + +Every recording target owns a small fixed provider-rejected corpus, independent of replay implementation support. +Those inputs are recorded separately from generated valid inputs. Local validation failures use no recorded response; +the parity suite checks those unsupported providers and models, malformed documents, invalid request formats, invalid +Azure Document Intelligence parameters, and invalid headers in sync and async SDK calls diff --git a/tests/test_litellm/ocr/fixtures/azure.py b/tests/test_litellm/ocr/fixtures/azure.py index bb7f54632a1..50521a9c0e8 100644 --- a/tests/test_litellm/ocr/fixtures/azure.py +++ b/tests/test_litellm/ocr/fixtures/azure.py @@ -5,20 +5,16 @@ from typing import Final, Literal, cast from hypothesis import strategies as st from hypothesis.strategies import SearchStrategy -from pydantic import Field, StrictInt, StrictStr, field_validator +from pydantic import StrictInt, StrictStr, field_validator -from tests.route_parity.fixtures.recording import ProviderSpec +from tests.route_parity.fixtures.recording import UpstreamEndpoint from tests.test_litellm.ocr.fixtures.base import OcrDocument, OcrSdkInputBase from tests.test_litellm.ocr.fixtures.common import ( OcrFixtureClient, OcrRecordingTarget, image_document, invoke_with_api_key, - parameter_strategy, pdf_document, - sampled_list_strategy, - sampled_parameter_group_strategy, - sampled_scalar_strategy, ) from tests.test_litellm.ocr.fixtures.mistral import ( MistralCompatibleOcrSdkInput, @@ -26,11 +22,15 @@ from tests.test_litellm.ocr.fixtures.mistral import ( ) AzureMistralModel = Literal["azure_ai/mistral-document-ai-2512",] +AzureMistralFixtureModel = AzureMistralModel | Literal["azure_ai/invalid-ocr-model-for-parity"] AzureDocumentIntelligenceModel = Literal[ "azure_ai/doc-intelligence/prebuilt-read", "azure_ai/doc-intelligence/prebuilt-layout", "azure_ai/doc-intelligence/prebuilt-document", ] +AzureDocumentIntelligenceFixtureModel = ( + AzureDocumentIntelligenceModel | Literal["azure_ai/doc-intelligence/invalid-ocr-model-for-parity"] +) AZURE_MISTRAL_MODELS: Final[tuple[AzureMistralModel, ...]] = ("azure_ai/mistral-document-ai-2512",) AZURE_DOCUMENT_INTELLIGENCE_MODELS: Final[tuple[AzureDocumentIntelligenceModel, ...]] = ( @@ -47,8 +47,8 @@ AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS: Final[tuple[AzureDocumentIntellige class AzureMistralOcrSdkInput(MistralCompatibleOcrSdkInput): - boundary: str = Field(default="azure_mistral", pattern=r"^azure_mistral$") - model: AzureMistralModel + contract: Literal["azure_mistral"] = "azure_mistral" + model: AzureMistralFixtureModel custom_llm_provider: Literal["azure_ai"] | None = None @field_validator("model") @@ -60,8 +60,8 @@ class AzureMistralOcrSdkInput(MistralCompatibleOcrSdkInput): class AzureDocumentIntelligenceOcrSdkInput(OcrSdkInputBase): - boundary: str = Field(default="azure_document_intelligence", pattern=r"^azure_document_intelligence$") - model: AzureDocumentIntelligenceModel + contract: Literal["azure_document_intelligence"] = "azure_document_intelligence" + model: AzureDocumentIntelligenceFixtureModel document: OcrDocument custom_llm_provider: Literal["azure_ai"] | None = None pages: str | list[StrictInt] | list[StrictStr] | None = None @@ -69,6 +69,20 @@ class AzureDocumentIntelligenceOcrSdkInput(OcrSdkInputBase): req_format: Literal["litellm"] = "litellm" +AZURE_MISTRAL_PROVIDER_REJECTED_INPUTS: Final[tuple[AzureMistralOcrSdkInput, ...]] = ( + AzureMistralOcrSdkInput( + model="azure_ai/invalid-ocr-model-for-parity", + document=pdf_document(), + ), +) +AZURE_DOCUMENT_INTELLIGENCE_PROVIDER_REJECTED_INPUTS: Final[tuple[AzureDocumentIntelligenceOcrSdkInput, ...]] = ( + AzureDocumentIntelligenceOcrSdkInput( + model="azure_ai/doc-intelligence/invalid-ocr-model-for-parity", + document=pdf_document(), + ), +) + + def _azure_mistral_input(values: dict[str, object], model: AzureMistralModel) -> AzureMistralOcrSdkInput: return AzureMistralOcrSdkInput.model_validate({**values, "model": model}) @@ -79,7 +93,7 @@ def azure_mistral_input_strategy(inline_image_data_uri: str) -> SearchStrategy[A return st.builds( _azure_mistral_input, values=mistral_input_values_strategy("2505", inline_image_data_uri, include_document_annotation_prompt=False), - model=sampled_scalar_strategy(AZURE_MISTRAL_MODELS), + model=st.sampled_from(AZURE_MISTRAL_MODELS), ) @@ -91,7 +105,7 @@ _AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL: Final[AzureDocumentIntelligenceMod def _document_intelligence_input( model: AzureDocumentIntelligenceModel, document: OcrDocument, - optional_params: dict[str, object] | None = None, + optional_params: Mapping[str, object] | None = None, ) -> AzureDocumentIntelligenceOcrSdkInput: return AzureDocumentIntelligenceOcrSdkInput.model_validate( {"model": model, "document": document, **(optional_params or {})} @@ -100,33 +114,25 @@ def _document_intelligence_input( def azure_document_intelligence_input_strategy() -> SearchStrategy[AzureDocumentIntelligenceOcrSdkInput]: document: Final = pdf_document() - pages: Final = parameter_strategy( - "pages", - st.one_of( - sampled_list_strategy(((0,), (2, 0, 0, 1))), - sampled_list_strategy((("1", "2-4"),)), - sampled_scalar_strategy(("1-4, 5",)), - ), - ) - features: Final = parameter_strategy( - "features", - st.one_of( - sampled_list_strategy( - ( - ("languages",), - ("ocrHighResolution",), - ("barcodes",), - ("formulas",), - ("styleFont",), - ("keyValuePairs",), - ) - ), - sampled_scalar_strategy(("languages, styleFont",)), - ), - ) - combined_query: Final = sampled_parameter_group_strategy( - ((("pages", (0, 1)), ("features", ("languages", "styleFont"))),) - ) + pages: Final = st.one_of( + st.sampled_from(((0,), (2, 0, 0, 1))).map(list), + st.just(["1", "2-4"]), + st.just("1-4, 5"), + ).map(lambda value: {"pages": value}) + features: Final = st.one_of( + st.sampled_from( + ( + ("languages",), + ("ocrHighResolution",), + ("barcodes",), + ("formulas",), + ("styleFont",), + ("keyValuePairs",), + ) + ).map(list), + st.just("languages, styleFont"), + ).map(lambda value: {"features": value}) + combined_query: Final = st.just({"pages": (0, 1), "features": ("languages", "styleFont")}) return st.one_of( st.sampled_from(AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS).map( lambda model: _document_intelligence_input(model, document) @@ -166,18 +172,19 @@ def azure_mistral_recording_targets( environ: Mapping[str, str], client: OcrFixtureClient, inline_image_data_uri: str ) -> tuple[OcrRecordingTarget, ...]: api_key: Final = environ.get("AZURE_AI_API_KEY") - upstream_base: Final = environ.get("AZURE_AI_API_BASE") - if not api_key or not upstream_base: + base_url: Final = environ.get("AZURE_AI_API_BASE") + if not api_key or not base_url: return () return ( OcrRecordingTarget( name="azure-mistral", - provider_spec=ProviderSpec(upstream_base=upstream_base.rstrip("/")), + upstream=UpstreamEndpoint(base_url=base_url.rstrip("/")), strategy=cast( SearchStrategy[OcrSdkInputBase], azure_mistral_input_strategy(inline_image_data_uri), ), invocation=invoke_with_api_key(client, api_key), + required_inputs=AZURE_MISTRAL_PROVIDER_REJECTED_INPUTS, ), ) @@ -186,14 +193,15 @@ def azure_document_intelligence_recording_targets( environ: Mapping[str, str], client: OcrFixtureClient ) -> tuple[OcrRecordingTarget, ...]: api_key: Final = environ.get("AZURE_DOCUMENT_INTELLIGENCE_API_KEY") - upstream_base: Final = environ.get("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT") - if not api_key or not upstream_base: + base_url: Final = environ.get("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT") + if not api_key or not base_url: return () return ( OcrRecordingTarget( name="azure-document-intelligence", - provider_spec=ProviderSpec(upstream_base=upstream_base.rstrip("/")), + upstream=UpstreamEndpoint(base_url=base_url.rstrip("/")), strategy=cast(SearchStrategy[OcrSdkInputBase], azure_document_intelligence_input_strategy()), invocation=invoke_with_api_key(client, api_key), + required_inputs=AZURE_DOCUMENT_INTELLIGENCE_PROVIDER_REJECTED_INPUTS, ), ) diff --git a/tests/test_litellm/ocr/fixtures/base.py b/tests/test_litellm/ocr/fixtures/base.py index b75b901faaf..71a3298cd74 100644 --- a/tests/test_litellm/ocr/fixtures/base.py +++ b/tests/test_litellm/ocr/fixtures/base.py @@ -21,7 +21,9 @@ __all__ = ( "OcrSdkInputBase", ) -OcrSdkInputBase = SdkInputBase + +class OcrSdkInputBase(SdkInputBase): + fixture_only_fields = ("contract",) class ImageUrlValue(FixtureModel): diff --git a/tests/test_litellm/ocr/fixtures/common.py b/tests/test_litellm/ocr/fixtures/common.py index d9733891e2c..3aa1ed61676 100644 --- a/tests/test_litellm/ocr/fixtures/common.py +++ b/tests/test_litellm/ocr/fixtures/common.py @@ -2,13 +2,13 @@ from __future__ import annotations from dataclasses import dataclass, field from functools import cache -from typing import Final, Literal, Protocol, TypeVar +from typing import Final, Literal, Protocol from hypothesis import strategies as st from hypothesis.strategies import SearchStrategy -from tests.route_parity.fixtures.media import dummy_image_url, structured_pdf_data_uri from tests.route_parity.fixtures.pipeline import RecordingTarget +from tests.route_parity.fixtures.media import dummy_image_url, structured_pdf_data_uri from tests.test_litellm.ocr.fixtures.base import ( DocumentUrlDocument, ImageUrlDocument, @@ -18,7 +18,6 @@ from tests.test_litellm.ocr.fixtures.base import ( ) OcrRecordingTarget = RecordingTarget[OcrSdkInputBase] -ValueT = TypeVar("ValueT") class OcrFixtureClient(Protocol): @@ -81,27 +80,6 @@ def document_transport_strategy(inline_image_data_uri: str) -> SearchStrategy[Im return st.sampled_from(transports).map(as_document) -def sampled_scalar_strategy(values: tuple[ValueT, ...]) -> SearchStrategy[ValueT]: - return st.sampled_from(values) - - -def sampled_list_strategy(values: tuple[tuple[ValueT, ...], ...]) -> SearchStrategy[list[ValueT]]: - return st.sampled_from(values).map(list) - - -def sampled_parameter_group_strategy( - values: tuple[tuple[tuple[str, object], ...], ...], -) -> SearchStrategy[dict[str, object]]: - return st.sampled_from(values).map(dict) - - -def parameter_strategy(name: str, values: SearchStrategy[ValueT]) -> SearchStrategy[dict[str, object]]: - def as_parameter(value: ValueT) -> dict[str, object]: - return {name: value} - - return values.map(as_parameter) - - def annotation_format(name: str) -> JsonSchemaResponseFormat: return JsonSchemaResponseFormat( type="json_schema", diff --git a/tests/test_litellm/ocr/fixtures/config.py b/tests/test_litellm/ocr/fixtures/config.py new file mode 100644 index 00000000000..fe5cb566e36 --- /dev/null +++ b/tests/test_litellm/ocr/fixtures/config.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +import os +from pathlib import Path +from typing import Final + +FIXTURE_DIR_ENV: Final = "LITELLM_OCR_FIXTURE_DIR" +DEFAULT_FIXTURE_DIRECTORY: Final = Path(__file__).with_name("data") + + +def configured_fixture_directory() -> Path: + configured: Final = os.environ.get(FIXTURE_DIR_ENV) + return Path(configured).expanduser() if configured is not None else DEFAULT_FIXTURE_DIRECTORY diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/0ab748e6b7315f0c7a36195cc4564eff8b03523dfa908ea59ff1bb7bd185ede8.json b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/0ab748e6b7315f0c7a36195cc4564eff8b03523dfa908ea59ff1bb7bd185ede8.json index ee8b1734500..c64c248ab9c 100644 --- a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/0ab748e6b7315f0c7a36195cc4564eff8b03523dfa908ea59ff1bb7bd185ede8.json +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/0ab748e6b7315f0c7a36195cc4564eff8b03523dfa908ea59ff1bb7bd185ede8.json @@ -67,10 +67,6 @@ "name": "Content-Type", "value": "application/json" }, - { - "name": "set-cookie", - "value": "__cf_bm=c5QAleICyoEptgycuGlRuItpEqcNe2MRZASOE9DeW80-1788292466.0205042-1.0.1.1-_PjnHuYd9Ba1XjxpUtrCWo_gUI5uKAvA73jcZAj6t_RARG9jiuX8N3t6bXu7K15RP3WG6AYFN4hbSKwlEpfksa9krmM7JYpf_JEpS5GDbst5vpFFLNtv.ymdX8lDSb5a; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:26 GMT" - }, { "name": "mistral-correlation-id", "value": "01a05e89-85bd-7075-af53-8e341b35a63e" diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/0c9204f5ed668c628835b9d60806a6cea47e1f67c0f48c9804cb09d1c4695f42.json b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/0c9204f5ed668c628835b9d60806a6cea47e1f67c0f48c9804cb09d1c4695f42.json index ab71aaa4507..d267cfa4515 100644 --- a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/0c9204f5ed668c628835b9d60806a6cea47e1f67c0f48c9804cb09d1c4695f42.json +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/0c9204f5ed668c628835b9d60806a6cea47e1f67c0f48c9804cb09d1c4695f42.json @@ -64,10 +64,6 @@ "name": "cf-cache-status", "value": "DYNAMIC" }, - { - "name": "set-cookie", - "value": "__cf_bm=guezPmBNpI0djDo3Kvw0miZywJszLF5AN8loY9Uma0E-1788292459.497073-1.0.1.1-meRuKWGaBsHhkiA58SnRIUHMOaMgSPVzPrmAuu4_xq4UqYETqJn06.dJRx2xrVcoseTuUZNKdyRxTbkz2pdMRMe2AMQWaAMXXW27bG1VkapsA2hW1KtB48F3ubpGbp0v; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:20 GMT" - }, { "name": "Strict-Transport-Security", "value": "max-age=15552000; includeSubDomains; preload" diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/216aacc1aa47a6060445f2ea49a2e0a8f4a8bde179faaaa28f39b049e7886f07.json b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/216aacc1aa47a6060445f2ea49a2e0a8f4a8bde179faaaa28f39b049e7886f07.json index ef889c69c0b..90c2a7e4793 100644 --- a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/216aacc1aa47a6060445f2ea49a2e0a8f4a8bde179faaaa28f39b049e7886f07.json +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/216aacc1aa47a6060445f2ea49a2e0a8f4a8bde179faaaa28f39b049e7886f07.json @@ -20,10 +20,6 @@ "name": "Content-Type", "value": "application/json" }, - { - "name": "set-cookie", - "value": "__cf_bm=XcXH8M77UDLCt6rebhVaPfmyZAa4b4OQ6qSuzN8zcxQ-1788292460.477192-1.0.1.1-jIqoIYlVPq1KWC06tEhpWZ2IGqr1TQU1jPokZk1wkK2UVb0L0LVboaLWA9UnCvYLvy5JzgUf1srFJQlzr7edR8HiWMaxsHfRsCVCPVlCr3H6S_Pikt3AqQmlW8xWM9vh; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:21 GMT" - }, { "name": "mistral-correlation-id", "value": "01a05e89-7015-7a8d-a18c-37b9ceacbe77" diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/2c893c555fe38cdfa0f03b43b5e678b8acd5c2dc460cd06f5302a0bf05f3089c.json b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/2c893c555fe38cdfa0f03b43b5e678b8acd5c2dc460cd06f5302a0bf05f3089c.json index 9078a82ffcb..825569a1936 100644 --- a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/2c893c555fe38cdfa0f03b43b5e678b8acd5c2dc460cd06f5302a0bf05f3089c.json +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/2c893c555fe38cdfa0f03b43b5e678b8acd5c2dc460cd06f5302a0bf05f3089c.json @@ -20,10 +20,6 @@ "name": "Content-Type", "value": "application/json" }, - { - "name": "set-cookie", - "value": "__cf_bm=zdJ57Tl12b3GlmPJbN9IJjqlrvpX6J37Pmt6.Jj4vEs-1788292454.9590237-1.0.1.1-KS5i0m8mBhan9C3JxsJ488KsWHHpjYMz7ASc2SsaJ9MdFLx6JQWyViWZcw3ohhK1y_8a7DwExnKFQEIkFvF8J6HRVhoD2qaYjFW7DHz3t.2DBZunmBgvOouYMcjJyi7X; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:15 GMT" - }, { "name": "mistral-correlation-id", "value": "01a05e89-5a87-7148-8ae4-4e14967bebf3" diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/3832acde595e59e0b3f663c560c8bf3940c6eddbe486e66ad40e18974464eda7.json b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/3832acde595e59e0b3f663c560c8bf3940c6eddbe486e66ad40e18974464eda7.json index b20830fc1d1..3955a0e14c5 100644 --- a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/3832acde595e59e0b3f663c560c8bf3940c6eddbe486e66ad40e18974464eda7.json +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/3832acde595e59e0b3f663c560c8bf3940c6eddbe486e66ad40e18974464eda7.json @@ -51,10 +51,6 @@ "name": "cf-cache-status", "value": "DYNAMIC" }, - { - "name": "set-cookie", - "value": "__cf_bm=UKi98pAYwl6ufVXdgT8WLzMQP_BLupXglkclkyRQJyw-1788306848.2623396-1.0.1.1-oc0jGJ7AsfTQjZuaW08y5nxSJ9fHZS0mt2dij5uO_lA7CeUprSoHUJzIhyVKjI9A2oUM313oVsj6kWfBnqiVnLSUPAf_BMsUqLOkEJtUSMHrIvzkmRCZqBTUJgAqNlK4; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Wed, 02 Sep 2026 00:24:08 GMT" - }, { "name": "Strict-Transport-Security", "value": "max-age=15552000; includeSubDomains; preload" diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/3e4f8a006942b85eb954833d42cc19740883cf42bb87221eafd16df3e597ceda.json b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/3e4f8a006942b85eb954833d42cc19740883cf42bb87221eafd16df3e597ceda.json index 54cf3a8e443..e850ffe1a9e 100644 --- a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/3e4f8a006942b85eb954833d42cc19740883cf42bb87221eafd16df3e597ceda.json +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/3e4f8a006942b85eb954833d42cc19740883cf42bb87221eafd16df3e597ceda.json @@ -39,10 +39,6 @@ "name": "Content-Type", "value": "application/json" }, - { - "name": "set-cookie", - "value": "__cf_bm=VjsfRKiwC7YjbVgfHbbpQfMBCW9MC1ehUXU3LsRZeUY-1788292455.9575524-1.0.1.1-KL3St9l4QfN1eWLUV07XDYmwE0xr2.26Z85yKf1yBMTDFCHBnz_3_RP.oNw6p.GfJsd2mnKb2lOvgPQcBDZygv7yJhN41wN87pIkAvurdr1z_snzwBpofXTvcjibOV1K; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:16 GMT" - }, { "name": "mistral-correlation-id", "value": "01a05e89-5e6e-70ac-890d-c0110a5e81af" diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/49328aed7d9a4833427d80c7ac4f54fc03ab0db9076004caf5262ce3fed4f06b.json b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/49328aed7d9a4833427d80c7ac4f54fc03ab0db9076004caf5262ce3fed4f06b.json index edf15aa1839..5c5284f6908 100644 --- a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/49328aed7d9a4833427d80c7ac4f54fc03ab0db9076004caf5262ce3fed4f06b.json +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/49328aed7d9a4833427d80c7ac4f54fc03ab0db9076004caf5262ce3fed4f06b.json @@ -19,10 +19,6 @@ "name": "Content-Type", "value": "application/json" }, - { - "name": "set-cookie", - "value": "__cf_bm=0BJujMXEX2uflOAjZGqeTJi9lXNgGJmX61YbHtpbRKI-1788292452.9600976-1.0.1.1-7idLuOz.nd3kpO10lpC8IkPMArZ8M9iDs13Z.kTwKLTBHVm4.lyUeSqedg4EQV_Mo3D6JLf6TJgvdhWHsemZe6NyIzTBQwA3cwyM0cMQ5Aw68sVgtDOuGBaqhtelZ0RW; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:13 GMT" - }, { "name": "mistral-correlation-id", "value": "01a05e89-52b4-7e7b-983e-5bc4e5469b39" diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/67a074c43483176b6a8da74f21869eb5c13a7327ed36f77a40c0b192c3a1bf4f.json b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/67a074c43483176b6a8da74f21869eb5c13a7327ed36f77a40c0b192c3a1bf4f.json index 9c7162f0ff6..b41ca54e61d 100644 --- a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/67a074c43483176b6a8da74f21869eb5c13a7327ed36f77a40c0b192c3a1bf4f.json +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/67a074c43483176b6a8da74f21869eb5c13a7327ed36f77a40c0b192c3a1bf4f.json @@ -51,10 +51,6 @@ "name": "Content-Type", "value": "application/json" }, - { - "name": "set-cookie", - "value": "__cf_bm=sPN0c81343UATX4OL40rLiXwZzjj6.f9wiR.sUEwidI-1788292464.0167594-1.0.1.1-1I1p0tYuzSZefNGWXeuwPPCH2oi83ZL7wHVVU_skvYhidQGDYesdwH32zscGrLGoxF1LPkg9h4yruaTw9Pai6ahKkyPofBTXU4pjatWxIYJOV8ILJQRwjJAY9r5x01g5; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:24 GMT" - }, { "name": "mistral-correlation-id", "value": "01a05e89-7deb-72b4-976d-4c244b056782" diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/9f220bfe8a2042fa74e1dada33facf8c80e20574f6ef9782997eaef9adc10e0b.json b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/9f220bfe8a2042fa74e1dada33facf8c80e20574f6ef9782997eaef9adc10e0b.json index 82a0edac1d9..2df35ce95fb 100644 --- a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/9f220bfe8a2042fa74e1dada33facf8c80e20574f6ef9782997eaef9adc10e0b.json +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/9f220bfe8a2042fa74e1dada33facf8c80e20574f6ef9782997eaef9adc10e0b.json @@ -64,10 +64,6 @@ "name": "cf-cache-status", "value": "DYNAMIC" }, - { - "name": "set-cookie", - "value": "__cf_bm=Ld7W50uqSPv414iFBL3xHBo1US6FgEGi176rm3Ir7jU-1788292455.4420552-1.0.1.1-U1t1LZ3WH0stmljgl3EZ7Q.vhOhxNGrc3Zj75kgYaDNZCxuwaMc55Khvf.CeGempT.Y5ajCYMNsBwGv0h3rJ4PtpKYxp6Jt16enjPTx6jUkPdvUVHJwVTRMSTNSJxf5S; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:15 GMT" - }, { "name": "Strict-Transport-Security", "value": "max-age=15552000; includeSubDomains; preload" diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/afc42ea89caa932d7ea33666d91e4f1a486bbe3165f5be777e67376075fd89a8.json b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/afc42ea89caa932d7ea33666d91e4f1a486bbe3165f5be777e67376075fd89a8.json index 728068b2e1e..a863479da52 100644 --- a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/afc42ea89caa932d7ea33666d91e4f1a486bbe3165f5be777e67376075fd89a8.json +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/afc42ea89caa932d7ea33666d91e4f1a486bbe3165f5be777e67376075fd89a8.json @@ -40,10 +40,6 @@ "name": "Content-Type", "value": "application/json" }, - { - "name": "set-cookie", - "value": "__cf_bm=kVVa_K4blNT7UfkHNHEr2jJrVZoVVcKoCLeEqC9ojSY-1788292457.9748948-1.0.1.1-4aZN8zUH7fXP1NZK_aaS15fCbNd8A8ZJbBeCartOJWpgZMx2hW3klcPhDDvC0KSV6e.K1Ahz9wRggeTiUZQjsVuKYbR_vJ8dGfKXVNP2KyYcOfgaHyr0Kb1JOZSDGGnS; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:18 GMT" - }, { "name": "mistral-correlation-id", "value": "01a05e89-664a-740d-a112-0a1183c302b3" diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/c74f9b5ad64bd97d8b4c48772c9ee17598a3a1265a630f9a6e42a2df352906bf.json b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/c74f9b5ad64bd97d8b4c48772c9ee17598a3a1265a630f9a6e42a2df352906bf.json index 1653d03634a..258236960e1 100644 --- a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/c74f9b5ad64bd97d8b4c48772c9ee17598a3a1265a630f9a6e42a2df352906bf.json +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/c74f9b5ad64bd97d8b4c48772c9ee17598a3a1265a630f9a6e42a2df352906bf.json @@ -64,10 +64,6 @@ "name": "cf-cache-status", "value": "DYNAMIC" }, - { - "name": "set-cookie", - "value": "__cf_bm=vGgqF1m3.RSN4S6h3XP_Kk8ojWPq3vypiJ2NZgCazl0-1788292461.4966974-1.0.1.1-OYrG7cwlLUmuwb_TTkXh4Te8fKHOoi5hlP5Mok.vCVqZD0lY1bv2Efm83lRx69__NMYwiYqJwrzDNChyYI5Vl9HpHmBybXfpbZ6WpjVEIscEyaLBDMJ_0UI5whsB2.xf; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:22 GMT" - }, { "name": "Strict-Transport-Security", "value": "max-age=15552000; includeSubDomains; preload" diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/cd2461bbef1fc222e80b3397231c79a7f6895b1d6ef5c8f544bae77854bfd951.json b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/cd2461bbef1fc222e80b3397231c79a7f6895b1d6ef5c8f544bae77854bfd951.json index 120e5a43007..8b0adfe8406 100644 --- a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/cd2461bbef1fc222e80b3397231c79a7f6895b1d6ef5c8f544bae77854bfd951.json +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/cd2461bbef1fc222e80b3397231c79a7f6895b1d6ef5c8f544bae77854bfd951.json @@ -22,10 +22,6 @@ "name": "Content-Type", "value": "application/json" }, - { - "name": "set-cookie", - "value": "__cf_bm=CJBaWJ.SsBbSNRd1h3ENyFHO8QA2S5qq0irYhSrOSQY-1788292453.9335105-1.0.1.1-IIqKfxCHaT.6ATkFZNdqQ87W8B2Lnv_N5RSB8sU.dZa5E0gROwfWTZcIsI3aQm_r6aqQT3o.c86aG9ltQq25grYhzFc1LXxWUsmCyMEW7WunMgbzYacSazYXeAxE0yPi; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:14 GMT" - }, { "name": "mistral-correlation-id", "value": "01a05e89-5685-747f-b76d-dab242ea7512" diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/d1aa25a0353a8ff15c49088247a6d73ea777503f904a6bff9960a25046085d0d.json b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/d1aa25a0353a8ff15c49088247a6d73ea777503f904a6bff9960a25046085d0d.json index 50d23042db5..56a42b8b1b3 100644 --- a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/d1aa25a0353a8ff15c49088247a6d73ea777503f904a6bff9960a25046085d0d.json +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/d1aa25a0353a8ff15c49088247a6d73ea777503f904a6bff9960a25046085d0d.json @@ -20,10 +20,6 @@ "name": "Content-Type", "value": "application/json" }, - { - "name": "set-cookie", - "value": "__cf_bm=RwMrJUmbvjo3BcW48Leja7wReidkx5xcJUtnE.9z924-1788292454.453593-1.0.1.1-oxMCFin79BJrtZjXcFewsGcQA0dzFQlC6NHkBN9AuMlUSYoT8EZty3nieW5ORoOnygaYSrpckLffOUJddtxb4TgxSNpkJCBFZOzs3QLLlGpkZrsJx8Rw9ype09j_OKZx; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:14 GMT" - }, { "name": "mistral-correlation-id", "value": "01a05e89-588e-7e32-a54b-f816bc6c1dc4" diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/e362d4eb7aa5f50706b54d0bbb777eebedf342e1844f9114d8db8b9de8a8b08b.json b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/e362d4eb7aa5f50706b54d0bbb777eebedf342e1844f9114d8db8b9de8a8b08b.json index e4417561fbf..26ef9198a4a 100644 --- a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/e362d4eb7aa5f50706b54d0bbb777eebedf342e1844f9114d8db8b9de8a8b08b.json +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/e362d4eb7aa5f50706b54d0bbb777eebedf342e1844f9114d8db8b9de8a8b08b.json @@ -83,10 +83,6 @@ "name": "cf-cache-status", "value": "DYNAMIC" }, - { - "name": "set-cookie", - "value": "__cf_bm=xoadxKi40.IFnjiaqOmRs2WH6OMEVyT3As6eFw.vBvY-1788292456.4529853-1.0.1.1-jbao7uSuL.oLriQMYYO4SXltYHP3IPGIYg7yiRKk3Xqlwsc3xaQu0ES59EpMR3XVjJADyl9rHQfnKY3HvKRxXdGIA8Jerdfcu68wRUZNHcXd_epa4pcO5nnks7YPh93z; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:17 GMT" - }, { "name": "Strict-Transport-Security", "value": "max-age=15552000; includeSubDomains; preload" diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/e9217f2a27a061f2ed26a567677d176db3731e07107f45318fb681740ee49a74.json b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/e9217f2a27a061f2ed26a567677d176db3731e07107f45318fb681740ee49a74.json index 56de121e6ce..ac946e47da2 100644 --- a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/e9217f2a27a061f2ed26a567677d176db3731e07107f45318fb681740ee49a74.json +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/e9217f2a27a061f2ed26a567677d176db3731e07107f45318fb681740ee49a74.json @@ -64,10 +64,6 @@ "name": "cf-cache-status", "value": "DYNAMIC" }, - { - "name": "set-cookie", - "value": "__cf_bm=O0x.ne9EKgb7nXAnOFoTqrbi69sVwHkdvll_fTS6PYE-1788292458.978935-1.0.1.1-2MYJEhW_I9Kf6XeYxBAqVLCXAnN8HABHbK351INmQpOQo036RmkEgf2..oRaM4BDqtR_F7Xrf72ekGCSm4C1h2tQaM6qXV0eCbt78fDRKI3i4N_cWLzghtZs0RJRw7Ir; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:19 GMT" - }, { "name": "Strict-Transport-Security", "value": "max-age=15552000; includeSubDomains; preload" diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/ea3c537a691b95003afc21b351549d5a62e67fa9e6c51ba30de4eaf4bba5a73f.json b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/ea3c537a691b95003afc21b351549d5a62e67fa9e6c51ba30de4eaf4bba5a73f.json index 348c234273d..e52dc13fe18 100644 --- a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/ea3c537a691b95003afc21b351549d5a62e67fa9e6c51ba30de4eaf4bba5a73f.json +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/ea3c537a691b95003afc21b351549d5a62e67fa9e6c51ba30de4eaf4bba5a73f.json @@ -64,10 +64,6 @@ "name": "cf-cache-status", "value": "DYNAMIC" }, - { - "name": "set-cookie", - "value": "__cf_bm=0rjHVJtpp3aXFzTWuyNNxtBvLU2V3RPEgQGcO8MQIJo-1788292463.4929135-1.0.1.1-39O3ne30GQFGDB22kGltMQVL6E9Hk2pTTGEyKXMcYXy8Dzs.BogwfvcphEP473aOQPn_RYs3f5Jz_fhcd4BzKbs1YHm0AQbi84qoKxg.2BQ7kpkarYT1GHgFIfOs_Pxw; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:23 GMT" - }, { "name": "Strict-Transport-Security", "value": "max-age=15552000; includeSubDomains; preload" diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/effe3c112538d2aa6ac98e8f47033086cb79e87d2bdb99bdbb3b7e11751c43d7.json b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/effe3c112538d2aa6ac98e8f47033086cb79e87d2bdb99bdbb3b7e11751c43d7.json index e9e3bdf836d..0282c98cc4e 100644 --- a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/effe3c112538d2aa6ac98e8f47033086cb79e87d2bdb99bdbb3b7e11751c43d7.json +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/effe3c112538d2aa6ac98e8f47033086cb79e87d2bdb99bdbb3b7e11751c43d7.json @@ -83,10 +83,6 @@ "name": "cf-cache-status", "value": "DYNAMIC" }, - { - "name": "set-cookie", - "value": "__cf_bm=dDZPnK7j_xGIDo5mFF6PHBwL.E.iE3aIhXdIZ6xZIPg-1788292465.0206954-1.0.1.1-XVajgKnQ8WEsB75xPcp1hdJXhMUfK.oz.Apkd1nRlGofoxlEWK4F7M2bC8p67VFjDyEZK3O6mCzpiePraLUty31M1H92EBrZBT9wRwuaet2H7P73KO8QHZmO2Jm5tgyC; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:25 GMT" - }, { "name": "Strict-Transport-Security", "value": "max-age=15552000; includeSubDomains; preload" diff --git a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/f430ede1f4069abab794731b9c233573014c6353f8d2486cc34753ceab6c7c1a.json b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/f430ede1f4069abab794731b9c233573014c6353f8d2486cc34753ceab6c7c1a.json index 1b79b9c7f81..5ac0eefa39b 100644 --- a/tests/test_litellm/ocr/fixtures/data/mistral-ocr/f430ede1f4069abab794731b9c233573014c6353f8d2486cc34753ceab6c7c1a.json +++ b/tests/test_litellm/ocr/fixtures/data/mistral-ocr/f430ede1f4069abab794731b9c233573014c6353f8d2486cc34753ceab6c7c1a.json @@ -20,10 +20,6 @@ "name": "Content-Type", "value": "application/json" }, - { - "name": "set-cookie", - "value": "__cf_bm=h45pW85qKj22bgXtkfz55PHEqryQZKiE7tImTTQx2u4-1788292462.5055265-1.0.1.1-kea4VbQ7Y.FRb9t5HgroFMzOCmbkU.byJfToALDpbgme.ScLO1wX13v5qEdll59C0DGKoXk2YYLAXvz0D6QfKN9mwTDRu23O72VJGgXtGu80xcMz16ZSRJfPn7hqnVFI; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:22 GMT" - }, { "name": "mistral-correlation-id", "value": "01a05e89-7802-768c-bf57-9021a209ab48" diff --git a/tests/test_litellm/ocr/fixtures/mistral.py b/tests/test_litellm/ocr/fixtures/mistral.py index 773a3f9cc3e..52adf113b91 100644 --- a/tests/test_litellm/ocr/fixtures/mistral.py +++ b/tests/test_litellm/ocr/fixtures/mistral.py @@ -5,10 +5,10 @@ from typing import Final, Literal, cast from hypothesis import strategies as st from hypothesis.strategies import SearchStrategy -from pydantic import Field, model_validator +from pydantic import model_validator from typing_extensions import Self -from tests.route_parity.fixtures.recording import ProviderSpec +from tests.route_parity.fixtures.recording import UpstreamEndpoint from tests.test_litellm.ocr.fixtures.base import ( JsonSchemaResponseFormat, OcrDocument, @@ -20,11 +20,7 @@ from tests.test_litellm.ocr.fixtures.common import ( annotation_format, document_transport_strategy, invoke_with_api_key, - parameter_strategy, pdf_document, - sampled_list_strategy, - sampled_parameter_group_strategy, - sampled_scalar_strategy, ) MistralModel = Literal[ @@ -43,6 +39,7 @@ MistralModel = Literal[ "mistral-ocr-4", "mistral-ocr-latest", ] +MistralFixtureModel = MistralModel | Literal["mistral/invalid-ocr-model-for-parity"] MISTRAL_MODELS: Final[tuple[MistralModel, ...]] = ( "mistral/mistral-ocr-3", @@ -79,8 +76,8 @@ class MistralCompatibleOcrSdkInput(OcrSdkInputBase): class MistralOcrSdkInput(MistralCompatibleOcrSdkInput): - boundary: str = Field(default="mistral", pattern=r"^mistral$") - model: MistralModel + contract: Literal["mistral"] = "mistral" + model: MistralFixtureModel custom_llm_provider: Literal["mistral"] | None = None @model_validator(mode="after") @@ -90,15 +87,9 @@ class MistralOcrSdkInput(MistralCompatibleOcrSdkInput): return self -class MistralProviderRejectedOcrSdkInput(MistralCompatibleOcrSdkInput): - boundary: str = Field(default="mistral", pattern=r"^mistral$") - model: Literal["mistral/invalid-ocr-model-for-parity"] - custom_llm_provider: Literal["mistral"] | None = None - - MISTRAL_MODEL: Final[MistralModel] = "mistral/mistral-ocr-latest" -MISTRAL_PROVIDER_REJECTED_INPUTS: Final[tuple[MistralProviderRejectedOcrSdkInput, ...]] = ( - MistralProviderRejectedOcrSdkInput( +MISTRAL_PROVIDER_REJECTED_INPUTS: Final[tuple[MistralOcrSdkInput, ...]] = ( + MistralOcrSdkInput( model="mistral/invalid-ocr-model-for-parity", document=pdf_document(), ), @@ -135,40 +126,35 @@ def _optional_param_strategies( ]: annotation: Final = annotation_format("document_title") common: Final[tuple[SearchStrategy[dict[str, object]], ...]] = ( - parameter_strategy("pages", sampled_list_strategy(((0,), (0, 1)))), - parameter_strategy("include_image_base64", sampled_scalar_strategy((False, True))), - parameter_strategy("image_limit", sampled_scalar_strategy((1,))), - parameter_strategy("image_min_size", sampled_scalar_strategy((300,))), - parameter_strategy( - "bbox_annotation_format", - sampled_scalar_strategy((annotation_format("bounding_boxes"),)), - ), - parameter_strategy("document_annotation_format", sampled_scalar_strategy((annotation,))), + st.sampled_from(((0,), (0, 1))).map(list).map(lambda value: {"pages": value}), + st.sampled_from((False, True)).map(lambda value: {"include_image_base64": value}), + st.just({"image_limit": 1}), + st.just({"image_min_size": 300}), + st.just({"bbox_annotation_format": annotation_format("bounding_boxes")}), + st.just({"document_annotation_format": annotation}), *( ( - sampled_parameter_group_strategy( - ( - ( - ("document_annotation_format", annotation), - ("document_annotation_prompt", "Extract the visible title"), - ), - ) + st.just( + { + "document_annotation_format": annotation, + "document_annotation_prompt": "Extract the visible title", + } ), ) if include_document_annotation_prompt else () ), - parameter_strategy("confidence_scores_granularity", sampled_scalar_strategy(("page", "word"))), + st.sampled_from(("page", "word")).map(lambda value: {"confidence_scores_granularity": value}), ) feature_2512: Final[tuple[SearchStrategy[dict[str, object]], ...]] = ( - parameter_strategy("extract_header", sampled_scalar_strategy((False, True))), - parameter_strategy("extract_footer", sampled_scalar_strategy((False, True))), - parameter_strategy("table_format", sampled_scalar_strategy(("markdown", "html"))), + st.sampled_from((False, True)).map(lambda value: {"extract_header": value}), + st.sampled_from((False, True)).map(lambda value: {"extract_footer": value}), + st.sampled_from(("markdown", "html")).map(lambda value: {"table_format": value}), ) feature_4: Final[tuple[SearchStrategy[dict[str, object]], ...]] = ( - parameter_strategy("pages", sampled_scalar_strategy(("0-2",))), - parameter_strategy("include_blocks", sampled_scalar_strategy((False, True))), - sampled_parameter_group_strategy(((("include_blocks", True), ("confidence_scores_granularity", "block")),)), + st.just({"pages": "0-2"}), + st.sampled_from((False, True)).map(lambda value: {"include_blocks": value}), + st.just({"include_blocks": True, "confidence_scores_granularity": "block"}), ) return common, feature_2512, feature_4 @@ -237,7 +223,7 @@ def _mistral_recording_strategy(inline_image_data_uri: str) -> SearchStrategy[Mi feature_2512_options: Final[SearchStrategy[dict[str, object]]] = st.one_of(*feature_2512) feature_4_options: Final[SearchStrategy[dict[str, object]]] = st.one_of(*feature_4) return st.one_of( - sampled_scalar_strategy(baseline_models).map(lambda model: _mistral_input(model, document)), + st.sampled_from(baseline_models).map(lambda model: _mistral_input(model, document)), document_transport_strategy(inline_image_data_uri).map( lambda selected_document: _mistral_input(MISTRAL_MODEL, selected_document) ), @@ -258,11 +244,11 @@ def mistral_recording_targets( if not api_key: return () configured: Final = environ.get("MISTRAL_API_BASE", "https://api.mistral.ai").rstrip("/") - upstream_base: Final = configured.removesuffix("/v1") + base_url: Final = configured.removesuffix("/v1") return ( OcrRecordingTarget( name="mistral-ocr", - provider_spec=ProviderSpec(upstream_base=upstream_base), + upstream=UpstreamEndpoint(base_url=base_url), strategy=cast( SearchStrategy[OcrSdkInputBase], _mistral_recording_strategy(inline_image_data_uri), diff --git a/tests/test_litellm/ocr/fixtures/models.py b/tests/test_litellm/ocr/fixtures/models.py index f26e8b42ac6..0dd8b7399df 100644 --- a/tests/test_litellm/ocr/fixtures/models.py +++ b/tests/test_litellm/ocr/fixtures/models.py @@ -3,48 +3,69 @@ from __future__ import annotations from collections.abc import Mapping from typing import Annotated, Final, cast -from pydantic import Discriminator, Tag +from pydantic import Field, model_validator from tests.route_parity.fixture_models import ParityCase from tests.test_litellm.ocr.fixtures.azure import ( AzureDocumentIntelligenceOcrSdkInput, AzureMistralOcrSdkInput, ) -from tests.test_litellm.ocr.fixtures.base import OcrSdkInputBase -from tests.test_litellm.ocr.fixtures.mistral import MistralOcrSdkInput, MistralProviderRejectedOcrSdkInput +from tests.test_litellm.ocr.fixtures.mistral import MistralOcrSdkInput from tests.test_litellm.ocr.fixtures.reducto import ReductoParseLegacySdkInput, ReductoParseV3SdkInput from tests.test_litellm.ocr.fixtures.vertex import VertexDeepSeekOcrSdkInput, VertexMistralOcrSdkInput __all__ = ("OcrParityCase", "OcrSdkInput") -def _ocr_boundary(value: object) -> str | None: - if isinstance(value, Mapping): - mapping: Final = cast(Mapping[object, object], value) - boundary: Final = mapping.get("boundary") - model: Final = mapping.get("model") - if boundary == "mistral" and model == "mistral/invalid-ocr-model-for-parity": - return "mistral_provider_rejected" - return boundary if isinstance(boundary, str) else None - if isinstance(value, OcrSdkInputBase): - if isinstance(value, MistralProviderRejectedOcrSdkInput): - return "mistral_provider_rejected" - return value.boundary - return None - - OcrSdkInput = Annotated[ - Annotated[MistralOcrSdkInput, Tag("mistral")] - | Annotated[MistralProviderRejectedOcrSdkInput, Tag("mistral_provider_rejected")] - | Annotated[AzureMistralOcrSdkInput, Tag("azure_mistral")] - | Annotated[VertexMistralOcrSdkInput, Tag("vertex_mistral")] - | Annotated[AzureDocumentIntelligenceOcrSdkInput, Tag("azure_document_intelligence")] - | Annotated[VertexDeepSeekOcrSdkInput, Tag("vertex_deepseek")] - | Annotated[ReductoParseV3SdkInput, Tag("reducto_v3")] - | Annotated[ReductoParseLegacySdkInput, Tag("reducto_legacy")], - Discriminator(_ocr_boundary), + MistralOcrSdkInput + | AzureMistralOcrSdkInput + | VertexMistralOcrSdkInput + | AzureDocumentIntelligenceOcrSdkInput + | VertexDeepSeekOcrSdkInput + | ReductoParseV3SdkInput + | ReductoParseLegacySdkInput, + Field(discriminator="contract"), ] class OcrParityCase(ParityCase[OcrSdkInput]): - pass + @model_validator(mode="before") + @classmethod + def load_legacy_contract(cls, value: object) -> object: + if not isinstance(value, Mapping): + return value + fixture: Final = cast(Mapping[str, object], value) + litellm_input: Final = fixture.get("litellm_input") + if not isinstance(litellm_input, Mapping) or "contract" in litellm_input: + return fixture + legacy_input: Final = cast(Mapping[str, object], litellm_input) + legacy_contract: Final = legacy_input.get("boundary") + if isinstance(legacy_contract, str): + return { + **fixture, + "litellm_input": { + "contract": legacy_contract, + **{key: item for key, item in legacy_input.items() if key != "boundary"}, + }, + } + model: Final = legacy_input.get("model") + if not isinstance(model, str): + return fixture + return {**fixture, "litellm_input": {"contract": _legacy_contract(model), **legacy_input}} + + +def _legacy_contract(model: str) -> str: + if model.startswith("azure_ai/doc-intelligence/"): + return "azure_document_intelligence" + if model.startswith("azure_ai/"): + return "azure_mistral" + if model.startswith("vertex_ai/deepseek"): + return "vertex_deepseek" + if model.startswith("vertex_ai/"): + return "vertex_mistral" + if model.endswith("parse-v3"): + return "reducto_v3" + if model.endswith("parse-legacy"): + return "reducto_legacy" + return "mistral" diff --git a/tests/test_litellm/ocr/fixtures/record.py b/tests/test_litellm/ocr/fixtures/record.py index 5151ab26d72..72fcc2c8e20 100644 --- a/tests/test_litellm/ocr/fixtures/record.py +++ b/tests/test_litellm/ocr/fixtures/record.py @@ -3,15 +3,15 @@ from __future__ import annotations import logging import os from collections.abc import Mapping -from pathlib import Path from typing import Final, cast from dotenv import load_dotenv import litellm from litellm.rust_bridge.ocr import use_litellm_rust +from tests.route_parity.fixtures.cli import parse_recording_args from tests.route_parity.fixtures.media import structured_image_data_uri -from tests.route_parity.fixtures.pipeline import parse_recording_args, record_fixtures +from tests.route_parity.fixtures.pipeline import record_fixtures from tests.route_parity.fixtures.store import fixture_directory from tests.test_litellm.ocr.fixtures.azure import ( azure_document_intelligence_recording_targets, @@ -19,13 +19,12 @@ from tests.test_litellm.ocr.fixtures.azure import ( ) from tests.test_litellm.ocr.fixtures.base import OcrSdkInputBase from tests.test_litellm.ocr.fixtures.common import OcrFixtureClient, OcrRecordingTarget, OcrSdkCall +from tests.test_litellm.ocr.fixtures.config import DEFAULT_FIXTURE_DIRECTORY, FIXTURE_DIR_ENV from tests.test_litellm.ocr.fixtures.mistral import mistral_recording_targets from tests.test_litellm.ocr.fixtures.models import OcrParityCase from tests.test_litellm.ocr.fixtures.reducto import reducto_recording_targets from tests.test_litellm.ocr.fixtures.vertex import vertex_recording_targets -FIXTURE_DIR_ENV: Final = "LITELLM_OCR_FIXTURE_DIR" - class LiteLLMOcrFixtureClient: def __init__(self, sdk_call: OcrSdkCall) -> None: @@ -65,7 +64,7 @@ def main() -> int: root: Final = fixture_directory( args.fixture_dir, os.environ.get(FIXTURE_DIR_ENV), - Path(__file__).with_name("data"), + DEFAULT_FIXTURE_DIRECTORY, ) use_litellm_rust(False, ocr=None, aocr=None) summary: Final = record_fixtures(targets, root, args.examples, args.concurrency, OcrParityCase) diff --git a/tests/test_litellm/ocr/fixtures/reducto.py b/tests/test_litellm/ocr/fixtures/reducto.py index 01babd7f031..59af22694bf 100644 --- a/tests/test_litellm/ocr/fixtures/reducto.py +++ b/tests/test_litellm/ocr/fixtures/reducto.py @@ -12,16 +12,13 @@ from typing_extensions import Self from tests.route_parity.fixture_models import FixtureModel, JsonObject from tests.route_parity.fixtures.media import structured_pdf_data_uri -from tests.route_parity.fixtures.recording import ProviderSpec +from tests.route_parity.fixtures.recording import UpstreamEndpoint from tests.test_litellm.ocr.fixtures.base import OcrSdkInputBase from tests.test_litellm.ocr.fixtures.common import ( OcrFixtureClient, OcrRecordingTarget, image_data_document, invoke_with_api_key, - parameter_strategy, - sampled_list_strategy, - sampled_scalar_strategy, ) @@ -204,7 +201,7 @@ class ReductoSettings(FixtureModel): class ReductoParseV3SdkInput(OcrSdkInputBase): - boundary: str = Field(default="reducto_v3", pattern=r"^reducto_v3$") + contract: Literal["reducto_v3"] = "reducto_v3" model: ReductoV3Model document: ReductoDocument custom_llm_provider: Literal["reducto"] | None = None @@ -220,7 +217,7 @@ class ReductoParseV3SdkInput(OcrSdkInputBase): class ReductoParseLegacySdkInput(OcrSdkInputBase): - boundary: str = Field(default="reducto_legacy", pattern=r"^reducto_legacy$") + contract: Literal["reducto_legacy"] = "reducto_legacy" model: ReductoLegacyModel document: ReductoDocument custom_llm_provider: Literal["reducto"] | None = None @@ -233,28 +230,44 @@ class ReductoParseLegacySdkInput(OcrSdkInputBase): return self +_REDUCTO_PROVIDER_REJECTED_DOCUMENT: Final = ReductoDocumentUrlDocument( + type="document_url", + document_url="reducto://invalid-document-for-parity", +) +REDUCTO_V3_PROVIDER_REJECTED_INPUTS: Final[tuple[ReductoParseV3SdkInput, ...]] = ( + ReductoParseV3SdkInput( + model="reducto/parse-v3", + document=_REDUCTO_PROVIDER_REJECTED_DOCUMENT, + ), +) +REDUCTO_LEGACY_PROVIDER_REJECTED_INPUTS: Final[tuple[ReductoParseLegacySdkInput, ...]] = ( + ReductoParseLegacySdkInput( + model="reducto/parse-legacy", + document=_REDUCTO_PROVIDER_REJECTED_DOCUMENT, + ), +) + + _REDUCTO_API_BASE: Final = "https://platform.reducto.ai" def _formatting_strategy() -> SearchStrategy[ReductoFormatting]: values: Final = st.one_of( - parameter_strategy( - "table_output_format", - sampled_scalar_strategy(("dynamic", "html", "md", "json", "csv", "jsonbbox")), - ), - parameter_strategy("add_page_markers", sampled_scalar_strategy((False, True))), - parameter_strategy("merge_tables", sampled_scalar_strategy((False, True))), - parameter_strategy( - "include", - sampled_list_strategy( - ( - (), - ("hyperlinks",), - ("change_tracking", "highlight", "comments"), - ("signatures", "ignore_watermarks"), - ) - ), + st.sampled_from(("dynamic", "html", "md", "json", "csv", "jsonbbox")).map( + lambda value: {"table_output_format": value} ), + st.sampled_from((False, True)).map(lambda value: {"add_page_markers": value}), + st.sampled_from((False, True)).map(lambda value: {"merge_tables": value}), + st.sampled_from( + ( + (), + ("hyperlinks",), + ("change_tracking", "highlight", "comments"), + ("signatures", "ignore_watermarks"), + ) + ) + .map(list) + .map(lambda value: {"include": value}), ) return values.map(ReductoFormatting.model_validate) @@ -265,21 +278,22 @@ def _chunking_strategy() -> SearchStrategy[ReductoChunking]: lambda mode: ReductoChunking(chunk_mode=mode) ), st.just(ReductoChunking(chunk_mode="variable")), - sampled_scalar_strategy((250, 1000, 1500)).map( - lambda size: ReductoChunking(chunk_mode="variable", chunk_size=size) - ), - sampled_scalar_strategy((32, 128)).map( + st.sampled_from((250, 1000, 1500)).map(lambda size: ReductoChunking(chunk_mode="variable", chunk_size=size)), + st.sampled_from((32, 128)).map( lambda overlap: ReductoChunking(chunk_mode="variable", chunk_size=1000, chunk_overlap=overlap) ), ) def _retrieval_strategy() -> SearchStrategy[ReductoRetrieval]: - filter_blocks: Final[SearchStrategy[list[ReductoBlockType]]] = sampled_list_strategy(_REDUCTO_FILTER_BLOCK_GROUPS) + filter_blocks: Final = cast( + SearchStrategy[list[ReductoBlockType]], + st.sampled_from(_REDUCTO_FILTER_BLOCK_GROUPS).map(list), + ) return st.one_of( _chunking_strategy().map(lambda chunking: ReductoRetrieval(chunking=chunking)), filter_blocks.map(lambda selected_blocks: ReductoRetrieval(filter_blocks=selected_blocks)), - sampled_scalar_strategy((False, True)).map( + st.sampled_from((False, True)).map( lambda optimized: ReductoRetrieval( chunking=ReductoChunking(chunk_mode="variable"), embedding_optimized=optimized, @@ -291,18 +305,20 @@ def _retrieval_strategy() -> SearchStrategy[ReductoRetrieval]: def _settings_strategy() -> SearchStrategy[ReductoSettings]: # force_url_result stays model-compatible but is not recorded until the # response transform follows and downloads result.url. - return_images: Final[SearchStrategy[list[ReductoReturnImage]]] = sampled_list_strategy(_REDUCTO_RETURN_IMAGE_GROUPS) + return_images: Final[SearchStrategy[list[ReductoReturnImage]]] = st.sampled_from(_REDUCTO_RETURN_IMAGE_GROUPS).map( + list + ) page_ranges: Final = st.one_of( st.just(ReductoPageRange(start=1, end=1)), st.just(ReductoPageRange(start=1, end=3)), - sampled_list_strategy( + st.sampled_from( ( ( ReductoPageRange(start=1, end=2), ReductoPageRange(start=4, end=5), ), ) - ), + ).map(list), ) return st.one_of( st.just(ReductoSettings(model="r-1")), @@ -311,10 +327,10 @@ def _settings_strategy() -> SearchStrategy[ReductoSettings]: st.just(ReductoSettings(return_ocr_data=True)), return_images.map(lambda selected_images: ReductoSettings(return_images=selected_images)), st.just(ReductoSettings(embed_pdf_metadata=True)), - sampled_scalar_strategy((50, 100, 250)).map( + st.sampled_from((50, 100, 250)).map( lambda dpi: ReductoSettings(embed_pdf_metadata=True, embed_pdf_metadata_dpi=dpi) ), - sampled_scalar_strategy((300.0,)).map(lambda timeout: ReductoSettings(timeout=timeout)), + st.just(ReductoSettings(timeout=300.0)), page_ranges.map(lambda page_range: ReductoSettings(page_range=page_range)), ) @@ -403,20 +419,22 @@ def reducto_recording_targets( api_key: Final = environ.get("REDUCTO_API_KEY") if not api_key: return () - upstream_base: Final = environ.get("REDUCTO_API_BASE", _REDUCTO_API_BASE).rstrip("/") + base_url: Final = environ.get("REDUCTO_API_BASE", _REDUCTO_API_BASE).rstrip("/") document: Final = ReductoDocumentUrlDocument(type="document_url", document_url=structured_pdf_data_uri()) invocation: Final = invoke_with_api_key(client, api_key) return ( OcrRecordingTarget( name="reducto-v3", - provider_spec=ProviderSpec(upstream_base=upstream_base), + upstream=UpstreamEndpoint(base_url=base_url), strategy=cast(SearchStrategy[OcrSdkInputBase], reducto_v3_input_strategy(inline_image_data_uri, document)), invocation=invocation, + required_inputs=REDUCTO_V3_PROVIDER_REJECTED_INPUTS, ), OcrRecordingTarget( name="reducto-legacy", - provider_spec=ProviderSpec(upstream_base=upstream_base), + upstream=UpstreamEndpoint(base_url=base_url), strategy=cast(SearchStrategy[OcrSdkInputBase], reducto_legacy_input_strategy(document)), invocation=invocation, + required_inputs=REDUCTO_LEGACY_PROVIDER_REJECTED_INPUTS, ), ) diff --git a/tests/test_litellm/ocr/fixtures/vertex.py b/tests/test_litellm/ocr/fixtures/vertex.py index c7014bb75ec..f4e51fb1cd2 100644 --- a/tests/test_litellm/ocr/fixtures/vertex.py +++ b/tests/test_litellm/ocr/fixtures/vertex.py @@ -5,16 +5,14 @@ from typing import Final, Literal, cast from hypothesis import strategies as st from hypothesis.strategies import DrawFn, SearchStrategy -from pydantic import Field -from tests.route_parity.fixtures.recording import ProviderSpec +from tests.route_parity.fixtures.recording import UpstreamEndpoint from tests.test_litellm.ocr.fixtures.base import OcrDocument, OcrSdkInputBase from tests.test_litellm.ocr.fixtures.common import ( OcrFixtureClient, OcrRecordingTarget, image_data_document, invoke_with_api_key, - sampled_scalar_strategy, ) from tests.test_litellm.ocr.fixtures.mistral import ( MistralCompatibleOcrSdkInput, @@ -23,28 +21,60 @@ from tests.test_litellm.ocr.fixtures.mistral import ( VertexMistralModel = Literal["vertex_ai/mistral-ocr-2505"] VertexDeepSeekModel = Literal["vertex_ai/deepseek-ai/deepseek-ocr-maas"] +VertexMistralFixtureModel = VertexMistralModel | Literal["vertex_ai/invalid-ocr-model-for-parity"] +VertexDeepSeekFixtureModel = VertexDeepSeekModel | Literal["vertex_ai/deepseek-ai/invalid-ocr-model-for-parity"] VERTEX_MISTRAL_MODELS: Final[tuple[VertexMistralModel, ...]] = ("vertex_ai/mistral-ocr-2505",) VERTEX_DEEPSEEK_MODELS: Final[tuple[VertexDeepSeekModel, ...]] = ("vertex_ai/deepseek-ai/deepseek-ocr-maas",) class VertexMistralOcrSdkInput(MistralCompatibleOcrSdkInput): - boundary: str = Field(default="vertex_mistral", pattern=r"^vertex_mistral$") - model: VertexMistralModel = "vertex_ai/mistral-ocr-2505" + contract: Literal["vertex_mistral"] = "vertex_mistral" + model: VertexMistralFixtureModel = "vertex_ai/mistral-ocr-2505" custom_llm_provider: Literal["vertex_ai"] | None = None vertex_project: str vertex_location: str = "us-central1" class VertexDeepSeekOcrSdkInput(OcrSdkInputBase): - boundary: str = Field(default="vertex_deepseek", pattern=r"^vertex_deepseek$") - model: VertexDeepSeekModel = "vertex_ai/deepseek-ai/deepseek-ocr-maas" + contract: Literal["vertex_deepseek"] = "vertex_deepseek" + model: VertexDeepSeekFixtureModel = "vertex_ai/deepseek-ai/deepseek-ocr-maas" document: OcrDocument custom_llm_provider: Literal["vertex_ai"] | None = None vertex_project: str vertex_location: str = "us-central1" +def vertex_mistral_provider_rejected_inputs( + project: str, + location: str, + inline_image_data_uri: str, +) -> tuple[VertexMistralOcrSdkInput, ...]: + return ( + VertexMistralOcrSdkInput( + model="vertex_ai/invalid-ocr-model-for-parity", + document=image_data_document(inline_image_data_uri), + vertex_project=project, + vertex_location=location, + ), + ) + + +def vertex_deepseek_provider_rejected_inputs( + project: str, + location: str, + inline_image_data_uri: str, +) -> tuple[VertexDeepSeekOcrSdkInput, ...]: + return ( + VertexDeepSeekOcrSdkInput( + model="vertex_ai/deepseek-ai/invalid-ocr-model-for-parity", + document=image_data_document(inline_image_data_uri), + vertex_project=project, + vertex_location=location, + ), + ) + + def _as_vertex_mistral( values: dict[str, object], project: str, @@ -65,7 +95,7 @@ def vertex_mistral_input_strategy( _as_vertex_mistral, project=st.just(project), location=st.just(location), - model=sampled_scalar_strategy(VERTEX_MISTRAL_MODELS), + model=st.sampled_from(VERTEX_MISTRAL_MODELS), values=mistral_input_values_strategy("2505", inline_image_data_uri), ) @@ -76,10 +106,7 @@ def vertex_deepseek_input_strategy( ) -> VertexDeepSeekOcrSdkInput: return VertexDeepSeekOcrSdkInput.model_validate( { - "model": draw(sampled_scalar_strategy(VERTEX_DEEPSEEK_MODELS)), - # The current Vertex model card documents image input only. Keep - # the broader fixture model for existing recordings, but do not - # spend a paid request on the transform's unsupported PDF branch. + "model": draw(st.sampled_from(VERTEX_DEEPSEEK_MODELS)), "document": image_data_document(inline_image_data_uri), "vertex_project": project, "vertex_location": location, @@ -95,25 +122,27 @@ def vertex_recording_targets( location: Final = environ.get("VERTEXAI_LOCATION") or environ.get("VERTEX_LOCATION") or "us-central1" if not api_key or not project: return () - upstream_base: Final = environ.get("VERTEX_AI_API_BASE") or f"https://{location}-aiplatform.googleapis.com" + base_url: Final = environ.get("VERTEX_AI_API_BASE") or f"https://{location}-aiplatform.googleapis.com" invocation: Final = invoke_with_api_key(client, api_key) return ( OcrRecordingTarget( name="vertex-mistral", - provider_spec=ProviderSpec(upstream_base=upstream_base.rstrip("/")), + upstream=UpstreamEndpoint(base_url=base_url.rstrip("/")), strategy=cast( SearchStrategy[OcrSdkInputBase], vertex_mistral_input_strategy(project, location, inline_image_data_uri), ), invocation=invocation, + required_inputs=vertex_mistral_provider_rejected_inputs(project, location, inline_image_data_uri), ), OcrRecordingTarget( name="vertex-deepseek", - provider_spec=ProviderSpec(upstream_base=upstream_base.rstrip("/")), + upstream=UpstreamEndpoint(base_url=base_url.rstrip("/")), strategy=cast( SearchStrategy[OcrSdkInputBase], vertex_deepseek_input_strategy(project, location, inline_image_data_uri), ), invocation=invocation, + required_inputs=vertex_deepseek_provider_rejected_inputs(project, location, inline_image_data_uri), ), ) diff --git a/tests/test_litellm/ocr/test_fixture_models.py b/tests/test_litellm/ocr/test_fixture_models.py index 24fb946e9bb..f97cf27db66 100644 --- a/tests/test_litellm/ocr/test_fixture_models.py +++ b/tests/test_litellm/ocr/test_fixture_models.py @@ -47,7 +47,7 @@ from tests.test_litellm.ocr.fixtures.base import ( OcrSdkInputBase, ) from tests.test_litellm.ocr.fixtures.mistral import MISTRAL_MODELS, MistralOcrSdkInput, mistral_input_strategy -from tests.test_litellm.ocr.fixtures.models import OcrParityCase +from tests.test_litellm.ocr.fixtures.models import OcrParityCase, OcrSdkInput from tests.test_litellm.ocr.fixtures.reducto import ( REDUCTO_LEGACY_MODELS, REDUCTO_V3_MODELS, @@ -73,7 +73,7 @@ from tests.test_litellm.ocr.fixtures.vertex import ( ) COMMON_FIELDS: Final = frozenset( - {"boundary", "model", "document", "custom_llm_provider", "vertex_project", "vertex_location"} + {"contract", "model", "document", "custom_llm_provider", "vertex_project", "vertex_location"} ) SUPPORTED_OCR_PROVIDERS: Final = frozenset({"mistral", "azure_ai", "reducto", "vertex_ai"}) ACTIVE_OCR_MODELS: Final = frozenset( @@ -342,9 +342,23 @@ def test_fixture_fields_match_provider_config( ), ), ) -def test_provider_boundary_is_explicit_but_not_forwarded(sdk_input: OcrSdkInputBase) -> None: - assert sdk_input.canonical_input()["boundary"] == sdk_input.boundary - assert "boundary" not in sdk_input.as_sdk_kwargs() +def test_provider_contract_is_explicit_but_not_forwarded(sdk_input: OcrSdkInput) -> None: + assert sdk_input.canonical_input()["contract"] == sdk_input.contract + assert "contract" not in sdk_input.as_sdk_kwargs() + + +@pytest.mark.parametrize("legacy_key", ("boundary", None)) +def test_ocr_parity_case_migrates_legacy_contract_metadata(legacy_key: str | None) -> None: + litellm_input: Final[dict[str, object]] = { + "model": "mistral/mistral-ocr-latest", + "document": {"type": "document_url", "document_url": "https://example.com/document.pdf"}, + } + if legacy_key is not None: + litellm_input[legacy_key] = "mistral" + + fixture: Final = OcrParityCase.model_validate({"litellm_input": litellm_input, "provider_responses": ()}) + + assert fixture.litellm_input.contract == "mistral" def test_mistral_input_preserves_omission_and_explicit_boolean_values() -> None: @@ -478,6 +492,7 @@ def test_vertex_deepseek_request_maps_both_document_types_to_image_content( data: Final = cast(dict[str, object], request.data) messages: Final = cast(list[dict[str, object]], data["messages"]) content: Final = cast(list[dict[str, object]], messages[0]["content"]) + assert data["model"] == "deepseek-ai/deepseek-ocr-maas" assert content == [{"type": "image_url", "image_url": document[source_key]}] @@ -1139,7 +1154,7 @@ def test_azure_document_intelligence_strategy_only_generates_litellm_inputs( ) -> None: assert sdk_input.req_format == "litellm" assert sdk_input.model in AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS - assert "boundary" not in sdk_input.as_sdk_kwargs() + assert "contract" not in sdk_input.as_sdk_kwargs() optional_fields: Final = frozenset(sdk_input.model_fields_set) - {"model", "document"} assert optional_fields in { frozenset[str](), @@ -1361,7 +1376,7 @@ def test_vertex_deepseek_strategy_only_generates_litellm_inputs( sdk_input: VertexDeepSeekOcrSdkInput, ) -> None: assert sdk_input.vertex_project == "project-1" - assert "boundary" not in sdk_input.as_sdk_kwargs() + assert "contract" not in sdk_input.as_sdk_kwargs() assert _document_transport(sdk_input.document) == ("image_url", "data") diff --git a/tests/test_litellm/ocr/test_fixture_store.py b/tests/test_litellm/ocr/test_fixture_store.py index e12c6b3aa4b..f13606e208b 100644 --- a/tests/test_litellm/ocr/test_fixture_store.py +++ b/tests/test_litellm/ocr/test_fixture_store.py @@ -7,7 +7,7 @@ from typing import Final, Protocol, cast import pytest -from tests.route_parity.fixtures.store import parametrize_recorded_fixtures +from tests.route_parity.fixtures.pytest_support import parametrize_recorded_fixtures from tests.test_litellm.ocr.conftest import ocr_fixture_id, ocr_fixture_marks from tests.test_litellm.ocr.fixtures.models import OcrParityCase @@ -45,7 +45,7 @@ def test_recorded_fixture_parametrization_applies_case_specific_marks() -> None: reducto_parameters: Final = tuple( parameter for parameter in parameters - if parameter.values[0].litellm_input.boundary in {"reducto_v3", "reducto_legacy"} + if parameter.values[0].litellm_input.contract in {"reducto_v3", "reducto_legacy"} ) supported_parameters: Final = tuple(parameter for parameter in parameters if parameter not in reducto_parameters) diff --git a/tests/test_litellm/ocr/test_record_fixtures.py b/tests/test_litellm/ocr/test_record_fixtures.py index dfb1d15ac63..4ed4afcca38 100644 --- a/tests/test_litellm/ocr/test_record_fixtures.py +++ b/tests/test_litellm/ocr/test_record_fixtures.py @@ -10,12 +10,14 @@ import pytest from hypothesis import find, settings from hypothesis.strategies import SearchStrategy +from tests.route_parity.fixtures.cli import parse_recording_args from tests.route_parity.fixtures.inputs import generate_case_inputs from tests.route_parity.fixtures.media import structured_pdf_data_uri -from tests.route_parity.fixtures.pipeline import parse_recording_args from tests.test_litellm.ocr.fixtures.azure import ( + AZURE_DOCUMENT_INTELLIGENCE_PROVIDER_REJECTED_INPUTS, AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS, AZURE_MISTRAL_MODELS, + AZURE_MISTRAL_PROVIDER_REJECTED_INPUTS, ) from tests.test_litellm.ocr.fixtures.base import OcrSdkInputBase from tests.test_litellm.ocr.fixtures.common import OcrFixtureClient, OcrRecordingTarget @@ -26,8 +28,18 @@ from tests.test_litellm.ocr.fixtures.record import ( from tests.test_litellm.ocr.fixtures.record import ( require_targets, ) -from tests.test_litellm.ocr.fixtures.reducto import REDUCTO_LEGACY_MODELS, REDUCTO_V3_MODELS -from tests.test_litellm.ocr.fixtures.vertex import VERTEX_DEEPSEEK_MODELS, VERTEX_MISTRAL_MODELS +from tests.test_litellm.ocr.fixtures.reducto import ( + REDUCTO_LEGACY_MODELS, + REDUCTO_LEGACY_PROVIDER_REJECTED_INPUTS, + REDUCTO_V3_MODELS, + REDUCTO_V3_PROVIDER_REJECTED_INPUTS, +) +from tests.test_litellm.ocr.fixtures.vertex import ( + VERTEX_DEEPSEEK_MODELS, + VERTEX_MISTRAL_MODELS, + vertex_deepseek_provider_rejected_inputs, + vertex_mistral_provider_rejected_inputs, +) class _UnusedOcrClient: @@ -186,7 +198,7 @@ def test_mistral_target_uses_canonical_model_and_normalized_base( assert len(targets) == 1 target: Final = targets[0] assert target.name == "mistral-ocr" - assert target.provider_spec.upstream_base == expected + assert target.upstream.base_url == expected assert "mistral-secret" not in repr(target) case_inputs: Final = generate_case_inputs(target.strategy, examples=1) assert len(case_inputs) == 1 @@ -316,11 +328,21 @@ def test_only_intentional_provider_failures_are_fixed_inputs() -> None: _UNUSED_OCR_CLIENT, ) - mistral: Final = next(target for target in targets if target.name == "mistral-ocr") - assert mistral.required_inputs == MISTRAL_PROVIDER_REJECTED_INPUTS - generated: Final = generate_case_inputs(mistral.strategy, examples=20) - assert all(case_input not in mistral.required_inputs for case_input in generated) - assert all(target.required_inputs == () for target in targets if target is not mistral) + expected: Final[dict[str, tuple[OcrSdkInputBase, ...]]] = { + "mistral-ocr": MISTRAL_PROVIDER_REJECTED_INPUTS, + "azure-mistral": AZURE_MISTRAL_PROVIDER_REJECTED_INPUTS, + "azure-document-intelligence": AZURE_DOCUMENT_INTELLIGENCE_PROVIDER_REJECTED_INPUTS, + "vertex-mistral": vertex_mistral_provider_rejected_inputs("project-1", "us-central1", _INLINE_IMAGE_DATA_URI), + "vertex-deepseek": vertex_deepseek_provider_rejected_inputs("project-1", "us-central1", _INLINE_IMAGE_DATA_URI), + "reducto-v3": REDUCTO_V3_PROVIDER_REJECTED_INPUTS, + "reducto-legacy": REDUCTO_LEGACY_PROVIDER_REJECTED_INPUTS, + } + + assert {target.name for target in targets} == expected.keys() + for target in targets: + assert target.required_inputs == expected[target.name] + generated: Final = generate_case_inputs(target.strategy, examples=20) + assert all(case_input not in target.required_inputs for case_input in generated) def test_mistral_adapters_preserve_omitted_optional_params() -> None: diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py index 1887bff1f97..b489cb0a280 100644 --- a/tests/test_litellm/ocr/test_rust_bridge.py +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -368,6 +368,18 @@ def test_load_rust_ocr_uses_compiled_extension(monkeypatch): def test_timeout_to_seconds_handles_float_timeout_and_none(): assert rust_bridge._timeout_to_seconds(12.5) == 12.5 assert rust_bridge._timeout_to_seconds(None) is None + + +def test_ocr_provider_error_uses_resolved_request_url(): + error = rust_bridge._OcrProviderError( + 429, + "rate limited", + "https://example.azure.com/documentintelligence/documentModels/read:analyze", + ) + + assert str(error.response.request.url) == ( + "https://example.azure.com/documentintelligence/documentModels/read:analyze" + ) assert rust_bridge._timeout_to_seconds(httpx.Timeout(30.0, read=42.0)) == 42.0 diff --git a/tests/test_litellm/ocr/test_sdk_parity.py b/tests/test_litellm/ocr/test_sdk_parity.py index a60e536a3ac..3f7fb84958b 100644 --- a/tests/test_litellm/ocr/test_sdk_parity.py +++ b/tests/test_litellm/ocr/test_sdk_parity.py @@ -1,7 +1,6 @@ from __future__ import annotations import asyncio -import os import sys import traceback from collections.abc import Awaitable, Callable, Coroutine, Generator @@ -32,17 +31,20 @@ from tests.route_parity.models import ( ) from tests.route_parity.replay import replay_server from tests.route_parity.runner import ( - PythonScriptRunner, - PythonScriptWorker, + ExecutionVariant, + SubprocessRunner, + SubprocessWorker, execution_worker_pair, parity_worker_main, run_execution, ) +from tests.test_litellm.ocr.fixtures.config import configured_fixture_directory from tests.test_litellm.ocr.fixtures.models import OcrParityCase, OcrSdkInput API_KEY: Final = "test-key" PYTHON_HTTP_SENTINEL: Final = "python-ocr-parity-fallback" -FIXTURE_DIR_ENV: Final = "LITELLM_OCR_FIXTURE_DIR" +PYTHON_VARIANT: Final = ExecutionVariant(name="Python", environment=(("LITELLM_USE_RUST_OCR", "0"),)) +RUST_VARIANT: Final = ExecutionVariant(name="Rust", environment=(("LITELLM_USE_RUST_OCR", "1"),)) class SDKRoute(str, Enum): @@ -120,6 +122,15 @@ INVALID_OCR_CASES: Final = ( expected_message="Document URL is required", expected_rust_calls=1, ), + InvalidOcrCase( + name="missing_image_url", + model="azure_ai/doc-intelligence/prebuilt-read", + document={"type": "image_url"}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="Document URL is required", + expected_rust_calls=1, + ), InvalidOcrCase( name="invalid_request_format", model="mistral/mistral-ocr-latest", @@ -304,22 +315,19 @@ def _native_spies() -> tuple[_RustOcrSpy, _RustAocrSpy]: @pytest.fixture(scope="module") -def sdk_workers() -> Generator[tuple[PythonScriptWorker, PythonScriptWorker]]: - runner: Final = PythonScriptRunner( +def sdk_workers() -> Generator[tuple[SubprocessWorker, SubprocessWorker]]: + runner: Final = SubprocessRunner( entrypoint=Path(__file__), - rust_env_var="LITELLM_USE_RUST_OCR", - python_user_agent=PYTHON_HTTP_SENTINEL, + baseline_user_agent=PYTHON_HTTP_SENTINEL, route_label="OCR", ) - with execution_worker_pair(runner) as workers: + with execution_worker_pair(runner, PYTHON_VARIANT, RUST_VARIANT) as workers: yield workers @pytest.fixture(scope="module") def startup_ocr_fixture() -> OcrParityCase: - default_directory: Final = Path(__file__).with_name("fixtures") / "data" - configured: Final = os.environ.get(FIXTURE_DIR_ENV) - directory: Final = Path(configured).expanduser() if configured is not None else default_directory + directory: Final = configured_fixture_directory() fixtures: Final = recorded_fixtures(directory, OcrParityCase) if not fixtures: pytest.skip(f"no recorded fixtures in {directory}") @@ -404,7 +412,7 @@ def test_invalid_ocr_sdk_parity(case: InvalidOcrCase, route: SDKRoute) -> None: def test_ocr_subprocess_startup_smoke( startup_ocr_fixture: OcrParityCase, tmp_path: Path, - sdk_workers: tuple[PythonScriptWorker, PythonScriptWorker], + sdk_workers: tuple[SubprocessWorker, SubprocessWorker], ) -> None: case_file: Final = tmp_path / "ocr-startup-smoke.json" case_file.write_text(startup_ocr_fixture.model_dump_json(indent=2, exclude_unset=True), encoding="utf-8")