mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-11 22:51:28 +00:00
refactor(tests): generalize route parity fixtures
This commit is contained in:
Yujong Lee
parent
fca4adacaa
commit
ade1f0df64
58 changed files with 847 additions and 644 deletions
|
|
@ -60,9 +60,6 @@ fn chat_completions_response_to_py(
|
|||
|
||||
fn core_error_to_pyerr(err: CoreError) -> PyErr {
|
||||
match err {
|
||||
CoreError::MissingField("document_url") => {
|
||||
PyValueError::new_err("Document URL is required")
|
||||
}
|
||||
CoreError::Auth(message) => PyValueError::new_err(message),
|
||||
CoreError::InvalidProvider(_)
|
||||
| CoreError::InvalidRequest(_)
|
||||
|
|
@ -74,6 +71,9 @@ fn core_error_to_pyerr(err: CoreError) -> PyErr {
|
|||
|
||||
fn ocr_error_to_pyerr(err: CoreError) -> PyErr {
|
||||
match err {
|
||||
CoreError::MissingField("document_url" | "image_url") => {
|
||||
PyValueError::new_err("Document URL is required")
|
||||
}
|
||||
CoreError::Http { status, body } => RustUpstreamError::new_err((status, body)),
|
||||
other => core_error_to_pyerr(other),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -177,8 +177,9 @@ class VertexAIDeepSeekOCRConfig(BaseOCRConfig):
|
|||
content_item = {"type": "image_url", "image_url": document_url}
|
||||
|
||||
# Build DeepSeek OCR request
|
||||
provider_model: Final = model if model.startswith("deepseek-ai/") else f"deepseek-ai/{model}"
|
||||
data: Final = {
|
||||
"model": "deepseek-ai/" + model,
|
||||
"model": provider_model,
|
||||
"messages": [{"role": "user", "content": [content_item]}],
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -52,12 +52,13 @@ class _PreparedOCRRequest:
|
|||
litellm_logging_obj: LiteLLMLoggingObj
|
||||
|
||||
|
||||
@dataclass
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _PreparedRustOCRCall:
|
||||
api_key: str | None
|
||||
api_base: str | None
|
||||
headers: dict[str, object]
|
||||
optional_params: dict[str, object]
|
||||
request_url: str
|
||||
|
||||
|
||||
_RUST_OCR_PROVIDERS: Final = {
|
||||
|
|
@ -191,6 +192,10 @@ def _prepare_ocr_request(
|
|||
def _rust_ocr_supported(prepared_request: _PreparedOCRRequest) -> bool:
|
||||
if prepared_request.optional_params.get(OCR_REQUEST_FORMAT_PARAM) == "native":
|
||||
return False
|
||||
if prepared_request.extra_headers is not None and any(
|
||||
not isinstance(value, str) for value in prepared_request.extra_headers.values()
|
||||
):
|
||||
return False
|
||||
return prepared_request.custom_llm_provider in _RUST_OCR_PROVIDERS
|
||||
|
||||
|
||||
|
|
@ -275,6 +280,7 @@ def _prepare_rust_ocr_call(
|
|||
api_base=rust_api_base,
|
||||
headers=cast(dict[str, object], resolved_headers),
|
||||
optional_params=rust_optional_params,
|
||||
request_url=resolved_complete_url,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -297,6 +303,7 @@ def _run_rust_ocr(
|
|||
extra_headers=prepared.headers,
|
||||
optional_params=prepared.optional_params,
|
||||
timeout=prepared_request.effective_timeout,
|
||||
request_url=prepared.request_url,
|
||||
)
|
||||
if rust_response is None:
|
||||
return None
|
||||
|
|
@ -322,6 +329,7 @@ async def _run_rust_aocr(
|
|||
extra_headers=prepared.headers,
|
||||
optional_params=prepared.optional_params,
|
||||
timeout=prepared_request.effective_timeout,
|
||||
request_url=prepared.request_url,
|
||||
)
|
||||
if rust_response is None:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -281,18 +281,6 @@ def rust_chat_completions_accepts(
|
|||
return True
|
||||
|
||||
|
||||
def _rust_bridge_exceptions() -> tuple[type[BaseException], type[BaseException]] | None:
|
||||
"""`(declined, upstream_failed)` from the native module, or None when absent."""
|
||||
native_bridge: Final = get_native_bridge()
|
||||
if native_bridge is None:
|
||||
return None
|
||||
declined: Final = getattr(native_bridge, "RustBridgeDeclined", None)
|
||||
upstream: Final = getattr(native_bridge, "RustUpstreamError", None)
|
||||
if declined is None or upstream is None:
|
||||
return None
|
||||
return declined, upstream
|
||||
|
||||
|
||||
def _reraise_or_decline(
|
||||
rust_error: BaseException,
|
||||
*,
|
||||
|
|
@ -306,25 +294,25 @@ def _reraise_or_decline(
|
|||
second attempt bills for it twice. Those surface as an `APIError` carrying
|
||||
the upstream status, which LiteLLM's exception mapping already understands.
|
||||
"""
|
||||
exceptions: Final = _rust_bridge_exceptions()
|
||||
from litellm.rust_bridge.errors import native_bridge_exceptions, rust_upstream_failure
|
||||
|
||||
native_bridge: Final = get_native_bridge()
|
||||
exceptions: Final = native_bridge_exceptions(native_bridge)
|
||||
if exceptions is None:
|
||||
verbose_logger.debug(
|
||||
"Rust chat completions bridge raised %s; falling back to Python path",
|
||||
type(rust_error).__name__,
|
||||
)
|
||||
return
|
||||
declined, upstream_failed = exceptions
|
||||
if isinstance(rust_error, upstream_failed):
|
||||
args: Final = rust_error.args
|
||||
status: Final = args[0] if args else 0
|
||||
message: Final = args[1] if len(args) > 1 else ""
|
||||
upstream_failure: Final = rust_upstream_failure(rust_error, native_bridge)
|
||||
if upstream_failure is not None:
|
||||
raise APIError(
|
||||
status_code=int(status) or 500,
|
||||
message=f"litellm rust chat completions: {message}",
|
||||
status_code=upstream_failure.status_code,
|
||||
message=f"litellm rust chat completions: {upstream_failure.message}",
|
||||
llm_provider=custom_llm_provider or "",
|
||||
model=model,
|
||||
)
|
||||
if not isinstance(rust_error, declined):
|
||||
if not isinstance(rust_error, exceptions.declined):
|
||||
raise rust_error
|
||||
verbose_logger.debug(
|
||||
"Rust chat completions declined before calling the provider (%s); using the Python path",
|
||||
|
|
|
|||
38
litellm/rust_bridge/errors.py
Normal file
38
litellm/rust_bridge/errors.py
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class NativeBridgeExceptions:
|
||||
declined: type[BaseException]
|
||||
upstream: type[BaseException]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RustUpstreamFailure:
|
||||
status_code: int
|
||||
message: str
|
||||
|
||||
|
||||
def native_bridge_exceptions(native_bridge: object | None) -> NativeBridgeExceptions | None:
|
||||
if native_bridge is None:
|
||||
return None
|
||||
declined: Final = getattr(native_bridge, "RustBridgeDeclined", None)
|
||||
upstream: Final = getattr(native_bridge, "RustUpstreamError", None)
|
||||
if not isinstance(declined, type) or not isinstance(upstream, type):
|
||||
return None
|
||||
return NativeBridgeExceptions(declined=declined, upstream=upstream)
|
||||
|
||||
|
||||
def rust_upstream_failure(
|
||||
error: BaseException,
|
||||
native_bridge: object | None,
|
||||
) -> RustUpstreamFailure | None:
|
||||
exceptions: Final = native_bridge_exceptions(native_bridge)
|
||||
if exceptions is None or not isinstance(error, exceptions.upstream):
|
||||
return None
|
||||
status: Final = error.args[0] if error.args else 0
|
||||
message: Final = error.args[1] if len(error.args) > 1 else ""
|
||||
return RustUpstreamFailure(status_code=int(status) or 500, message=str(message))
|
||||
|
|
@ -66,25 +66,21 @@ _rust_aocr_impl: RustAocr | None = None
|
|||
|
||||
|
||||
class _OcrProviderError(Exception):
|
||||
def __init__(self, status_code: int, message: str, api_base: str | None) -> None:
|
||||
def __init__(self, status_code: int, message: str, request_url: str | None) -> None:
|
||||
super().__init__(message)
|
||||
self.status_code: Final = status_code
|
||||
self.response: Final = httpx.Response(
|
||||
status_code=status_code,
|
||||
request=httpx.Request("POST", api_base or "https://api.mistral.ai/v1/ocr"),
|
||||
)
|
||||
request: Final = httpx.Request("POST", request_url) if request_url is not None else None
|
||||
self.response: Final = httpx.Response(status_code=status_code, request=request)
|
||||
|
||||
|
||||
def _raise_provider_error(error: BaseException, api_base: str | None) -> None:
|
||||
def _raise_provider_error(error: BaseException, request_url: str | None) -> None:
|
||||
from litellm.rust_bridge import get_native_bridge
|
||||
from litellm.rust_bridge.errors import rust_upstream_failure
|
||||
|
||||
native_bridge: Final = get_native_bridge()
|
||||
upstream_error: Final = getattr(native_bridge, "RustUpstreamError", None) if native_bridge is not None else None
|
||||
if upstream_error is None or not isinstance(error, upstream_error):
|
||||
failure: Final = rust_upstream_failure(error, get_native_bridge())
|
||||
if failure is None:
|
||||
raise error
|
||||
status: Final = error.args[0] if error.args else 0
|
||||
message: Final = error.args[1] if len(error.args) > 1 else ""
|
||||
raise _OcrProviderError(int(status) or 500, str(message), api_base) from error
|
||||
raise _OcrProviderError(failure.status_code, failure.message, request_url) from error
|
||||
|
||||
|
||||
def use_litellm_rust(
|
||||
|
|
@ -170,6 +166,7 @@ def ocr(
|
|||
extra_headers: dict[str, object] | None,
|
||||
optional_params: dict[str, object],
|
||||
timeout: float | httpx.Timeout | None,
|
||||
request_url: str | None = None,
|
||||
) -> dict[str, object] | None:
|
||||
rust_ocr: Final = load_rust_ocr()
|
||||
if rust_ocr is None:
|
||||
|
|
@ -186,7 +183,7 @@ def ocr(
|
|||
timeout_seconds=_timeout_to_seconds(timeout),
|
||||
)
|
||||
except Exception as error:
|
||||
_raise_provider_error(error, api_base)
|
||||
_raise_provider_error(error, request_url)
|
||||
|
||||
|
||||
async def aocr(
|
||||
|
|
@ -199,6 +196,7 @@ async def aocr(
|
|||
extra_headers: dict[str, object] | None,
|
||||
optional_params: dict[str, object],
|
||||
timeout: float | httpx.Timeout | None,
|
||||
request_url: str | None = None,
|
||||
) -> dict[str, object] | None:
|
||||
rust_aocr: Final = load_rust_aocr()
|
||||
if rust_aocr is None:
|
||||
|
|
@ -215,4 +213,4 @@ async def aocr(
|
|||
timeout_seconds=_timeout_to_seconds(timeout),
|
||||
)
|
||||
except Exception as error:
|
||||
_raise_provider_error(error, api_base)
|
||||
_raise_provider_error(error, request_url)
|
||||
|
|
|
|||
|
|
@ -94,6 +94,11 @@ from fixture_mode import (
|
|||
current_test_key,
|
||||
parse_fixture_mode,
|
||||
)
|
||||
from tests.provider_record_replay.http import (
|
||||
dropped_request_headers,
|
||||
dropped_response_headers,
|
||||
is_streaming_response,
|
||||
)
|
||||
|
||||
EDGE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{
|
||||
|
|
@ -104,29 +109,6 @@ EDGE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType(
|
|||
|
||||
REPLAY_MISS_STATUS: Final = 599
|
||||
|
||||
_HOP_BY_HOP_HEADERS: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
"connection",
|
||||
"keep-alive",
|
||||
"proxy-authenticate",
|
||||
"proxy-authorization",
|
||||
"te",
|
||||
"trailers",
|
||||
"transfer-encoding",
|
||||
"upgrade",
|
||||
}
|
||||
)
|
||||
_REQUEST_DROPPED_HEADERS: Final[frozenset[str]] = _HOP_BY_HOP_HEADERS | {
|
||||
"host",
|
||||
"content-length",
|
||||
"accept-encoding",
|
||||
}
|
||||
_RESPONSE_DROPPED_HEADERS: Final[frozenset[str]] = _HOP_BY_HOP_HEADERS | {
|
||||
"content-encoding",
|
||||
"content-length",
|
||||
"set-cookie",
|
||||
}
|
||||
|
||||
_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
|
||||
|
||||
|
||||
|
|
@ -566,9 +548,8 @@ def _filtered_response_headers(headers: Mapping[str, str]) -> dict[str, str]:
|
|||
"""What the edge stores and serves: the provider's headers minus hop-by-hop and
|
||||
volatile entries. Framing headers are in that set, so a stored header can never
|
||||
contradict the framing the edge chooses when it serves the response."""
|
||||
return {
|
||||
name: value for name, value in headers.items() if name not in _RESPONSE_DROPPED_HEADERS
|
||||
}
|
||||
excluded: Final = dropped_response_headers(headers.items())
|
||||
return {name: value for name, value in headers.items() if name.lower() not in excluded}
|
||||
|
||||
|
||||
def _network_error_response(message: str) -> RecordedHttpResponse:
|
||||
|
|
@ -610,7 +591,7 @@ def _is_streamed(headers: Mapping[str, str]) -> bool:
|
|||
move nearly every recording to the streamed shape for no gain. The content type
|
||||
is the header that says "consume this as it arrives", and it is already how the
|
||||
harness defines streaming everywhere else."""
|
||||
return "text/event-stream" in _header_value(headers, "content-type").lower()
|
||||
return is_streaming_response(_header_value(headers, "content-type"))
|
||||
|
||||
|
||||
def _upstream_url(upstream_base: str, upstream_path: str, query: str) -> str:
|
||||
|
|
@ -698,8 +679,9 @@ def _handle_record(
|
|||
timeout: float,
|
||||
) -> EdgeOutcome:
|
||||
test_key: Final = current_test_key()
|
||||
excluded: Final = dropped_request_headers(headers.items())
|
||||
forwarded: Final = {
|
||||
name: value for name, value in headers.items() if name.lower() not in _REQUEST_DROPPED_HEADERS
|
||||
name: value for name, value in headers.items() if name.lower() not in excluded
|
||||
}
|
||||
head: Final = forward_stream(method, url, headers=forwarded, body=body, timeout=timeout)
|
||||
match head:
|
||||
|
|
|
|||
1
tests/provider_record_replay/__init__.py
Normal file
1
tests/provider_record_replay/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
|
||||
54
tests/provider_record_replay/http.py
Normal file
54
tests/provider_record_replay/http.py
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Final
|
||||
|
||||
HOP_BY_HOP_HEADERS: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
"connection",
|
||||
"keep-alive",
|
||||
"proxy-authenticate",
|
||||
"proxy-authorization",
|
||||
"te",
|
||||
"trailer",
|
||||
"trailers",
|
||||
"transfer-encoding",
|
||||
"upgrade",
|
||||
}
|
||||
)
|
||||
|
||||
REQUEST_DROPPED_HEADERS: Final[frozenset[str]] = HOP_BY_HOP_HEADERS | {
|
||||
"host",
|
||||
"content-length",
|
||||
"accept-encoding",
|
||||
}
|
||||
|
||||
RESPONSE_DROPPED_HEADERS: Final[frozenset[str]] = HOP_BY_HOP_HEADERS | {
|
||||
"content-encoding",
|
||||
"content-length",
|
||||
"set-cookie",
|
||||
}
|
||||
|
||||
|
||||
def connection_header_names(headers: Iterable[tuple[str, str]]) -> frozenset[str]:
|
||||
return frozenset(
|
||||
token.strip().lower()
|
||||
for name, value in headers
|
||||
if name.lower() == "connection"
|
||||
for token in value.split(",")
|
||||
if token.strip()
|
||||
)
|
||||
|
||||
|
||||
def dropped_request_headers(headers: Iterable[tuple[str, str]]) -> frozenset[str]:
|
||||
materialized: Final = tuple(headers)
|
||||
return REQUEST_DROPPED_HEADERS | connection_header_names(materialized)
|
||||
|
||||
|
||||
def dropped_response_headers(headers: Iterable[tuple[str, str]]) -> frozenset[str]:
|
||||
materialized: Final = tuple(headers)
|
||||
return RESPONSE_DROPPED_HEADERS | connection_header_names(materialized)
|
||||
|
||||
|
||||
def is_streaming_response(content_type: str) -> bool:
|
||||
return "text/event-stream" in content_type.lower()
|
||||
|
|
@ -1,11 +1,11 @@
|
|||
# Python/Rust parity testing in the Python SDK interface
|
||||
# Implementation parity testing through the SDK interface
|
||||
|
||||
> Given the same SDK call and identical provider behavior, does the PyO3 implementation behave same as Python?
|
||||
> Given the same SDK call and identical provider behavior, do two implementations expose the same SDK contract?
|
||||
|
||||
## What the harness compares
|
||||
|
||||
- A fixture contains a LiteLLM SDK input and a recorded upstream provider response
|
||||
- The same LiteLLM input is transformed by isolated Python and Rust workers
|
||||
- The same LiteLLM input is transformed by isolated baseline and candidate implementations
|
||||
- The resulting provider requests must match in method, path, headers, and body, excluding runtime-specific HTTP metadata
|
||||
- The recorded provider response is then replayed unchanged to both workers
|
||||
- The harness compares the values returned through the Python SDK interface
|
||||
|
|
@ -17,7 +17,7 @@
|
|||
|
||||
## Process isolation
|
||||
|
||||
- SDK object and stream parity runs Python and Rust sequentially in the same process so tests can retain the returned objects
|
||||
- SDK object and stream parity runs both implementations sequentially in the same process so tests can retain returned objects
|
||||
- Every test saves and restores the original bridge state
|
||||
- A small subprocess smoke test verifies environment-based startup configuration and detects fallback to the Python HTTP implementation
|
||||
|
||||
|
|
@ -28,59 +28,17 @@
|
|||
- Property-based tests define strategies for valid inputs and properties that must hold for every generated example
|
||||
- Hypothesis generates combinations from those strategies and normally shrinks a failing example to a smaller reproducible case
|
||||
- In this harness, Hypothesis is used only during fixture generation to expand the LiteLLM input corpus
|
||||
- The current OCR strategy varies supported optional parameters while keeping inputs valid
|
||||
- Each API owns the strategies that vary its supported inputs
|
||||
- Fixture generation is deterministic, and each generated input is recorded with the raw provider response it received
|
||||
- The parity tests use committed fixtures and do not call the provider or generate new Hypothesis examples
|
||||
- Provider responses are replayed unchanged, so the parity test does not fuzz or validate provider behavior
|
||||
- Because Hypothesis does not run the parity assertion directly, parity failures are not automatically shrunk
|
||||
|
||||
## Recording fixtures
|
||||
## API-owned fixtures
|
||||
|
||||
The recording command runs four explicit stages:
|
||||
|
||||
1. Generate deterministic SDK inputs for every configured OCR target
|
||||
2. Build target-scoped, deduplicated recording jobs
|
||||
3. Record upstream responses through one globally bounded worker pool
|
||||
4. Persist each fixture and report whether it was recorded, cached, or failed
|
||||
|
||||
Run it with:
|
||||
|
||||
```shell
|
||||
uv run python -m tests.test_litellm.ocr.fixtures.record --examples 4 --concurrency 4
|
||||
```
|
||||
|
||||
`--concurrency` caps all provider calls across all targets. Each completed job is reported immediately, and the final
|
||||
summary reports recorded, cached, and failed totals. Independent jobs finish after a failure, then the command exits
|
||||
nonzero if any job failed
|
||||
|
||||
## OCR input boundaries
|
||||
|
||||
OCR strategies generate only public `litellm.ocr()` and `litellm.aocr()` inputs. Every case contains the normalized
|
||||
`model`, Mistral-shaped `document`, optional `custom_llm_provider`, and LiteLLM keyword arguments. The fixture-only
|
||||
`boundary` tag selects the valid keyword-argument set and is removed before calling the SDK. Strategies never build
|
||||
provider wire payloads.
|
||||
|
||||
Each boundary has a required corpus containing a baseline and one case for every supported top-level LiteLLM OCR
|
||||
parameter for every active registered model that uses that transformation. Models whose registry deprecation date has
|
||||
passed are excluded. `--examples` controls additional Hypothesis-generated cases; it does not replace the required
|
||||
corpus.
|
||||
|
||||
The explicit boundaries are Mistral, Azure-hosted Mistral, Vertex-hosted Mistral, Azure Document Intelligence,
|
||||
Vertex DeepSeek, Reducto v3, and Reducto legacy. Provider credentials and endpoints only control target discovery, so
|
||||
a machine records the boundaries it has configured and skips the rest. Azure-hosted Mistral enumerates its active
|
||||
registered models rather than requiring a separately configured deployment model. Reducto fixtures record both upload
|
||||
and parse responses. Their parity cases are non-strict expected failures until the Rust OCR bridge supports Reducto, so
|
||||
both expected failures and unexpected passes keep CI green during the rollout.
|
||||
|
||||
The committed corpus does not need to contain live recordings for every configured target. In particular, Azure and
|
||||
Vertex generation paths are covered by unit tests without requiring their credentials in CI. Recordings can be added
|
||||
later without changing the fixture schema or runner.
|
||||
|
||||
Invalid OCR inputs do not use recorded provider responses. The parity suite checks unsupported providers and models,
|
||||
malformed documents, invalid request formats, invalid Azure Document Intelligence parameters, and invalid headers in
|
||||
both sync and async SDK calls. These cases must return the same public exception fields without sending a provider
|
||||
request. A malformed Azure Document Intelligence document reaches the Rust bridge so its native validation error is
|
||||
also compared against Python
|
||||
The shared package owns recording, replay, persistence, execution, comparison, and route-neutral media constructors.
|
||||
Each API package owns its input models, explicit strategies, provider targets, route-specific assets, fixture directory,
|
||||
and regeneration command. See the API package documentation for its configured contracts and recording command
|
||||
|
||||
## References
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Final, TypeVar
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final, TypeVar, cast
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
|
@ -9,26 +10,26 @@ from tests.route_parity.models import CapturedRequest, Execution
|
|||
ModelT = TypeVar("ModelT", bound=BaseModel)
|
||||
|
||||
|
||||
def validate_harness(python: Execution, accelerated: Execution, python_user_agent: str) -> None:
|
||||
for request in python.requests:
|
||||
if request.user_agent != python_user_agent:
|
||||
def validate_harness(baseline: Execution, candidate: Execution, baseline_user_agent: str) -> None:
|
||||
for request in baseline.requests:
|
||||
if request.user_agent != baseline_user_agent:
|
||||
raise AssertionError(
|
||||
f"Python provider request did not carry fallback sentinel user-agent {python_user_agent!r}: "
|
||||
f"baseline provider request did not carry sentinel user-agent {baseline_user_agent!r}: "
|
||||
f"{request.user_agent!r}"
|
||||
)
|
||||
for request in accelerated.requests:
|
||||
if request.user_agent == python_user_agent:
|
||||
raise AssertionError("accelerated route fell back to the Python HTTP implementation")
|
||||
for request in candidate.requests:
|
||||
if request.user_agent == baseline_user_agent:
|
||||
raise AssertionError("candidate route fell back to the baseline HTTP implementation")
|
||||
|
||||
|
||||
def _request_after_transformation(request: CapturedRequest) -> CapturedRequest:
|
||||
return request.model_copy(update={"user_agent": None})
|
||||
|
||||
|
||||
def assert_request_parity(python: tuple[CapturedRequest, ...], accelerated: tuple[CapturedRequest, ...]) -> None:
|
||||
python_requests: Final = tuple(_request_after_transformation(request) for request in python)
|
||||
accelerated_requests: Final = tuple(_request_after_transformation(request) for request in accelerated)
|
||||
assert python_requests == accelerated_requests
|
||||
def assert_request_parity(baseline: tuple[CapturedRequest, ...], candidate: tuple[CapturedRequest, ...]) -> None:
|
||||
baseline_requests: Final = tuple(_request_after_transformation(request) for request in baseline)
|
||||
candidate_requests: Final = tuple(_request_after_transformation(request) for request in candidate)
|
||||
assert baseline_requests == candidate_requests
|
||||
|
||||
|
||||
def public_model_copy(model: ModelT) -> ModelT:
|
||||
|
|
@ -37,12 +38,39 @@ def public_model_copy(model: ModelT) -> ModelT:
|
|||
return copied
|
||||
|
||||
|
||||
def assert_model_parity(python: BaseModel, accelerated: BaseModel) -> None:
|
||||
assert type(python) is type(accelerated)
|
||||
assert public_model_copy(python) == public_model_copy(accelerated)
|
||||
def assert_model_parity(baseline: BaseModel, candidate: BaseModel) -> None:
|
||||
assert type(baseline) is type(candidate)
|
||||
_assert_value_parity(
|
||||
public_model_copy(baseline).model_dump(mode="python"),
|
||||
public_model_copy(candidate).model_dump(mode="python"),
|
||||
path="$",
|
||||
)
|
||||
|
||||
|
||||
def assert_parity(python: Execution, accelerated: Execution, python_user_agent: str) -> None:
|
||||
validate_harness(python, accelerated, python_user_agent)
|
||||
assert_request_parity(python.requests, accelerated.requests)
|
||||
assert python.report == accelerated.report
|
||||
def _assert_value_parity(baseline: object, candidate: object, *, path: str) -> None:
|
||||
if isinstance(baseline, Mapping) and isinstance(candidate, Mapping):
|
||||
baseline_mapping: Final = cast(Mapping[object, object], baseline)
|
||||
candidate_mapping: Final = cast(Mapping[object, object], candidate)
|
||||
assert baseline_mapping.keys() == candidate_mapping.keys(), f"mapping keys differ at {path}"
|
||||
for key in baseline_mapping:
|
||||
_assert_value_parity(baseline_mapping[key], candidate_mapping[key], path=f"{path}.{key}")
|
||||
return
|
||||
if (
|
||||
isinstance(baseline, Sequence)
|
||||
and not isinstance(baseline, (str, bytes))
|
||||
and isinstance(candidate, Sequence)
|
||||
and not isinstance(candidate, (str, bytes))
|
||||
):
|
||||
baseline_sequence: Final = cast(Sequence[object], baseline)
|
||||
candidate_sequence: Final = cast(Sequence[object], candidate)
|
||||
assert len(baseline_sequence) == len(candidate_sequence), f"sequence lengths differ at {path}"
|
||||
for index, (baseline_item, candidate_item) in enumerate(zip(baseline_sequence, candidate_sequence)):
|
||||
_assert_value_parity(baseline_item, candidate_item, path=f"{path}[{index}]")
|
||||
return
|
||||
assert baseline == candidate, f"value mismatch at {path}: {baseline!r} != {candidate!r}"
|
||||
|
||||
|
||||
def assert_parity(baseline: Execution, candidate: Execution, baseline_user_agent: str) -> None:
|
||||
validate_harness(baseline, candidate, baseline_user_agent)
|
||||
assert_request_parity(baseline.requests, candidate.requests)
|
||||
assert baseline.report == candidate.report
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Final, Generic, Literal, TypeVar, cast
|
||||
from typing import ClassVar, Final, Generic, Literal, TypeVar, cast
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, model_validator
|
||||
|
||||
|
|
@ -15,15 +15,22 @@ class FixtureModel(BaseModel):
|
|||
|
||||
|
||||
class SdkInputBase(FixtureModel):
|
||||
boundary: str = "default"
|
||||
fixture_only_fields: ClassVar[tuple[str, ...]] = ()
|
||||
|
||||
def as_sdk_kwargs(self) -> dict[str, object]:
|
||||
dumped: Final = cast(dict[str, object], self.model_dump(mode="python", exclude_unset=True))
|
||||
return {key: value for key, value in dumped.items() if key != "boundary"}
|
||||
return cast(
|
||||
dict[str, object],
|
||||
self.model_dump(
|
||||
mode="python",
|
||||
exclude_unset=True,
|
||||
exclude=set(self.fixture_only_fields),
|
||||
),
|
||||
)
|
||||
|
||||
def canonical_input(self) -> dict[str, object]:
|
||||
dumped: Final = cast(dict[str, object], self.model_dump(mode="json", exclude_unset=True))
|
||||
return {"boundary": self.boundary, **{key: value for key, value in dumped.items() if key != "boundary"}}
|
||||
fixture_fields: Final = {field: getattr(self, field) for field in self.fixture_only_fields}
|
||||
return {**fixture_fields, **dumped}
|
||||
|
||||
|
||||
class JsonSchemaDefinition(FixtureModel):
|
||||
|
|
@ -50,29 +57,8 @@ class ParityCase(FixtureModel, Generic[InputT]):
|
|||
def load_legacy_single_response(cls, value: object) -> object:
|
||||
if not isinstance(value, Mapping):
|
||||
return value
|
||||
migrated: Final = dict(value)
|
||||
migrated: Final = dict(cast(Mapping[str, object], value))
|
||||
provider_response: Final = migrated.pop("provider_response", None)
|
||||
if "provider_responses" not in migrated and provider_response is not None:
|
||||
migrated["provider_responses"] = (provider_response,)
|
||||
litellm_input: Final = migrated.get("litellm_input")
|
||||
if isinstance(litellm_input, Mapping) and "boundary" not in litellm_input:
|
||||
model: Final = litellm_input.get("model")
|
||||
if isinstance(model, str):
|
||||
migrated["litellm_input"] = {"boundary": _legacy_boundary(model), **litellm_input}
|
||||
return migrated
|
||||
|
||||
|
||||
def _legacy_boundary(model: str) -> str:
|
||||
if model.startswith("azure_ai/doc-intelligence/"):
|
||||
return "azure_document_intelligence"
|
||||
if model.startswith("azure_ai/"):
|
||||
return "azure_mistral"
|
||||
if model.startswith("vertex_ai/deepseek"):
|
||||
return "vertex_deepseek"
|
||||
if model.startswith("vertex_ai/"):
|
||||
return "vertex_mistral"
|
||||
if model.endswith("parse-v3"):
|
||||
return "reducto_v3"
|
||||
if model.endswith("parse-legacy"):
|
||||
return "reducto_legacy"
|
||||
return "mistral"
|
||||
|
|
|
|||
34
tests/route_parity/fixtures/cli.py
Normal file
34
tests/route_parity/fixtures/cli.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final, cast
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RecordingArgs:
|
||||
concurrency: int
|
||||
examples: int
|
||||
fixture_dir: Path | None
|
||||
|
||||
|
||||
def _positive_int(value: str) -> int:
|
||||
parsed: Final = int(value)
|
||||
if parsed < 1:
|
||||
raise argparse.ArgumentTypeError("must be at least 1")
|
||||
return parsed
|
||||
|
||||
|
||||
def parse_recording_args(argv: Sequence[str] | None = None) -> RecordingArgs:
|
||||
parser: Final = argparse.ArgumentParser()
|
||||
parser.add_argument("--concurrency", type=_positive_int, default=4)
|
||||
parser.add_argument("--examples", type=_positive_int, default=4)
|
||||
parser.add_argument("--fixture-dir", type=Path)
|
||||
namespace: Final = parser.parse_args(argv)
|
||||
return RecordingArgs(
|
||||
concurrency=cast(int, namespace.concurrency),
|
||||
examples=cast(int, namespace.examples),
|
||||
fixture_dir=cast(Path | None, namespace.fixture_dir),
|
||||
)
|
||||
|
|
@ -19,9 +19,9 @@ def dummy_image_url(text: str, font_size: int, width: int = 800, height: int = 3
|
|||
|
||||
|
||||
_GLYPHS: Final = {
|
||||
"D": ("11110", "10001", "10001", "10001", "10001", "10001", "11110"),
|
||||
"O": ("01110", "10001", "10001", "10001", "10001", "10001", "01110"),
|
||||
"C": ("01111", "10000", "10000", "10000", "10000", "10000", "01111"),
|
||||
"R": ("11110", "10001", "10001", "11110", "10100", "10010", "10001"),
|
||||
"1": ("00100", "01100", "00100", "00100", "00100", "00100", "01110"),
|
||||
"2": ("01110", "10001", "00001", "00010", "00100", "01000", "11111"),
|
||||
"3": ("11110", "00001", "00001", "01110", "00001", "00001", "11110"),
|
||||
|
|
@ -34,7 +34,7 @@ def structured_image_bytes() -> bytes:
|
|||
draw: Final = ImageDraw.Draw(image)
|
||||
scale: Final = 8
|
||||
cursor_x = 24
|
||||
for character in "OCR 123":
|
||||
for character in "DOC 123":
|
||||
if character == " ":
|
||||
cursor_x += scale * 3
|
||||
continue
|
||||
|
|
@ -108,7 +108,7 @@ def _draw_table_page(pdf: canvas.Canvas) -> None:
|
|||
(
|
||||
("Item", "Quantity", "Amount", 707),
|
||||
("Document analysis", "2", "120.00", 672),
|
||||
("OCR verification", "1", "80.00", 637),
|
||||
("Document verification", "1", "80.00", 637),
|
||||
),
|
||||
),
|
||||
(
|
||||
|
|
@ -200,9 +200,9 @@ def structured_pdf_bytes() -> bytes:
|
|||
output: Final = BytesIO()
|
||||
pdf: Final = canvas.Canvas(output, pagesize=letter, pageCompression=0, invariant=1)
|
||||
pdf.setTitle("Quarterly Operations Report")
|
||||
pdf.setAuthor("LiteLLM OCR fixture generator")
|
||||
pdf.setSubject("Semantic OCR coverage for tables, figures, annotations, and metadata")
|
||||
pdf.setKeywords("OCR, invoice, table, figure, annotation")
|
||||
pdf.setAuthor("LiteLLM parity fixture generator")
|
||||
pdf.setSubject("Semantic document coverage for tables, figures, annotations, and metadata")
|
||||
pdf.setKeywords("document, invoice, table, figure, annotation")
|
||||
pages: Final = (
|
||||
("Invoice Summary and Line Items", _draw_table_page),
|
||||
("Revenue Chart and Formula Review", _draw_chart_page),
|
||||
|
|
|
|||
|
|
@ -1,19 +1,17 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
from collections.abc import Sequence
|
||||
from concurrent.futures import Future, ThreadPoolExecutor, as_completed
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Generic, Literal, Protocol, TypeVar, cast
|
||||
from typing import Final, Generic, Literal, Protocol, TypeVar
|
||||
|
||||
from hypothesis.strategies import SearchStrategy
|
||||
from pydantic import BaseModel
|
||||
|
||||
from tests.route_parity.fixtures.inputs import generate_case_inputs
|
||||
from tests.route_parity.fixtures.recording import ProviderSpec, record_upstream_responses
|
||||
from tests.route_parity.fixtures.recording import UpstreamEndpoint, record_upstream_responses
|
||||
from tests.route_parity.fixtures.store import (
|
||||
FixtureInput,
|
||||
canonical_json,
|
||||
|
|
@ -30,13 +28,6 @@ InputT_contra = TypeVar("InputT_contra", bound=FixtureInput, contravariant=True)
|
|||
CaseT = TypeVar("CaseT", bound=BaseModel)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RecordingArgs:
|
||||
concurrency: int
|
||||
examples: int
|
||||
fixture_dir: Path | None
|
||||
|
||||
|
||||
class RecordingInvocation(Protocol[InputT_contra]):
|
||||
def execute(self, provider_url: str, case_input: InputT_contra) -> None: ...
|
||||
|
||||
|
|
@ -44,7 +35,7 @@ class RecordingInvocation(Protocol[InputT_contra]):
|
|||
@dataclass(frozen=True, slots=True)
|
||||
class RecordingTarget(Generic[InputT]):
|
||||
name: str
|
||||
provider_spec: ProviderSpec
|
||||
upstream: UpstreamEndpoint
|
||||
strategy: SearchStrategy[InputT]
|
||||
invocation: RecordingInvocation[InputT] = field(repr=False)
|
||||
required_inputs: tuple[InputT, ...] = ()
|
||||
|
|
@ -54,7 +45,7 @@ class RecordingTarget(Generic[InputT]):
|
|||
class RecordingJob(Generic[InputT]):
|
||||
target_name: str
|
||||
directory: Path
|
||||
provider_spec: ProviderSpec
|
||||
upstream: UpstreamEndpoint
|
||||
case_input: InputT
|
||||
invocation: RecordingInvocation[InputT] = field(repr=False)
|
||||
|
||||
|
|
@ -118,7 +109,7 @@ def build_recording_jobs(
|
|||
RecordingJob(
|
||||
target_name=target.name,
|
||||
directory=root / target.name,
|
||||
provider_spec=target.provider_spec,
|
||||
upstream=target.upstream,
|
||||
case_input=case_input,
|
||||
invocation=target.invocation,
|
||||
)
|
||||
|
|
@ -136,7 +127,7 @@ def _record_job(job: RecordingJob[InputT], case_type: type[CaseT]) -> RecordedFi
|
|||
path=fixture_path(job.directory, job.case_input),
|
||||
)
|
||||
provider_responses: Final = record_upstream_responses(
|
||||
job.provider_spec,
|
||||
job.upstream,
|
||||
job.case_input,
|
||||
job.invocation.execute,
|
||||
)
|
||||
|
|
@ -199,23 +190,3 @@ def record_fixtures(
|
|||
len(summary.failed),
|
||||
)
|
||||
return summary
|
||||
|
||||
|
||||
def _positive_int(value: str) -> int:
|
||||
parsed: Final = int(value)
|
||||
if parsed < 1:
|
||||
raise argparse.ArgumentTypeError("must be at least 1")
|
||||
return parsed
|
||||
|
||||
|
||||
def parse_recording_args(argv: Sequence[str] | None = None) -> RecordingArgs:
|
||||
parser: Final = argparse.ArgumentParser()
|
||||
parser.add_argument("--concurrency", type=_positive_int, default=4)
|
||||
parser.add_argument("--examples", type=_positive_int, default=4)
|
||||
parser.add_argument("--fixture-dir", type=Path)
|
||||
namespace: Final = parser.parse_args(argv)
|
||||
return RecordingArgs(
|
||||
concurrency=cast(int, namespace.concurrency),
|
||||
examples=cast(int, namespace.examples),
|
||||
fixture_dir=cast(Path | None, namespace.fixture_dir),
|
||||
)
|
||||
|
|
|
|||
66
tests/route_parity/fixtures/pytest_support.py
Normal file
66
tests/route_parity/fixtures/pytest_support.py
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import Final, TypeVar
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
from tests.route_parity.fixtures.store import recorded_fixtures
|
||||
|
||||
CaseT = TypeVar("CaseT", bound=BaseModel)
|
||||
|
||||
|
||||
def parametrize_recorded_fixtures(
|
||||
metafunc: pytest.Metafunc,
|
||||
*,
|
||||
fixture_name: str,
|
||||
case_type: type[CaseT],
|
||||
env_var: str,
|
||||
default_directory: Path,
|
||||
regeneration_command: str,
|
||||
id_builder: Callable[[CaseT], str],
|
||||
marks_builder: Callable[[CaseT], tuple[pytest.MarkDecorator, ...]] | None = None,
|
||||
) -> None:
|
||||
if fixture_name not in metafunc.fixturenames:
|
||||
return
|
||||
configured: Final = os.environ.get(env_var)
|
||||
if configured == "":
|
||||
raise pytest.UsageError(f"{env_var} is set but empty")
|
||||
directory: Final = Path(configured).expanduser() if configured is not None else default_directory
|
||||
try:
|
||||
fixtures: Final = recorded_fixtures(directory, case_type)
|
||||
except (ValidationError, ValueError) as error:
|
||||
raise pytest.UsageError(
|
||||
f"Invalid parity fixture bundle at {directory}. "
|
||||
"Each fixture must use the current versioned envelope. "
|
||||
f"Record fresh fixtures in an empty directory with: `{regeneration_command}`. "
|
||||
f"Validation details: {error}"
|
||||
) from error
|
||||
if fixtures:
|
||||
metafunc.parametrize(
|
||||
fixture_name,
|
||||
tuple(
|
||||
pytest.param(
|
||||
fixture,
|
||||
id=id_builder(fixture),
|
||||
marks=marks_builder(fixture) if marks_builder is not None else (),
|
||||
)
|
||||
for fixture in fixtures
|
||||
),
|
||||
)
|
||||
return
|
||||
if configured is not None:
|
||||
raise pytest.UsageError(f"no recorded fixtures in {directory}")
|
||||
metafunc.parametrize(
|
||||
fixture_name,
|
||||
(
|
||||
pytest.param(
|
||||
None,
|
||||
marks=pytest.mark.skip(reason=f"no recorded fixtures in {directory}"),
|
||||
id="no-recorded-fixtures",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
|
@ -11,6 +11,11 @@ from urllib.parse import urlsplit, urlunsplit
|
|||
|
||||
import httpx
|
||||
|
||||
from tests.provider_record_replay.http import (
|
||||
dropped_request_headers,
|
||||
dropped_response_headers,
|
||||
is_streaming_response,
|
||||
)
|
||||
from tests.route_parity.recorded_http import (
|
||||
HttpHeader,
|
||||
RecordedHttpResponse,
|
||||
|
|
@ -21,39 +26,17 @@ from tests.route_parity.recorded_http import (
|
|||
|
||||
_PARITY_PROVIDER_HOST: Final = "parity-provider.invalid"
|
||||
|
||||
_HOP_BY_HOP_HEADERS: Final = frozenset(
|
||||
{
|
||||
"connection",
|
||||
"keep-alive",
|
||||
"proxy-authenticate",
|
||||
"proxy-authorization",
|
||||
"te",
|
||||
"trailer",
|
||||
"transfer-encoding",
|
||||
"upgrade",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
InputT = TypeVar("InputT")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProviderSpec:
|
||||
upstream_base: str
|
||||
|
||||
|
||||
def _excluded_headers(headers: tuple[tuple[str, str], ...]) -> frozenset[str]:
|
||||
connection_values: Final = tuple(value for name, value in headers if name.lower() == "connection")
|
||||
connection_headers: Final = frozenset(
|
||||
token.strip().lower() for value in connection_values for token in value.split(",") if token.strip()
|
||||
)
|
||||
return _HOP_BY_HOP_HEADERS | connection_headers
|
||||
class UpstreamEndpoint:
|
||||
base_url: str
|
||||
|
||||
|
||||
def _end_to_end_headers(headers: httpx.Headers) -> tuple[HttpHeader, ...]:
|
||||
decoded: Final = tuple((name.decode("ascii"), value.decode("latin-1")) for name, value in headers.raw)
|
||||
excluded: Final = _excluded_headers(decoded) | {"content-encoding", "content-length"}
|
||||
excluded: Final = dropped_response_headers(decoded)
|
||||
return tuple(
|
||||
HttpHeader(name=name, value=_normalized_response_header(name, value))
|
||||
for name, value in decoded
|
||||
|
|
@ -82,7 +65,7 @@ def local_response_header(name: str, value: str, provider_url: str) -> str:
|
|||
class _RecordingProvider(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
|
||||
def __init__(self, spec: ProviderSpec) -> None:
|
||||
def __init__(self, spec: UpstreamEndpoint) -> None:
|
||||
super().__init__(("127.0.0.1", 0), _RecordingHandler)
|
||||
self.spec: Final = spec
|
||||
self.responses: queue.Queue[RecordedResponse] = queue.Queue()
|
||||
|
|
@ -109,15 +92,24 @@ class _RecordingHandler(BaseHTTPRequestHandler):
|
|||
def do_GET(self) -> None:
|
||||
self._forward()
|
||||
|
||||
def do_PUT(self) -> None:
|
||||
self._forward()
|
||||
|
||||
def do_PATCH(self) -> None:
|
||||
self._forward()
|
||||
|
||||
def do_DELETE(self) -> None:
|
||||
self._forward()
|
||||
|
||||
def _forward(self) -> None:
|
||||
provider: Final = self.server
|
||||
assert isinstance(provider, _RecordingProvider)
|
||||
length: Final = int(self.headers.get("content-length") or "0")
|
||||
request_body: Final = self.rfile.read(length) if length else b""
|
||||
raw_headers: Final = tuple(self.headers.raw_items())
|
||||
excluded: Final = _excluded_headers(raw_headers) | {"host", "content-length"}
|
||||
excluded: Final = dropped_request_headers(raw_headers)
|
||||
forwarded_headers: Final = tuple((name, value) for name, value in raw_headers if name.lower() not in excluded)
|
||||
upstream_url: Final = f"{provider.spec.upstream_base.rstrip('/')}{self.path}"
|
||||
upstream_url: Final = f"{provider.spec.base_url.rstrip('/')}{self.path}"
|
||||
|
||||
try:
|
||||
with httpx.stream(
|
||||
|
|
@ -145,7 +137,7 @@ class _RecordingHandler(BaseHTTPRequestHandler):
|
|||
headers: tuple[HttpHeader, ...],
|
||||
) -> RecordedResponse:
|
||||
content_type: Final = cast(str, upstream.headers.get("content-type", ""))
|
||||
if content_type.lower().startswith("text/event-stream"):
|
||||
if is_streaming_response(content_type):
|
||||
return self._record_stream(upstream, headers)
|
||||
response_body: Final = b"".join(upstream.iter_bytes())
|
||||
return RecordedHttpResponse.from_bytes(
|
||||
|
|
@ -199,7 +191,7 @@ class _RecordingHandler(BaseHTTPRequestHandler):
|
|||
|
||||
|
||||
@contextmanager
|
||||
def _recording_provider(spec: ProviderSpec) -> Generator[_RecordingProvider]:
|
||||
def _recording_provider(spec: UpstreamEndpoint) -> Generator[_RecordingProvider]:
|
||||
server: Final = _RecordingProvider(spec)
|
||||
thread: Final = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
|
|
@ -227,7 +219,7 @@ def _invoke_and_take_responses(
|
|||
|
||||
|
||||
def record_upstream_responses(
|
||||
spec: ProviderSpec,
|
||||
spec: UpstreamEndpoint,
|
||||
case_input: InputT,
|
||||
sdk_call: Callable[[str, InputT], object],
|
||||
) -> tuple[RecordedResponse, ...]:
|
||||
|
|
|
|||
|
|
@ -2,13 +2,11 @@ from __future__ import annotations
|
|||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Callable, Mapping
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Final, Protocol, TypeVar, cast
|
||||
|
||||
import pytest
|
||||
from pydantic import AwareDatetime, BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
|
||||
FIXTURE_SCHEMA_VERSION: Final = 1
|
||||
|
|
@ -99,56 +97,3 @@ def fixture_id(case_input: FixtureInput, prefix: str) -> str:
|
|||
input_json: Final = canonical_json(case_input.canonical_input())
|
||||
digest: Final = hashlib.sha256(input_json.encode("utf-8")).hexdigest()[:8]
|
||||
return f"{prefix}-{digest}"
|
||||
|
||||
|
||||
def parametrize_recorded_fixtures(
|
||||
metafunc: pytest.Metafunc,
|
||||
*,
|
||||
fixture_name: str,
|
||||
case_type: type[CaseT],
|
||||
env_var: str,
|
||||
default_directory: Path,
|
||||
regeneration_command: str,
|
||||
id_builder: Callable[[CaseT], str],
|
||||
marks_builder: Callable[[CaseT], tuple[pytest.MarkDecorator, ...]] | None = None,
|
||||
) -> None:
|
||||
if fixture_name not in metafunc.fixturenames:
|
||||
return
|
||||
configured: Final = os.environ.get(env_var)
|
||||
if configured == "":
|
||||
raise pytest.UsageError(f"{env_var} is set but empty")
|
||||
directory: Final = Path(configured).expanduser() if configured is not None else default_directory
|
||||
try:
|
||||
fixtures: Final = recorded_fixtures(directory, case_type)
|
||||
except (ValidationError, ValueError) as error:
|
||||
raise pytest.UsageError(
|
||||
f"Invalid parity fixture bundle at {directory}. "
|
||||
"Each fixture must use the current versioned envelope. "
|
||||
f"Record fresh fixtures in an empty directory with: `{regeneration_command}`. "
|
||||
f"Validation details: {error}"
|
||||
) from error
|
||||
if fixtures:
|
||||
metafunc.parametrize(
|
||||
fixture_name,
|
||||
tuple(
|
||||
pytest.param(
|
||||
fixture,
|
||||
id=id_builder(fixture),
|
||||
marks=marks_builder(fixture) if marks_builder is not None else (),
|
||||
)
|
||||
for fixture in fixtures
|
||||
),
|
||||
)
|
||||
return
|
||||
if configured is not None:
|
||||
raise pytest.UsageError(f"no recorded fixtures in {directory}")
|
||||
metafunc.parametrize(
|
||||
fixture_name,
|
||||
(
|
||||
pytest.param(
|
||||
None,
|
||||
marks=pytest.mark.skip(reason=f"no recorded fixtures in {directory}"),
|
||||
id="no-recorded-fixtures",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from tests.route_parity.fixtures.pipeline import (
|
|||
build_recording_jobs,
|
||||
record_fixtures,
|
||||
)
|
||||
from tests.route_parity.fixtures.recording import ProviderSpec
|
||||
from tests.route_parity.fixtures.recording import UpstreamEndpoint
|
||||
from tests.route_parity.fixtures.store import fixture_path
|
||||
from tests.route_parity.recorded_http import RecordedResponse
|
||||
|
||||
|
|
@ -117,7 +117,7 @@ def _target(
|
|||
) -> RecordingTarget[_FixtureInput]:
|
||||
return RecordingTarget(
|
||||
name=name,
|
||||
provider_spec=ProviderSpec(upstream_base=upstream_url),
|
||||
upstream=UpstreamEndpoint(base_url=upstream_url),
|
||||
strategy=st.just(case_input),
|
||||
invocation=invocation,
|
||||
required_inputs=(case_input,),
|
||||
|
|
@ -129,7 +129,7 @@ def test_build_jobs_keeps_required_inputs_before_generated_inputs_and_deduplicat
|
|||
generated: Final = _FixtureInput(identifier="generated")
|
||||
target: Final = RecordingTarget(
|
||||
name="ordered",
|
||||
provider_spec=ProviderSpec(upstream_base="https://provider.invalid"),
|
||||
upstream=UpstreamEndpoint(base_url="https://provider.invalid"),
|
||||
strategy=st.just(generated),
|
||||
invocation=_Invocation(),
|
||||
required_inputs=(required, required),
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from hypothesis import strategies as st
|
|||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from tests.route_parity.fixtures.pipeline import RecordingTarget, record_fixtures
|
||||
from tests.route_parity.fixtures.recording import ProviderSpec, record_upstream_responses
|
||||
from tests.route_parity.fixtures.recording import UpstreamEndpoint, record_upstream_responses
|
||||
from tests.route_parity.fixtures.store import (
|
||||
FIXTURE_SCHEMA_VERSION,
|
||||
fixture_path,
|
||||
|
|
@ -97,7 +97,7 @@ class _ControlledUpstreamHandler(BaseHTTPRequestHandler):
|
|||
length: Final = int(self.headers.get("content-length") or "0")
|
||||
self.rfile.read(length)
|
||||
if self.path == "/upload":
|
||||
self._send_json(200, b'{"file_id":"reducto://fixture.pdf"}')
|
||||
self._send_json(200, b'{"file_id":"fixture://document.pdf"}')
|
||||
return
|
||||
if self.path == "/parse":
|
||||
self._send_json(200, b'{"result":{"chunks":[]}}')
|
||||
|
|
@ -131,6 +131,7 @@ class _ControlledUpstreamHandler(BaseHTTPRequestHandler):
|
|||
body: Final = b"{}"
|
||||
self.send_response(200)
|
||||
self.send_header("content-type", "application/json")
|
||||
self.send_header("set-cookie", "session=must-not-be-recorded")
|
||||
self.send_header("content-length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
|
@ -143,6 +144,15 @@ class _ControlledUpstreamHandler(BaseHTTPRequestHandler):
|
|||
return
|
||||
self.send_error(404)
|
||||
|
||||
def do_PUT(self) -> None:
|
||||
self.do_POST()
|
||||
|
||||
def do_PATCH(self) -> None:
|
||||
self.do_POST()
|
||||
|
||||
def do_DELETE(self) -> None:
|
||||
self.do_POST()
|
||||
|
||||
def _send_json(self, status: int, body: bytes) -> None:
|
||||
self.send_response(status)
|
||||
self.send_header("content-type", "application/json")
|
||||
|
|
@ -172,7 +182,7 @@ def _case(identifier: str) -> _FixtureInput:
|
|||
|
||||
|
||||
def _sdk_call(api_base: str, case_input: _FixtureInput) -> object:
|
||||
return httpx.post(f"{api_base}/v1/ocr", content=b"{}", timeout=5)
|
||||
return httpx.post(f"{api_base}/v1/operation", content=b"{}", timeout=5)
|
||||
|
||||
|
||||
def _stream_sdk_call(api_base: str, case_input: _FixtureInput) -> object:
|
||||
|
|
@ -185,6 +195,13 @@ def _error_sdk_call(api_base: str, case_input: _FixtureInput) -> object:
|
|||
return response
|
||||
|
||||
|
||||
def _method_sdk_call(method: str) -> Callable[[str, _FixtureInput], object]:
|
||||
def call(api_base: str, case_input: _FixtureInput) -> object:
|
||||
return httpx.request(method, f"{api_base}/method", json={"id": case_input.identifier}, timeout=5)
|
||||
|
||||
return call
|
||||
|
||||
|
||||
def _multi_sdk_call(api_base: str, case_input: _FixtureInput) -> object:
|
||||
upload: Final = httpx.post(f"{api_base}/upload", json={"document": case_input.identifier}, timeout=5)
|
||||
upload.raise_for_status()
|
||||
|
|
@ -204,18 +221,18 @@ def _polling_sdk_call(api_base: str, case_input: _FixtureInput) -> object:
|
|||
def test_recording_deduplicates_per_target_and_caps_global_concurrency(tmp_path: Path) -> None:
|
||||
shared_input: Final = _case("shared")
|
||||
with _controlled_upstream() as upstream:
|
||||
spec: Final = ProviderSpec(upstream_base=upstream.url)
|
||||
spec: Final = UpstreamEndpoint(base_url=upstream.url)
|
||||
targets: Final = (
|
||||
RecordingTarget(
|
||||
name="first",
|
||||
provider_spec=spec,
|
||||
upstream=spec,
|
||||
strategy=st.just(shared_input),
|
||||
invocation=_Invocation(_sdk_call),
|
||||
required_inputs=(shared_input, shared_input),
|
||||
),
|
||||
RecordingTarget(
|
||||
name="second",
|
||||
provider_spec=spec,
|
||||
upstream=spec,
|
||||
strategy=st.just(shared_input),
|
||||
invocation=_Invocation(_sdk_call),
|
||||
required_inputs=(shared_input,),
|
||||
|
|
@ -244,7 +261,7 @@ def test_pipeline_rejects_stale_fixture_before_provider_call(tmp_path: Path) ->
|
|||
path.write_text('{"schema_version": 0}\n', encoding="utf-8")
|
||||
target: Final = RecordingTarget(
|
||||
name="stale-target",
|
||||
provider_spec=ProviderSpec(upstream_base="http://127.0.0.1:1"),
|
||||
upstream=UpstreamEndpoint(base_url="http://127.0.0.1:1"),
|
||||
strategy=st.just(case_input),
|
||||
invocation=_Invocation(_sdk_call),
|
||||
)
|
||||
|
|
@ -271,7 +288,7 @@ def test_cached_fixture_is_reported_without_provider_call(tmp_path: Path) -> Non
|
|||
with _controlled_upstream() as upstream:
|
||||
target: Final = RecordingTarget(
|
||||
name="cached-target",
|
||||
provider_spec=ProviderSpec(upstream_base=upstream.url),
|
||||
upstream=UpstreamEndpoint(base_url=upstream.url),
|
||||
strategy=st.just(case_input),
|
||||
invocation=_Invocation(_sdk_call),
|
||||
)
|
||||
|
|
@ -286,7 +303,7 @@ def test_cached_fixture_is_reported_without_provider_call(tmp_path: Path) -> Non
|
|||
def test_streaming_response_records_and_replays_chunks() -> None:
|
||||
with _controlled_upstream() as upstream:
|
||||
responses: Final = record_upstream_responses(
|
||||
ProviderSpec(upstream_base=upstream.url),
|
||||
UpstreamEndpoint(base_url=upstream.url),
|
||||
_case("stream"),
|
||||
_stream_sdk_call,
|
||||
)
|
||||
|
|
@ -308,7 +325,7 @@ def test_streaming_response_records_and_replays_chunks() -> None:
|
|||
def test_non_successful_provider_response_is_recorded() -> None:
|
||||
with _controlled_upstream() as upstream:
|
||||
responses: Final = record_upstream_responses(
|
||||
ProviderSpec(upstream_base=upstream.url),
|
||||
UpstreamEndpoint(base_url=upstream.url),
|
||||
_case("provider-error"),
|
||||
_error_sdk_call,
|
||||
)
|
||||
|
|
@ -317,6 +334,34 @@ def test_non_successful_provider_response_is_recorded() -> None:
|
|||
assert response.status_code == 429
|
||||
|
||||
|
||||
def test_sensitive_response_headers_are_not_recorded() -> None:
|
||||
with _controlled_upstream() as upstream:
|
||||
responses: Final = record_upstream_responses(
|
||||
UpstreamEndpoint(base_url=upstream.url),
|
||||
_case("headers"),
|
||||
_sdk_call,
|
||||
)
|
||||
|
||||
assert all(header.name.lower() != "set-cookie" for header in responses[0].headers)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ("PUT", "PATCH", "DELETE"))
|
||||
def test_recording_and_replay_support_mutating_http_methods(method: str) -> None:
|
||||
sdk_call: Final = _method_sdk_call(method)
|
||||
with _controlled_upstream() as upstream:
|
||||
responses: Final = record_upstream_responses(
|
||||
UpstreamEndpoint(base_url=upstream.url),
|
||||
_case(method),
|
||||
sdk_call,
|
||||
)
|
||||
with replay_server() as provider:
|
||||
provider.enqueue_response(responses[0])
|
||||
sdk_call(provider.url, _case(method))
|
||||
requests: Final = provider.take_requests(1)
|
||||
|
||||
assert requests[0].method == method
|
||||
|
||||
|
||||
def test_stream_response_model_rejects_buffered_body() -> None:
|
||||
with pytest.raises(ValueError, match="Extra inputs are not permitted"):
|
||||
RecordedHttpStreamResponse.model_validate(
|
||||
|
|
@ -336,7 +381,7 @@ def test_multiple_provider_calls_record_and_replay_in_order(
|
|||
) -> None:
|
||||
with _controlled_upstream() as upstream:
|
||||
responses: Final = record_upstream_responses(
|
||||
ProviderSpec(upstream_base=upstream.url),
|
||||
UpstreamEndpoint(base_url=upstream.url),
|
||||
_case(sdk_call.__name__),
|
||||
sdk_call,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -65,6 +65,15 @@ class _ReplayHandler(BaseHTTPRequestHandler):
|
|||
def do_GET(self) -> None:
|
||||
self._replay()
|
||||
|
||||
def do_PUT(self) -> None:
|
||||
self._replay()
|
||||
|
||||
def do_PATCH(self) -> None:
|
||||
self._replay()
|
||||
|
||||
def do_DELETE(self) -> None:
|
||||
self._replay()
|
||||
|
||||
def _replay(self) -> None:
|
||||
provider: Final = self.server
|
||||
assert isinstance(provider, ReplayServer)
|
||||
|
|
|
|||
|
|
@ -29,10 +29,9 @@ WORKER_RESULT_ADAPTER: Final[TypeAdapter[WorkerResult]] = TypeAdapter(WorkerResu
|
|||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PythonScriptRunner:
|
||||
class SubprocessRunner:
|
||||
entrypoint: Path
|
||||
rust_env_var: str
|
||||
python_user_agent: str
|
||||
baseline_user_agent: str
|
||||
route_label: str
|
||||
|
||||
def command(self, provider_url: str) -> tuple[str, ...]:
|
||||
|
|
@ -44,17 +43,23 @@ class PythonScriptRunner:
|
|||
)
|
||||
|
||||
|
||||
class PythonScriptWorker:
|
||||
def __init__(self, runner: PythonScriptRunner, provider: ReplayServer, rust_enabled: bool) -> None:
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ExecutionVariant:
|
||||
name: str
|
||||
environment: tuple[tuple[str, str], ...]
|
||||
|
||||
|
||||
class SubprocessWorker:
|
||||
def __init__(self, runner: SubprocessRunner, provider: ReplayServer, variant: ExecutionVariant) -> None:
|
||||
project_root: Final = str(runner.entrypoint.resolve().parents[3])
|
||||
existing_pythonpath: Final = os.environ.get("PYTHONPATH")
|
||||
env: Final = {
|
||||
**os.environ,
|
||||
runner.rust_env_var: "1" if rust_enabled else "0",
|
||||
"LITELLM_USER_AGENT": runner.python_user_agent,
|
||||
**dict(variant.environment),
|
||||
"LITELLM_USER_AGENT": runner.baseline_user_agent,
|
||||
"PYTHONPATH": os.pathsep.join(path for path in (project_root, existing_pythonpath) if path),
|
||||
}
|
||||
self.mode: Final = "Rust" if rust_enabled else "Python"
|
||||
self.mode: Final = variant.name
|
||||
self.route_label: Final = runner.route_label
|
||||
self.provider: Final = provider
|
||||
self.process: Final = subprocess.Popen(
|
||||
|
|
@ -146,11 +151,11 @@ class PythonScriptWorker:
|
|||
|
||||
@contextmanager
|
||||
def execution_worker(
|
||||
runner: PythonScriptRunner,
|
||||
rust_enabled: bool,
|
||||
) -> Generator[PythonScriptWorker]:
|
||||
runner: SubprocessRunner,
|
||||
variant: ExecutionVariant,
|
||||
) -> Generator[SubprocessWorker]:
|
||||
with replay_server() as provider:
|
||||
worker: Final = PythonScriptWorker(runner, provider, rust_enabled)
|
||||
worker: Final = SubprocessWorker(runner, provider, variant)
|
||||
try:
|
||||
yield worker
|
||||
finally:
|
||||
|
|
@ -158,7 +163,7 @@ def execution_worker(
|
|||
|
||||
|
||||
def run_execution(
|
||||
worker: PythonScriptWorker,
|
||||
worker: SubprocessWorker,
|
||||
case_file: Path,
|
||||
route: str,
|
||||
responses: tuple[RecordedResponse, ...],
|
||||
|
|
@ -168,11 +173,13 @@ def run_execution(
|
|||
|
||||
@contextmanager
|
||||
def execution_worker_pair(
|
||||
runner: PythonScriptRunner,
|
||||
) -> Generator[tuple[PythonScriptWorker, PythonScriptWorker]]:
|
||||
with execution_worker(runner, rust_enabled=False) as python_worker:
|
||||
with execution_worker(runner, rust_enabled=True) as accelerated_worker:
|
||||
yield python_worker, accelerated_worker
|
||||
runner: SubprocessRunner,
|
||||
baseline: ExecutionVariant,
|
||||
candidate: ExecutionVariant,
|
||||
) -> Generator[tuple[SubprocessWorker, SubprocessWorker]]:
|
||||
with execution_worker(runner, baseline) as baseline_worker:
|
||||
with execution_worker(runner, candidate) as candidate_worker:
|
||||
yield baseline_worker, candidate_worker
|
||||
|
||||
|
||||
def parity_worker_main(
|
||||
|
|
|
|||
|
|
@ -185,16 +185,16 @@ def normalize_chunk(chunk: object) -> object:
|
|||
|
||||
|
||||
def assert_stream_parity(
|
||||
python: StreamOutcome,
|
||||
accelerated: StreamOutcome,
|
||||
baseline: StreamOutcome,
|
||||
candidate: StreamOutcome,
|
||||
*,
|
||||
normalize: ChunkNormalizer = normalize_chunk,
|
||||
) -> None:
|
||||
assert python.wrapper_type is accelerated.wrapper_type
|
||||
assert python.supports_sync_iteration is accelerated.supports_sync_iteration
|
||||
assert python.supports_async_iteration is accelerated.supports_async_iteration
|
||||
assert python.chunk_types == accelerated.chunk_types
|
||||
assert len(python.chunks) == len(accelerated.chunks)
|
||||
for python_chunk, accelerated_chunk in zip(python.chunks, accelerated.chunks, strict=True):
|
||||
assert normalize(python_chunk) == normalize(accelerated_chunk)
|
||||
assert python.terminal == accelerated.terminal
|
||||
assert baseline.wrapper_type is candidate.wrapper_type
|
||||
assert baseline.supports_sync_iteration is candidate.supports_sync_iteration
|
||||
assert baseline.supports_async_iteration is candidate.supports_async_iteration
|
||||
assert baseline.chunk_types == candidate.chunk_types
|
||||
assert len(baseline.chunks) == len(candidate.chunks)
|
||||
for baseline_chunk, candidate_chunk in zip(baseline.chunks, candidate.chunks, strict=True):
|
||||
assert normalize(baseline_chunk) == normalize(candidate_chunk)
|
||||
assert baseline.terminal == candidate.terminal
|
||||
|
|
|
|||
|
|
@ -23,6 +23,10 @@ class _DifferentResponse(BaseModel):
|
|||
value: str
|
||||
|
||||
|
||||
class _FloatResponse(BaseModel):
|
||||
values: list[float]
|
||||
|
||||
|
||||
class _PublicError(ValueError):
|
||||
status_code: Final = 400
|
||||
|
||||
|
|
@ -69,7 +73,7 @@ def test_parity_rejects_error_difference() -> None:
|
|||
error_type=None,
|
||||
param=None,
|
||||
model="test-model",
|
||||
llm_provider="mistral",
|
||||
llm_provider="test-provider",
|
||||
),
|
||||
)
|
||||
rust: Final = python.model_copy(update={"report": python.report.model_copy(update={"status_code": 500})})
|
||||
|
|
@ -116,3 +120,19 @@ def test_model_parity_rejects_public_value_difference() -> None:
|
|||
def test_model_parity_rejects_type_difference() -> None:
|
||||
with pytest.raises(AssertionError):
|
||||
assert_model_parity(_ComparableResponse(value="same"), _DifferentResponse(value="same"))
|
||||
|
||||
|
||||
def test_model_parity_rejects_wire_float_rounding_difference() -> None:
|
||||
with pytest.raises(AssertionError, match=r"\$\.values\[0\]"):
|
||||
assert_model_parity(
|
||||
_FloatResponse(values=[0.22590550796036835]),
|
||||
_FloatResponse(values=[0.22590550796036837]),
|
||||
)
|
||||
|
||||
|
||||
def test_model_parity_rejects_meaningful_float_difference() -> None:
|
||||
with pytest.raises(AssertionError, match=r"\$\.values\[0\]"):
|
||||
assert_model_parity(
|
||||
_FloatResponse(values=[0.22590550796036835]),
|
||||
_FloatResponse(values=[0.2259]),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,15 +1,14 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.route_parity.fixtures.store import fixture_id, parametrize_recorded_fixtures
|
||||
from tests.route_parity.fixtures.pytest_support import parametrize_recorded_fixtures
|
||||
from tests.route_parity.fixtures.store import fixture_id
|
||||
from tests.test_litellm.ocr.fixtures.config import DEFAULT_FIXTURE_DIRECTORY, FIXTURE_DIR_ENV
|
||||
from tests.test_litellm.ocr.fixtures.models import OcrParityCase
|
||||
|
||||
FIXTURE_DIR_ENV: Final = "LITELLM_OCR_FIXTURE_DIR"
|
||||
|
||||
|
||||
def ocr_fixture_id(fixture: OcrParityCase) -> str:
|
||||
case_input: Final = fixture.litellm_input
|
||||
|
|
@ -19,26 +18,25 @@ def ocr_fixture_id(fixture: OcrParityCase) -> str:
|
|||
|
||||
|
||||
def ocr_fixture_marks(fixture: OcrParityCase) -> tuple[pytest.MarkDecorator, ...]:
|
||||
if fixture.litellm_input.boundary not in {"reducto_v3", "reducto_legacy"}:
|
||||
if fixture.litellm_input.contract not in {"reducto_v3", "reducto_legacy"}:
|
||||
return ()
|
||||
return (
|
||||
pytest.mark.xfail(
|
||||
reason="Reducto does not have a Rust OCR boundary",
|
||||
reason="Reducto does not have a Rust OCR contract",
|
||||
strict=False,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
|
||||
default_directory: Final = Path(__file__).with_name("fixtures") / "data"
|
||||
parametrize_recorded_fixtures(
|
||||
metafunc,
|
||||
fixture_name="ocr_fixture",
|
||||
case_type=OcrParityCase,
|
||||
env_var=FIXTURE_DIR_ENV,
|
||||
default_directory=default_directory,
|
||||
default_directory=DEFAULT_FIXTURE_DIRECTORY,
|
||||
regeneration_command=(
|
||||
f"uv run python -m tests.test_litellm.ocr.fixtures.record --fixture-dir {default_directory}"
|
||||
f"uv run python -m tests.test_litellm.ocr.fixtures.record --fixture-dir {DEFAULT_FIXTURE_DIRECTORY}"
|
||||
),
|
||||
id_builder=ocr_fixture_id,
|
||||
marks_builder=ocr_fixture_marks,
|
||||
|
|
|
|||
35
tests/test_litellm/ocr/fixtures/README.md
Normal file
35
tests/test_litellm/ocr/fixtures/README.md
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
# OCR parity fixtures
|
||||
|
||||
The recording command runs four stages:
|
||||
|
||||
1. Generate deterministic SDK inputs for every configured OCR target
|
||||
2. Build target-scoped, deduplicated recording jobs
|
||||
3. Record upstream responses through one globally bounded worker pool
|
||||
4. Persist each fixture and report whether it was recorded, cached, or failed
|
||||
|
||||
Run it with:
|
||||
|
||||
```shell
|
||||
uv run python -m tests.test_litellm.ocr.fixtures.record --examples 4 --concurrency 4
|
||||
```
|
||||
|
||||
`--concurrency` caps provider calls across all targets. Independent jobs finish after a failure, then the command exits
|
||||
nonzero if any job failed
|
||||
|
||||
OCR strategies generate public `litellm.ocr()` and `litellm.aocr()` inputs. Every case contains the normalized model,
|
||||
document, optional provider override, and LiteLLM keyword arguments. The fixture-only `contract` literal selects the
|
||||
input schema and is removed before calling the SDK. Strategies never build provider wire payloads
|
||||
|
||||
Each contract has a required corpus containing a baseline and cases for its supported top-level OCR parameters. The
|
||||
contracts are Mistral, Azure-hosted Mistral, Vertex-hosted Mistral, Azure Document Intelligence, Vertex DeepSeek,
|
||||
Reducto v3, and Reducto legacy. Credentials and endpoints only control target discovery, so a machine records the
|
||||
contracts it has configured and skips the rest
|
||||
|
||||
Reducto fixtures record upload and parse responses. Their parity cases remain non-strict expected failures until the
|
||||
Rust OCR bridge supports Reducto. Azure and Vertex generation paths are unit-tested without credentials in CI, so the
|
||||
committed corpus does not need live recordings for every target
|
||||
|
||||
Every recording target owns a small fixed provider-rejected corpus, independent of replay implementation support.
|
||||
Those inputs are recorded separately from generated valid inputs. Local validation failures use no recorded response;
|
||||
the parity suite checks those unsupported providers and models, malformed documents, invalid request formats, invalid
|
||||
Azure Document Intelligence parameters, and invalid headers in sync and async SDK calls
|
||||
|
|
@ -5,20 +5,16 @@ from typing import Final, Literal, cast
|
|||
|
||||
from hypothesis import strategies as st
|
||||
from hypothesis.strategies import SearchStrategy
|
||||
from pydantic import Field, StrictInt, StrictStr, field_validator
|
||||
from pydantic import StrictInt, StrictStr, field_validator
|
||||
|
||||
from tests.route_parity.fixtures.recording import ProviderSpec
|
||||
from tests.route_parity.fixtures.recording import UpstreamEndpoint
|
||||
from tests.test_litellm.ocr.fixtures.base import OcrDocument, OcrSdkInputBase
|
||||
from tests.test_litellm.ocr.fixtures.common import (
|
||||
OcrFixtureClient,
|
||||
OcrRecordingTarget,
|
||||
image_document,
|
||||
invoke_with_api_key,
|
||||
parameter_strategy,
|
||||
pdf_document,
|
||||
sampled_list_strategy,
|
||||
sampled_parameter_group_strategy,
|
||||
sampled_scalar_strategy,
|
||||
)
|
||||
from tests.test_litellm.ocr.fixtures.mistral import (
|
||||
MistralCompatibleOcrSdkInput,
|
||||
|
|
@ -26,11 +22,15 @@ from tests.test_litellm.ocr.fixtures.mistral import (
|
|||
)
|
||||
|
||||
AzureMistralModel = Literal["azure_ai/mistral-document-ai-2512",]
|
||||
AzureMistralFixtureModel = AzureMistralModel | Literal["azure_ai/invalid-ocr-model-for-parity"]
|
||||
AzureDocumentIntelligenceModel = Literal[
|
||||
"azure_ai/doc-intelligence/prebuilt-read",
|
||||
"azure_ai/doc-intelligence/prebuilt-layout",
|
||||
"azure_ai/doc-intelligence/prebuilt-document",
|
||||
]
|
||||
AzureDocumentIntelligenceFixtureModel = (
|
||||
AzureDocumentIntelligenceModel | Literal["azure_ai/doc-intelligence/invalid-ocr-model-for-parity"]
|
||||
)
|
||||
|
||||
AZURE_MISTRAL_MODELS: Final[tuple[AzureMistralModel, ...]] = ("azure_ai/mistral-document-ai-2512",)
|
||||
AZURE_DOCUMENT_INTELLIGENCE_MODELS: Final[tuple[AzureDocumentIntelligenceModel, ...]] = (
|
||||
|
|
@ -47,8 +47,8 @@ AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS: Final[tuple[AzureDocumentIntellige
|
|||
|
||||
|
||||
class AzureMistralOcrSdkInput(MistralCompatibleOcrSdkInput):
|
||||
boundary: str = Field(default="azure_mistral", pattern=r"^azure_mistral$")
|
||||
model: AzureMistralModel
|
||||
contract: Literal["azure_mistral"] = "azure_mistral"
|
||||
model: AzureMistralFixtureModel
|
||||
custom_llm_provider: Literal["azure_ai"] | None = None
|
||||
|
||||
@field_validator("model")
|
||||
|
|
@ -60,8 +60,8 @@ class AzureMistralOcrSdkInput(MistralCompatibleOcrSdkInput):
|
|||
|
||||
|
||||
class AzureDocumentIntelligenceOcrSdkInput(OcrSdkInputBase):
|
||||
boundary: str = Field(default="azure_document_intelligence", pattern=r"^azure_document_intelligence$")
|
||||
model: AzureDocumentIntelligenceModel
|
||||
contract: Literal["azure_document_intelligence"] = "azure_document_intelligence"
|
||||
model: AzureDocumentIntelligenceFixtureModel
|
||||
document: OcrDocument
|
||||
custom_llm_provider: Literal["azure_ai"] | None = None
|
||||
pages: str | list[StrictInt] | list[StrictStr] | None = None
|
||||
|
|
@ -69,6 +69,20 @@ class AzureDocumentIntelligenceOcrSdkInput(OcrSdkInputBase):
|
|||
req_format: Literal["litellm"] = "litellm"
|
||||
|
||||
|
||||
AZURE_MISTRAL_PROVIDER_REJECTED_INPUTS: Final[tuple[AzureMistralOcrSdkInput, ...]] = (
|
||||
AzureMistralOcrSdkInput(
|
||||
model="azure_ai/invalid-ocr-model-for-parity",
|
||||
document=pdf_document(),
|
||||
),
|
||||
)
|
||||
AZURE_DOCUMENT_INTELLIGENCE_PROVIDER_REJECTED_INPUTS: Final[tuple[AzureDocumentIntelligenceOcrSdkInput, ...]] = (
|
||||
AzureDocumentIntelligenceOcrSdkInput(
|
||||
model="azure_ai/doc-intelligence/invalid-ocr-model-for-parity",
|
||||
document=pdf_document(),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _azure_mistral_input(values: dict[str, object], model: AzureMistralModel) -> AzureMistralOcrSdkInput:
|
||||
return AzureMistralOcrSdkInput.model_validate({**values, "model": model})
|
||||
|
||||
|
|
@ -79,7 +93,7 @@ def azure_mistral_input_strategy(inline_image_data_uri: str) -> SearchStrategy[A
|
|||
return st.builds(
|
||||
_azure_mistral_input,
|
||||
values=mistral_input_values_strategy("2505", inline_image_data_uri, include_document_annotation_prompt=False),
|
||||
model=sampled_scalar_strategy(AZURE_MISTRAL_MODELS),
|
||||
model=st.sampled_from(AZURE_MISTRAL_MODELS),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -91,7 +105,7 @@ _AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL: Final[AzureDocumentIntelligenceMod
|
|||
def _document_intelligence_input(
|
||||
model: AzureDocumentIntelligenceModel,
|
||||
document: OcrDocument,
|
||||
optional_params: dict[str, object] | None = None,
|
||||
optional_params: Mapping[str, object] | None = None,
|
||||
) -> AzureDocumentIntelligenceOcrSdkInput:
|
||||
return AzureDocumentIntelligenceOcrSdkInput.model_validate(
|
||||
{"model": model, "document": document, **(optional_params or {})}
|
||||
|
|
@ -100,33 +114,25 @@ def _document_intelligence_input(
|
|||
|
||||
def azure_document_intelligence_input_strategy() -> SearchStrategy[AzureDocumentIntelligenceOcrSdkInput]:
|
||||
document: Final = pdf_document()
|
||||
pages: Final = parameter_strategy(
|
||||
"pages",
|
||||
st.one_of(
|
||||
sampled_list_strategy(((0,), (2, 0, 0, 1))),
|
||||
sampled_list_strategy((("1", "2-4"),)),
|
||||
sampled_scalar_strategy(("1-4, 5",)),
|
||||
),
|
||||
)
|
||||
features: Final = parameter_strategy(
|
||||
"features",
|
||||
st.one_of(
|
||||
sampled_list_strategy(
|
||||
(
|
||||
("languages",),
|
||||
("ocrHighResolution",),
|
||||
("barcodes",),
|
||||
("formulas",),
|
||||
("styleFont",),
|
||||
("keyValuePairs",),
|
||||
)
|
||||
),
|
||||
sampled_scalar_strategy(("languages, styleFont",)),
|
||||
),
|
||||
)
|
||||
combined_query: Final = sampled_parameter_group_strategy(
|
||||
((("pages", (0, 1)), ("features", ("languages", "styleFont"))),)
|
||||
)
|
||||
pages: Final = st.one_of(
|
||||
st.sampled_from(((0,), (2, 0, 0, 1))).map(list),
|
||||
st.just(["1", "2-4"]),
|
||||
st.just("1-4, 5"),
|
||||
).map(lambda value: {"pages": value})
|
||||
features: Final = st.one_of(
|
||||
st.sampled_from(
|
||||
(
|
||||
("languages",),
|
||||
("ocrHighResolution",),
|
||||
("barcodes",),
|
||||
("formulas",),
|
||||
("styleFont",),
|
||||
("keyValuePairs",),
|
||||
)
|
||||
).map(list),
|
||||
st.just("languages, styleFont"),
|
||||
).map(lambda value: {"features": value})
|
||||
combined_query: Final = st.just({"pages": (0, 1), "features": ("languages", "styleFont")})
|
||||
return st.one_of(
|
||||
st.sampled_from(AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS).map(
|
||||
lambda model: _document_intelligence_input(model, document)
|
||||
|
|
@ -166,18 +172,19 @@ def azure_mistral_recording_targets(
|
|||
environ: Mapping[str, str], client: OcrFixtureClient, inline_image_data_uri: str
|
||||
) -> tuple[OcrRecordingTarget, ...]:
|
||||
api_key: Final = environ.get("AZURE_AI_API_KEY")
|
||||
upstream_base: Final = environ.get("AZURE_AI_API_BASE")
|
||||
if not api_key or not upstream_base:
|
||||
base_url: Final = environ.get("AZURE_AI_API_BASE")
|
||||
if not api_key or not base_url:
|
||||
return ()
|
||||
return (
|
||||
OcrRecordingTarget(
|
||||
name="azure-mistral",
|
||||
provider_spec=ProviderSpec(upstream_base=upstream_base.rstrip("/")),
|
||||
upstream=UpstreamEndpoint(base_url=base_url.rstrip("/")),
|
||||
strategy=cast(
|
||||
SearchStrategy[OcrSdkInputBase],
|
||||
azure_mistral_input_strategy(inline_image_data_uri),
|
||||
),
|
||||
invocation=invoke_with_api_key(client, api_key),
|
||||
required_inputs=AZURE_MISTRAL_PROVIDER_REJECTED_INPUTS,
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -186,14 +193,15 @@ def azure_document_intelligence_recording_targets(
|
|||
environ: Mapping[str, str], client: OcrFixtureClient
|
||||
) -> tuple[OcrRecordingTarget, ...]:
|
||||
api_key: Final = environ.get("AZURE_DOCUMENT_INTELLIGENCE_API_KEY")
|
||||
upstream_base: Final = environ.get("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT")
|
||||
if not api_key or not upstream_base:
|
||||
base_url: Final = environ.get("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT")
|
||||
if not api_key or not base_url:
|
||||
return ()
|
||||
return (
|
||||
OcrRecordingTarget(
|
||||
name="azure-document-intelligence",
|
||||
provider_spec=ProviderSpec(upstream_base=upstream_base.rstrip("/")),
|
||||
upstream=UpstreamEndpoint(base_url=base_url.rstrip("/")),
|
||||
strategy=cast(SearchStrategy[OcrSdkInputBase], azure_document_intelligence_input_strategy()),
|
||||
invocation=invoke_with_api_key(client, api_key),
|
||||
required_inputs=AZURE_DOCUMENT_INTELLIGENCE_PROVIDER_REJECTED_INPUTS,
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -21,7 +21,9 @@ __all__ = (
|
|||
"OcrSdkInputBase",
|
||||
)
|
||||
|
||||
OcrSdkInputBase = SdkInputBase
|
||||
|
||||
class OcrSdkInputBase(SdkInputBase):
|
||||
fixture_only_fields = ("contract",)
|
||||
|
||||
|
||||
class ImageUrlValue(FixtureModel):
|
||||
|
|
|
|||
|
|
@ -2,13 +2,13 @@ from __future__ import annotations
|
|||
|
||||
from dataclasses import dataclass, field
|
||||
from functools import cache
|
||||
from typing import Final, Literal, Protocol, TypeVar
|
||||
from typing import Final, Literal, Protocol
|
||||
|
||||
from hypothesis import strategies as st
|
||||
from hypothesis.strategies import SearchStrategy
|
||||
|
||||
from tests.route_parity.fixtures.media import dummy_image_url, structured_pdf_data_uri
|
||||
from tests.route_parity.fixtures.pipeline import RecordingTarget
|
||||
from tests.route_parity.fixtures.media import dummy_image_url, structured_pdf_data_uri
|
||||
from tests.test_litellm.ocr.fixtures.base import (
|
||||
DocumentUrlDocument,
|
||||
ImageUrlDocument,
|
||||
|
|
@ -18,7 +18,6 @@ from tests.test_litellm.ocr.fixtures.base import (
|
|||
)
|
||||
|
||||
OcrRecordingTarget = RecordingTarget[OcrSdkInputBase]
|
||||
ValueT = TypeVar("ValueT")
|
||||
|
||||
|
||||
class OcrFixtureClient(Protocol):
|
||||
|
|
@ -81,27 +80,6 @@ def document_transport_strategy(inline_image_data_uri: str) -> SearchStrategy[Im
|
|||
return st.sampled_from(transports).map(as_document)
|
||||
|
||||
|
||||
def sampled_scalar_strategy(values: tuple[ValueT, ...]) -> SearchStrategy[ValueT]:
|
||||
return st.sampled_from(values)
|
||||
|
||||
|
||||
def sampled_list_strategy(values: tuple[tuple[ValueT, ...], ...]) -> SearchStrategy[list[ValueT]]:
|
||||
return st.sampled_from(values).map(list)
|
||||
|
||||
|
||||
def sampled_parameter_group_strategy(
|
||||
values: tuple[tuple[tuple[str, object], ...], ...],
|
||||
) -> SearchStrategy[dict[str, object]]:
|
||||
return st.sampled_from(values).map(dict)
|
||||
|
||||
|
||||
def parameter_strategy(name: str, values: SearchStrategy[ValueT]) -> SearchStrategy[dict[str, object]]:
|
||||
def as_parameter(value: ValueT) -> dict[str, object]:
|
||||
return {name: value}
|
||||
|
||||
return values.map(as_parameter)
|
||||
|
||||
|
||||
def annotation_format(name: str) -> JsonSchemaResponseFormat:
|
||||
return JsonSchemaResponseFormat(
|
||||
type="json_schema",
|
||||
|
|
|
|||
13
tests/test_litellm/ocr/fixtures/config.py
Normal file
13
tests/test_litellm/ocr/fixtures/config.py
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
FIXTURE_DIR_ENV: Final = "LITELLM_OCR_FIXTURE_DIR"
|
||||
DEFAULT_FIXTURE_DIRECTORY: Final = Path(__file__).with_name("data")
|
||||
|
||||
|
||||
def configured_fixture_directory() -> Path:
|
||||
configured: Final = os.environ.get(FIXTURE_DIR_ENV)
|
||||
return Path(configured).expanduser() if configured is not None else DEFAULT_FIXTURE_DIRECTORY
|
||||
|
|
@ -67,10 +67,6 @@
|
|||
"name": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=c5QAleICyoEptgycuGlRuItpEqcNe2MRZASOE9DeW80-1788292466.0205042-1.0.1.1-_PjnHuYd9Ba1XjxpUtrCWo_gUI5uKAvA73jcZAj6t_RARG9jiuX8N3t6bXu7K15RP3WG6AYFN4hbSKwlEpfksa9krmM7JYpf_JEpS5GDbst5vpFFLNtv.ymdX8lDSb5a; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:26 GMT"
|
||||
},
|
||||
{
|
||||
"name": "mistral-correlation-id",
|
||||
"value": "01a05e89-85bd-7075-af53-8e341b35a63e"
|
||||
|
|
|
|||
|
|
@ -64,10 +64,6 @@
|
|||
"name": "cf-cache-status",
|
||||
"value": "DYNAMIC"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=guezPmBNpI0djDo3Kvw0miZywJszLF5AN8loY9Uma0E-1788292459.497073-1.0.1.1-meRuKWGaBsHhkiA58SnRIUHMOaMgSPVzPrmAuu4_xq4UqYETqJn06.dJRx2xrVcoseTuUZNKdyRxTbkz2pdMRMe2AMQWaAMXXW27bG1VkapsA2hW1KtB48F3ubpGbp0v; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:20 GMT"
|
||||
},
|
||||
{
|
||||
"name": "Strict-Transport-Security",
|
||||
"value": "max-age=15552000; includeSubDomains; preload"
|
||||
|
|
|
|||
|
|
@ -20,10 +20,6 @@
|
|||
"name": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=XcXH8M77UDLCt6rebhVaPfmyZAa4b4OQ6qSuzN8zcxQ-1788292460.477192-1.0.1.1-jIqoIYlVPq1KWC06tEhpWZ2IGqr1TQU1jPokZk1wkK2UVb0L0LVboaLWA9UnCvYLvy5JzgUf1srFJQlzr7edR8HiWMaxsHfRsCVCPVlCr3H6S_Pikt3AqQmlW8xWM9vh; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:21 GMT"
|
||||
},
|
||||
{
|
||||
"name": "mistral-correlation-id",
|
||||
"value": "01a05e89-7015-7a8d-a18c-37b9ceacbe77"
|
||||
|
|
|
|||
|
|
@ -20,10 +20,6 @@
|
|||
"name": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=zdJ57Tl12b3GlmPJbN9IJjqlrvpX6J37Pmt6.Jj4vEs-1788292454.9590237-1.0.1.1-KS5i0m8mBhan9C3JxsJ488KsWHHpjYMz7ASc2SsaJ9MdFLx6JQWyViWZcw3ohhK1y_8a7DwExnKFQEIkFvF8J6HRVhoD2qaYjFW7DHz3t.2DBZunmBgvOouYMcjJyi7X; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:15 GMT"
|
||||
},
|
||||
{
|
||||
"name": "mistral-correlation-id",
|
||||
"value": "01a05e89-5a87-7148-8ae4-4e14967bebf3"
|
||||
|
|
|
|||
|
|
@ -51,10 +51,6 @@
|
|||
"name": "cf-cache-status",
|
||||
"value": "DYNAMIC"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=UKi98pAYwl6ufVXdgT8WLzMQP_BLupXglkclkyRQJyw-1788306848.2623396-1.0.1.1-oc0jGJ7AsfTQjZuaW08y5nxSJ9fHZS0mt2dij5uO_lA7CeUprSoHUJzIhyVKjI9A2oUM313oVsj6kWfBnqiVnLSUPAf_BMsUqLOkEJtUSMHrIvzkmRCZqBTUJgAqNlK4; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Wed, 02 Sep 2026 00:24:08 GMT"
|
||||
},
|
||||
{
|
||||
"name": "Strict-Transport-Security",
|
||||
"value": "max-age=15552000; includeSubDomains; preload"
|
||||
|
|
|
|||
|
|
@ -39,10 +39,6 @@
|
|||
"name": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=VjsfRKiwC7YjbVgfHbbpQfMBCW9MC1ehUXU3LsRZeUY-1788292455.9575524-1.0.1.1-KL3St9l4QfN1eWLUV07XDYmwE0xr2.26Z85yKf1yBMTDFCHBnz_3_RP.oNw6p.GfJsd2mnKb2lOvgPQcBDZygv7yJhN41wN87pIkAvurdr1z_snzwBpofXTvcjibOV1K; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:16 GMT"
|
||||
},
|
||||
{
|
||||
"name": "mistral-correlation-id",
|
||||
"value": "01a05e89-5e6e-70ac-890d-c0110a5e81af"
|
||||
|
|
|
|||
|
|
@ -19,10 +19,6 @@
|
|||
"name": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=0BJujMXEX2uflOAjZGqeTJi9lXNgGJmX61YbHtpbRKI-1788292452.9600976-1.0.1.1-7idLuOz.nd3kpO10lpC8IkPMArZ8M9iDs13Z.kTwKLTBHVm4.lyUeSqedg4EQV_Mo3D6JLf6TJgvdhWHsemZe6NyIzTBQwA3cwyM0cMQ5Aw68sVgtDOuGBaqhtelZ0RW; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:13 GMT"
|
||||
},
|
||||
{
|
||||
"name": "mistral-correlation-id",
|
||||
"value": "01a05e89-52b4-7e7b-983e-5bc4e5469b39"
|
||||
|
|
|
|||
|
|
@ -51,10 +51,6 @@
|
|||
"name": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=sPN0c81343UATX4OL40rLiXwZzjj6.f9wiR.sUEwidI-1788292464.0167594-1.0.1.1-1I1p0tYuzSZefNGWXeuwPPCH2oi83ZL7wHVVU_skvYhidQGDYesdwH32zscGrLGoxF1LPkg9h4yruaTw9Pai6ahKkyPofBTXU4pjatWxIYJOV8ILJQRwjJAY9r5x01g5; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:24 GMT"
|
||||
},
|
||||
{
|
||||
"name": "mistral-correlation-id",
|
||||
"value": "01a05e89-7deb-72b4-976d-4c244b056782"
|
||||
|
|
|
|||
|
|
@ -64,10 +64,6 @@
|
|||
"name": "cf-cache-status",
|
||||
"value": "DYNAMIC"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=Ld7W50uqSPv414iFBL3xHBo1US6FgEGi176rm3Ir7jU-1788292455.4420552-1.0.1.1-U1t1LZ3WH0stmljgl3EZ7Q.vhOhxNGrc3Zj75kgYaDNZCxuwaMc55Khvf.CeGempT.Y5ajCYMNsBwGv0h3rJ4PtpKYxp6Jt16enjPTx6jUkPdvUVHJwVTRMSTNSJxf5S; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:15 GMT"
|
||||
},
|
||||
{
|
||||
"name": "Strict-Transport-Security",
|
||||
"value": "max-age=15552000; includeSubDomains; preload"
|
||||
|
|
|
|||
|
|
@ -40,10 +40,6 @@
|
|||
"name": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=kVVa_K4blNT7UfkHNHEr2jJrVZoVVcKoCLeEqC9ojSY-1788292457.9748948-1.0.1.1-4aZN8zUH7fXP1NZK_aaS15fCbNd8A8ZJbBeCartOJWpgZMx2hW3klcPhDDvC0KSV6e.K1Ahz9wRggeTiUZQjsVuKYbR_vJ8dGfKXVNP2KyYcOfgaHyr0Kb1JOZSDGGnS; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:18 GMT"
|
||||
},
|
||||
{
|
||||
"name": "mistral-correlation-id",
|
||||
"value": "01a05e89-664a-740d-a112-0a1183c302b3"
|
||||
|
|
|
|||
|
|
@ -64,10 +64,6 @@
|
|||
"name": "cf-cache-status",
|
||||
"value": "DYNAMIC"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=vGgqF1m3.RSN4S6h3XP_Kk8ojWPq3vypiJ2NZgCazl0-1788292461.4966974-1.0.1.1-OYrG7cwlLUmuwb_TTkXh4Te8fKHOoi5hlP5Mok.vCVqZD0lY1bv2Efm83lRx69__NMYwiYqJwrzDNChyYI5Vl9HpHmBybXfpbZ6WpjVEIscEyaLBDMJ_0UI5whsB2.xf; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:22 GMT"
|
||||
},
|
||||
{
|
||||
"name": "Strict-Transport-Security",
|
||||
"value": "max-age=15552000; includeSubDomains; preload"
|
||||
|
|
|
|||
|
|
@ -22,10 +22,6 @@
|
|||
"name": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=CJBaWJ.SsBbSNRd1h3ENyFHO8QA2S5qq0irYhSrOSQY-1788292453.9335105-1.0.1.1-IIqKfxCHaT.6ATkFZNdqQ87W8B2Lnv_N5RSB8sU.dZa5E0gROwfWTZcIsI3aQm_r6aqQT3o.c86aG9ltQq25grYhzFc1LXxWUsmCyMEW7WunMgbzYacSazYXeAxE0yPi; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:14 GMT"
|
||||
},
|
||||
{
|
||||
"name": "mistral-correlation-id",
|
||||
"value": "01a05e89-5685-747f-b76d-dab242ea7512"
|
||||
|
|
|
|||
|
|
@ -20,10 +20,6 @@
|
|||
"name": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=RwMrJUmbvjo3BcW48Leja7wReidkx5xcJUtnE.9z924-1788292454.453593-1.0.1.1-oxMCFin79BJrtZjXcFewsGcQA0dzFQlC6NHkBN9AuMlUSYoT8EZty3nieW5ORoOnygaYSrpckLffOUJddtxb4TgxSNpkJCBFZOzs3QLLlGpkZrsJx8Rw9ype09j_OKZx; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:14 GMT"
|
||||
},
|
||||
{
|
||||
"name": "mistral-correlation-id",
|
||||
"value": "01a05e89-588e-7e32-a54b-f816bc6c1dc4"
|
||||
|
|
|
|||
|
|
@ -83,10 +83,6 @@
|
|||
"name": "cf-cache-status",
|
||||
"value": "DYNAMIC"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=xoadxKi40.IFnjiaqOmRs2WH6OMEVyT3As6eFw.vBvY-1788292456.4529853-1.0.1.1-jbao7uSuL.oLriQMYYO4SXltYHP3IPGIYg7yiRKk3Xqlwsc3xaQu0ES59EpMR3XVjJADyl9rHQfnKY3HvKRxXdGIA8Jerdfcu68wRUZNHcXd_epa4pcO5nnks7YPh93z; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:17 GMT"
|
||||
},
|
||||
{
|
||||
"name": "Strict-Transport-Security",
|
||||
"value": "max-age=15552000; includeSubDomains; preload"
|
||||
|
|
|
|||
|
|
@ -64,10 +64,6 @@
|
|||
"name": "cf-cache-status",
|
||||
"value": "DYNAMIC"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=O0x.ne9EKgb7nXAnOFoTqrbi69sVwHkdvll_fTS6PYE-1788292458.978935-1.0.1.1-2MYJEhW_I9Kf6XeYxBAqVLCXAnN8HABHbK351INmQpOQo036RmkEgf2..oRaM4BDqtR_F7Xrf72ekGCSm4C1h2tQaM6qXV0eCbt78fDRKI3i4N_cWLzghtZs0RJRw7Ir; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:19 GMT"
|
||||
},
|
||||
{
|
||||
"name": "Strict-Transport-Security",
|
||||
"value": "max-age=15552000; includeSubDomains; preload"
|
||||
|
|
|
|||
|
|
@ -64,10 +64,6 @@
|
|||
"name": "cf-cache-status",
|
||||
"value": "DYNAMIC"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=0rjHVJtpp3aXFzTWuyNNxtBvLU2V3RPEgQGcO8MQIJo-1788292463.4929135-1.0.1.1-39O3ne30GQFGDB22kGltMQVL6E9Hk2pTTGEyKXMcYXy8Dzs.BogwfvcphEP473aOQPn_RYs3f5Jz_fhcd4BzKbs1YHm0AQbi84qoKxg.2BQ7kpkarYT1GHgFIfOs_Pxw; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:23 GMT"
|
||||
},
|
||||
{
|
||||
"name": "Strict-Transport-Security",
|
||||
"value": "max-age=15552000; includeSubDomains; preload"
|
||||
|
|
|
|||
|
|
@ -83,10 +83,6 @@
|
|||
"name": "cf-cache-status",
|
||||
"value": "DYNAMIC"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=dDZPnK7j_xGIDo5mFF6PHBwL.E.iE3aIhXdIZ6xZIPg-1788292465.0206954-1.0.1.1-XVajgKnQ8WEsB75xPcp1hdJXhMUfK.oz.Apkd1nRlGofoxlEWK4F7M2bC8p67VFjDyEZK3O6mCzpiePraLUty31M1H92EBrZBT9wRwuaet2H7P73KO8QHZmO2Jm5tgyC; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:25 GMT"
|
||||
},
|
||||
{
|
||||
"name": "Strict-Transport-Security",
|
||||
"value": "max-age=15552000; includeSubDomains; preload"
|
||||
|
|
|
|||
|
|
@ -20,10 +20,6 @@
|
|||
"name": "Content-Type",
|
||||
"value": "application/json"
|
||||
},
|
||||
{
|
||||
"name": "set-cookie",
|
||||
"value": "__cf_bm=h45pW85qKj22bgXtkfz55PHEqryQZKiE7tImTTQx2u4-1788292462.5055265-1.0.1.1-kea4VbQ7Y.FRb9t5HgroFMzOCmbkU.byJfToALDpbgme.ScLO1wX13v5qEdll59C0DGKoXk2YYLAXvz0D6QfKN9mwTDRu23O72VJGgXtGu80xcMz16ZSRJfPn7hqnVFI; HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai; Expires=Tue, 01 Sep 2026 20:24:22 GMT"
|
||||
},
|
||||
{
|
||||
"name": "mistral-correlation-id",
|
||||
"value": "01a05e89-7802-768c-bf57-9021a209ab48"
|
||||
|
|
|
|||
|
|
@ -5,10 +5,10 @@ from typing import Final, Literal, cast
|
|||
|
||||
from hypothesis import strategies as st
|
||||
from hypothesis.strategies import SearchStrategy
|
||||
from pydantic import Field, model_validator
|
||||
from pydantic import model_validator
|
||||
from typing_extensions import Self
|
||||
|
||||
from tests.route_parity.fixtures.recording import ProviderSpec
|
||||
from tests.route_parity.fixtures.recording import UpstreamEndpoint
|
||||
from tests.test_litellm.ocr.fixtures.base import (
|
||||
JsonSchemaResponseFormat,
|
||||
OcrDocument,
|
||||
|
|
@ -20,11 +20,7 @@ from tests.test_litellm.ocr.fixtures.common import (
|
|||
annotation_format,
|
||||
document_transport_strategy,
|
||||
invoke_with_api_key,
|
||||
parameter_strategy,
|
||||
pdf_document,
|
||||
sampled_list_strategy,
|
||||
sampled_parameter_group_strategy,
|
||||
sampled_scalar_strategy,
|
||||
)
|
||||
|
||||
MistralModel = Literal[
|
||||
|
|
@ -43,6 +39,7 @@ MistralModel = Literal[
|
|||
"mistral-ocr-4",
|
||||
"mistral-ocr-latest",
|
||||
]
|
||||
MistralFixtureModel = MistralModel | Literal["mistral/invalid-ocr-model-for-parity"]
|
||||
|
||||
MISTRAL_MODELS: Final[tuple[MistralModel, ...]] = (
|
||||
"mistral/mistral-ocr-3",
|
||||
|
|
@ -79,8 +76,8 @@ class MistralCompatibleOcrSdkInput(OcrSdkInputBase):
|
|||
|
||||
|
||||
class MistralOcrSdkInput(MistralCompatibleOcrSdkInput):
|
||||
boundary: str = Field(default="mistral", pattern=r"^mistral$")
|
||||
model: MistralModel
|
||||
contract: Literal["mistral"] = "mistral"
|
||||
model: MistralFixtureModel
|
||||
custom_llm_provider: Literal["mistral"] | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
|
|
@ -90,15 +87,9 @@ class MistralOcrSdkInput(MistralCompatibleOcrSdkInput):
|
|||
return self
|
||||
|
||||
|
||||
class MistralProviderRejectedOcrSdkInput(MistralCompatibleOcrSdkInput):
|
||||
boundary: str = Field(default="mistral", pattern=r"^mistral$")
|
||||
model: Literal["mistral/invalid-ocr-model-for-parity"]
|
||||
custom_llm_provider: Literal["mistral"] | None = None
|
||||
|
||||
|
||||
MISTRAL_MODEL: Final[MistralModel] = "mistral/mistral-ocr-latest"
|
||||
MISTRAL_PROVIDER_REJECTED_INPUTS: Final[tuple[MistralProviderRejectedOcrSdkInput, ...]] = (
|
||||
MistralProviderRejectedOcrSdkInput(
|
||||
MISTRAL_PROVIDER_REJECTED_INPUTS: Final[tuple[MistralOcrSdkInput, ...]] = (
|
||||
MistralOcrSdkInput(
|
||||
model="mistral/invalid-ocr-model-for-parity",
|
||||
document=pdf_document(),
|
||||
),
|
||||
|
|
@ -135,40 +126,35 @@ def _optional_param_strategies(
|
|||
]:
|
||||
annotation: Final = annotation_format("document_title")
|
||||
common: Final[tuple[SearchStrategy[dict[str, object]], ...]] = (
|
||||
parameter_strategy("pages", sampled_list_strategy(((0,), (0, 1)))),
|
||||
parameter_strategy("include_image_base64", sampled_scalar_strategy((False, True))),
|
||||
parameter_strategy("image_limit", sampled_scalar_strategy((1,))),
|
||||
parameter_strategy("image_min_size", sampled_scalar_strategy((300,))),
|
||||
parameter_strategy(
|
||||
"bbox_annotation_format",
|
||||
sampled_scalar_strategy((annotation_format("bounding_boxes"),)),
|
||||
),
|
||||
parameter_strategy("document_annotation_format", sampled_scalar_strategy((annotation,))),
|
||||
st.sampled_from(((0,), (0, 1))).map(list).map(lambda value: {"pages": value}),
|
||||
st.sampled_from((False, True)).map(lambda value: {"include_image_base64": value}),
|
||||
st.just({"image_limit": 1}),
|
||||
st.just({"image_min_size": 300}),
|
||||
st.just({"bbox_annotation_format": annotation_format("bounding_boxes")}),
|
||||
st.just({"document_annotation_format": annotation}),
|
||||
*(
|
||||
(
|
||||
sampled_parameter_group_strategy(
|
||||
(
|
||||
(
|
||||
("document_annotation_format", annotation),
|
||||
("document_annotation_prompt", "Extract the visible title"),
|
||||
),
|
||||
)
|
||||
st.just(
|
||||
{
|
||||
"document_annotation_format": annotation,
|
||||
"document_annotation_prompt": "Extract the visible title",
|
||||
}
|
||||
),
|
||||
)
|
||||
if include_document_annotation_prompt
|
||||
else ()
|
||||
),
|
||||
parameter_strategy("confidence_scores_granularity", sampled_scalar_strategy(("page", "word"))),
|
||||
st.sampled_from(("page", "word")).map(lambda value: {"confidence_scores_granularity": value}),
|
||||
)
|
||||
feature_2512: Final[tuple[SearchStrategy[dict[str, object]], ...]] = (
|
||||
parameter_strategy("extract_header", sampled_scalar_strategy((False, True))),
|
||||
parameter_strategy("extract_footer", sampled_scalar_strategy((False, True))),
|
||||
parameter_strategy("table_format", sampled_scalar_strategy(("markdown", "html"))),
|
||||
st.sampled_from((False, True)).map(lambda value: {"extract_header": value}),
|
||||
st.sampled_from((False, True)).map(lambda value: {"extract_footer": value}),
|
||||
st.sampled_from(("markdown", "html")).map(lambda value: {"table_format": value}),
|
||||
)
|
||||
feature_4: Final[tuple[SearchStrategy[dict[str, object]], ...]] = (
|
||||
parameter_strategy("pages", sampled_scalar_strategy(("0-2",))),
|
||||
parameter_strategy("include_blocks", sampled_scalar_strategy((False, True))),
|
||||
sampled_parameter_group_strategy(((("include_blocks", True), ("confidence_scores_granularity", "block")),)),
|
||||
st.just({"pages": "0-2"}),
|
||||
st.sampled_from((False, True)).map(lambda value: {"include_blocks": value}),
|
||||
st.just({"include_blocks": True, "confidence_scores_granularity": "block"}),
|
||||
)
|
||||
return common, feature_2512, feature_4
|
||||
|
||||
|
|
@ -237,7 +223,7 @@ def _mistral_recording_strategy(inline_image_data_uri: str) -> SearchStrategy[Mi
|
|||
feature_2512_options: Final[SearchStrategy[dict[str, object]]] = st.one_of(*feature_2512)
|
||||
feature_4_options: Final[SearchStrategy[dict[str, object]]] = st.one_of(*feature_4)
|
||||
return st.one_of(
|
||||
sampled_scalar_strategy(baseline_models).map(lambda model: _mistral_input(model, document)),
|
||||
st.sampled_from(baseline_models).map(lambda model: _mistral_input(model, document)),
|
||||
document_transport_strategy(inline_image_data_uri).map(
|
||||
lambda selected_document: _mistral_input(MISTRAL_MODEL, selected_document)
|
||||
),
|
||||
|
|
@ -258,11 +244,11 @@ def mistral_recording_targets(
|
|||
if not api_key:
|
||||
return ()
|
||||
configured: Final = environ.get("MISTRAL_API_BASE", "https://api.mistral.ai").rstrip("/")
|
||||
upstream_base: Final = configured.removesuffix("/v1")
|
||||
base_url: Final = configured.removesuffix("/v1")
|
||||
return (
|
||||
OcrRecordingTarget(
|
||||
name="mistral-ocr",
|
||||
provider_spec=ProviderSpec(upstream_base=upstream_base),
|
||||
upstream=UpstreamEndpoint(base_url=base_url),
|
||||
strategy=cast(
|
||||
SearchStrategy[OcrSdkInputBase],
|
||||
_mistral_recording_strategy(inline_image_data_uri),
|
||||
|
|
|
|||
|
|
@ -3,48 +3,69 @@ from __future__ import annotations
|
|||
from collections.abc import Mapping
|
||||
from typing import Annotated, Final, cast
|
||||
|
||||
from pydantic import Discriminator, Tag
|
||||
from pydantic import Field, model_validator
|
||||
|
||||
from tests.route_parity.fixture_models import ParityCase
|
||||
from tests.test_litellm.ocr.fixtures.azure import (
|
||||
AzureDocumentIntelligenceOcrSdkInput,
|
||||
AzureMistralOcrSdkInput,
|
||||
)
|
||||
from tests.test_litellm.ocr.fixtures.base import OcrSdkInputBase
|
||||
from tests.test_litellm.ocr.fixtures.mistral import MistralOcrSdkInput, MistralProviderRejectedOcrSdkInput
|
||||
from tests.test_litellm.ocr.fixtures.mistral import MistralOcrSdkInput
|
||||
from tests.test_litellm.ocr.fixtures.reducto import ReductoParseLegacySdkInput, ReductoParseV3SdkInput
|
||||
from tests.test_litellm.ocr.fixtures.vertex import VertexDeepSeekOcrSdkInput, VertexMistralOcrSdkInput
|
||||
|
||||
__all__ = ("OcrParityCase", "OcrSdkInput")
|
||||
|
||||
|
||||
def _ocr_boundary(value: object) -> str | None:
|
||||
if isinstance(value, Mapping):
|
||||
mapping: Final = cast(Mapping[object, object], value)
|
||||
boundary: Final = mapping.get("boundary")
|
||||
model: Final = mapping.get("model")
|
||||
if boundary == "mistral" and model == "mistral/invalid-ocr-model-for-parity":
|
||||
return "mistral_provider_rejected"
|
||||
return boundary if isinstance(boundary, str) else None
|
||||
if isinstance(value, OcrSdkInputBase):
|
||||
if isinstance(value, MistralProviderRejectedOcrSdkInput):
|
||||
return "mistral_provider_rejected"
|
||||
return value.boundary
|
||||
return None
|
||||
|
||||
|
||||
OcrSdkInput = Annotated[
|
||||
Annotated[MistralOcrSdkInput, Tag("mistral")]
|
||||
| Annotated[MistralProviderRejectedOcrSdkInput, Tag("mistral_provider_rejected")]
|
||||
| Annotated[AzureMistralOcrSdkInput, Tag("azure_mistral")]
|
||||
| Annotated[VertexMistralOcrSdkInput, Tag("vertex_mistral")]
|
||||
| Annotated[AzureDocumentIntelligenceOcrSdkInput, Tag("azure_document_intelligence")]
|
||||
| Annotated[VertexDeepSeekOcrSdkInput, Tag("vertex_deepseek")]
|
||||
| Annotated[ReductoParseV3SdkInput, Tag("reducto_v3")]
|
||||
| Annotated[ReductoParseLegacySdkInput, Tag("reducto_legacy")],
|
||||
Discriminator(_ocr_boundary),
|
||||
MistralOcrSdkInput
|
||||
| AzureMistralOcrSdkInput
|
||||
| VertexMistralOcrSdkInput
|
||||
| AzureDocumentIntelligenceOcrSdkInput
|
||||
| VertexDeepSeekOcrSdkInput
|
||||
| ReductoParseV3SdkInput
|
||||
| ReductoParseLegacySdkInput,
|
||||
Field(discriminator="contract"),
|
||||
]
|
||||
|
||||
|
||||
class OcrParityCase(ParityCase[OcrSdkInput]):
|
||||
pass
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def load_legacy_contract(cls, value: object) -> object:
|
||||
if not isinstance(value, Mapping):
|
||||
return value
|
||||
fixture: Final = cast(Mapping[str, object], value)
|
||||
litellm_input: Final = fixture.get("litellm_input")
|
||||
if not isinstance(litellm_input, Mapping) or "contract" in litellm_input:
|
||||
return fixture
|
||||
legacy_input: Final = cast(Mapping[str, object], litellm_input)
|
||||
legacy_contract: Final = legacy_input.get("boundary")
|
||||
if isinstance(legacy_contract, str):
|
||||
return {
|
||||
**fixture,
|
||||
"litellm_input": {
|
||||
"contract": legacy_contract,
|
||||
**{key: item for key, item in legacy_input.items() if key != "boundary"},
|
||||
},
|
||||
}
|
||||
model: Final = legacy_input.get("model")
|
||||
if not isinstance(model, str):
|
||||
return fixture
|
||||
return {**fixture, "litellm_input": {"contract": _legacy_contract(model), **legacy_input}}
|
||||
|
||||
|
||||
def _legacy_contract(model: str) -> str:
|
||||
if model.startswith("azure_ai/doc-intelligence/"):
|
||||
return "azure_document_intelligence"
|
||||
if model.startswith("azure_ai/"):
|
||||
return "azure_mistral"
|
||||
if model.startswith("vertex_ai/deepseek"):
|
||||
return "vertex_deepseek"
|
||||
if model.startswith("vertex_ai/"):
|
||||
return "vertex_mistral"
|
||||
if model.endswith("parse-v3"):
|
||||
return "reducto_v3"
|
||||
if model.endswith("parse-legacy"):
|
||||
return "reducto_legacy"
|
||||
return "mistral"
|
||||
|
|
|
|||
|
|
@ -3,15 +3,15 @@ from __future__ import annotations
|
|||
import logging
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from typing import Final, cast
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
import litellm
|
||||
from litellm.rust_bridge.ocr import use_litellm_rust
|
||||
from tests.route_parity.fixtures.cli import parse_recording_args
|
||||
from tests.route_parity.fixtures.media import structured_image_data_uri
|
||||
from tests.route_parity.fixtures.pipeline import parse_recording_args, record_fixtures
|
||||
from tests.route_parity.fixtures.pipeline import record_fixtures
|
||||
from tests.route_parity.fixtures.store import fixture_directory
|
||||
from tests.test_litellm.ocr.fixtures.azure import (
|
||||
azure_document_intelligence_recording_targets,
|
||||
|
|
@ -19,13 +19,12 @@ from tests.test_litellm.ocr.fixtures.azure import (
|
|||
)
|
||||
from tests.test_litellm.ocr.fixtures.base import OcrSdkInputBase
|
||||
from tests.test_litellm.ocr.fixtures.common import OcrFixtureClient, OcrRecordingTarget, OcrSdkCall
|
||||
from tests.test_litellm.ocr.fixtures.config import DEFAULT_FIXTURE_DIRECTORY, FIXTURE_DIR_ENV
|
||||
from tests.test_litellm.ocr.fixtures.mistral import mistral_recording_targets
|
||||
from tests.test_litellm.ocr.fixtures.models import OcrParityCase
|
||||
from tests.test_litellm.ocr.fixtures.reducto import reducto_recording_targets
|
||||
from tests.test_litellm.ocr.fixtures.vertex import vertex_recording_targets
|
||||
|
||||
FIXTURE_DIR_ENV: Final = "LITELLM_OCR_FIXTURE_DIR"
|
||||
|
||||
|
||||
class LiteLLMOcrFixtureClient:
|
||||
def __init__(self, sdk_call: OcrSdkCall) -> None:
|
||||
|
|
@ -65,7 +64,7 @@ def main() -> int:
|
|||
root: Final = fixture_directory(
|
||||
args.fixture_dir,
|
||||
os.environ.get(FIXTURE_DIR_ENV),
|
||||
Path(__file__).with_name("data"),
|
||||
DEFAULT_FIXTURE_DIRECTORY,
|
||||
)
|
||||
use_litellm_rust(False, ocr=None, aocr=None)
|
||||
summary: Final = record_fixtures(targets, root, args.examples, args.concurrency, OcrParityCase)
|
||||
|
|
|
|||
|
|
@ -12,16 +12,13 @@ from typing_extensions import Self
|
|||
|
||||
from tests.route_parity.fixture_models import FixtureModel, JsonObject
|
||||
from tests.route_parity.fixtures.media import structured_pdf_data_uri
|
||||
from tests.route_parity.fixtures.recording import ProviderSpec
|
||||
from tests.route_parity.fixtures.recording import UpstreamEndpoint
|
||||
from tests.test_litellm.ocr.fixtures.base import OcrSdkInputBase
|
||||
from tests.test_litellm.ocr.fixtures.common import (
|
||||
OcrFixtureClient,
|
||||
OcrRecordingTarget,
|
||||
image_data_document,
|
||||
invoke_with_api_key,
|
||||
parameter_strategy,
|
||||
sampled_list_strategy,
|
||||
sampled_scalar_strategy,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -204,7 +201,7 @@ class ReductoSettings(FixtureModel):
|
|||
|
||||
|
||||
class ReductoParseV3SdkInput(OcrSdkInputBase):
|
||||
boundary: str = Field(default="reducto_v3", pattern=r"^reducto_v3$")
|
||||
contract: Literal["reducto_v3"] = "reducto_v3"
|
||||
model: ReductoV3Model
|
||||
document: ReductoDocument
|
||||
custom_llm_provider: Literal["reducto"] | None = None
|
||||
|
|
@ -220,7 +217,7 @@ class ReductoParseV3SdkInput(OcrSdkInputBase):
|
|||
|
||||
|
||||
class ReductoParseLegacySdkInput(OcrSdkInputBase):
|
||||
boundary: str = Field(default="reducto_legacy", pattern=r"^reducto_legacy$")
|
||||
contract: Literal["reducto_legacy"] = "reducto_legacy"
|
||||
model: ReductoLegacyModel
|
||||
document: ReductoDocument
|
||||
custom_llm_provider: Literal["reducto"] | None = None
|
||||
|
|
@ -233,28 +230,44 @@ class ReductoParseLegacySdkInput(OcrSdkInputBase):
|
|||
return self
|
||||
|
||||
|
||||
_REDUCTO_PROVIDER_REJECTED_DOCUMENT: Final = ReductoDocumentUrlDocument(
|
||||
type="document_url",
|
||||
document_url="reducto://invalid-document-for-parity",
|
||||
)
|
||||
REDUCTO_V3_PROVIDER_REJECTED_INPUTS: Final[tuple[ReductoParseV3SdkInput, ...]] = (
|
||||
ReductoParseV3SdkInput(
|
||||
model="reducto/parse-v3",
|
||||
document=_REDUCTO_PROVIDER_REJECTED_DOCUMENT,
|
||||
),
|
||||
)
|
||||
REDUCTO_LEGACY_PROVIDER_REJECTED_INPUTS: Final[tuple[ReductoParseLegacySdkInput, ...]] = (
|
||||
ReductoParseLegacySdkInput(
|
||||
model="reducto/parse-legacy",
|
||||
document=_REDUCTO_PROVIDER_REJECTED_DOCUMENT,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
_REDUCTO_API_BASE: Final = "https://platform.reducto.ai"
|
||||
|
||||
|
||||
def _formatting_strategy() -> SearchStrategy[ReductoFormatting]:
|
||||
values: Final = st.one_of(
|
||||
parameter_strategy(
|
||||
"table_output_format",
|
||||
sampled_scalar_strategy(("dynamic", "html", "md", "json", "csv", "jsonbbox")),
|
||||
),
|
||||
parameter_strategy("add_page_markers", sampled_scalar_strategy((False, True))),
|
||||
parameter_strategy("merge_tables", sampled_scalar_strategy((False, True))),
|
||||
parameter_strategy(
|
||||
"include",
|
||||
sampled_list_strategy(
|
||||
(
|
||||
(),
|
||||
("hyperlinks",),
|
||||
("change_tracking", "highlight", "comments"),
|
||||
("signatures", "ignore_watermarks"),
|
||||
)
|
||||
),
|
||||
st.sampled_from(("dynamic", "html", "md", "json", "csv", "jsonbbox")).map(
|
||||
lambda value: {"table_output_format": value}
|
||||
),
|
||||
st.sampled_from((False, True)).map(lambda value: {"add_page_markers": value}),
|
||||
st.sampled_from((False, True)).map(lambda value: {"merge_tables": value}),
|
||||
st.sampled_from(
|
||||
(
|
||||
(),
|
||||
("hyperlinks",),
|
||||
("change_tracking", "highlight", "comments"),
|
||||
("signatures", "ignore_watermarks"),
|
||||
)
|
||||
)
|
||||
.map(list)
|
||||
.map(lambda value: {"include": value}),
|
||||
)
|
||||
return values.map(ReductoFormatting.model_validate)
|
||||
|
||||
|
|
@ -265,21 +278,22 @@ def _chunking_strategy() -> SearchStrategy[ReductoChunking]:
|
|||
lambda mode: ReductoChunking(chunk_mode=mode)
|
||||
),
|
||||
st.just(ReductoChunking(chunk_mode="variable")),
|
||||
sampled_scalar_strategy((250, 1000, 1500)).map(
|
||||
lambda size: ReductoChunking(chunk_mode="variable", chunk_size=size)
|
||||
),
|
||||
sampled_scalar_strategy((32, 128)).map(
|
||||
st.sampled_from((250, 1000, 1500)).map(lambda size: ReductoChunking(chunk_mode="variable", chunk_size=size)),
|
||||
st.sampled_from((32, 128)).map(
|
||||
lambda overlap: ReductoChunking(chunk_mode="variable", chunk_size=1000, chunk_overlap=overlap)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _retrieval_strategy() -> SearchStrategy[ReductoRetrieval]:
|
||||
filter_blocks: Final[SearchStrategy[list[ReductoBlockType]]] = sampled_list_strategy(_REDUCTO_FILTER_BLOCK_GROUPS)
|
||||
filter_blocks: Final = cast(
|
||||
SearchStrategy[list[ReductoBlockType]],
|
||||
st.sampled_from(_REDUCTO_FILTER_BLOCK_GROUPS).map(list),
|
||||
)
|
||||
return st.one_of(
|
||||
_chunking_strategy().map(lambda chunking: ReductoRetrieval(chunking=chunking)),
|
||||
filter_blocks.map(lambda selected_blocks: ReductoRetrieval(filter_blocks=selected_blocks)),
|
||||
sampled_scalar_strategy((False, True)).map(
|
||||
st.sampled_from((False, True)).map(
|
||||
lambda optimized: ReductoRetrieval(
|
||||
chunking=ReductoChunking(chunk_mode="variable"),
|
||||
embedding_optimized=optimized,
|
||||
|
|
@ -291,18 +305,20 @@ def _retrieval_strategy() -> SearchStrategy[ReductoRetrieval]:
|
|||
def _settings_strategy() -> SearchStrategy[ReductoSettings]:
|
||||
# force_url_result stays model-compatible but is not recorded until the
|
||||
# response transform follows and downloads result.url.
|
||||
return_images: Final[SearchStrategy[list[ReductoReturnImage]]] = sampled_list_strategy(_REDUCTO_RETURN_IMAGE_GROUPS)
|
||||
return_images: Final[SearchStrategy[list[ReductoReturnImage]]] = st.sampled_from(_REDUCTO_RETURN_IMAGE_GROUPS).map(
|
||||
list
|
||||
)
|
||||
page_ranges: Final = st.one_of(
|
||||
st.just(ReductoPageRange(start=1, end=1)),
|
||||
st.just(ReductoPageRange(start=1, end=3)),
|
||||
sampled_list_strategy(
|
||||
st.sampled_from(
|
||||
(
|
||||
(
|
||||
ReductoPageRange(start=1, end=2),
|
||||
ReductoPageRange(start=4, end=5),
|
||||
),
|
||||
)
|
||||
),
|
||||
).map(list),
|
||||
)
|
||||
return st.one_of(
|
||||
st.just(ReductoSettings(model="r-1")),
|
||||
|
|
@ -311,10 +327,10 @@ def _settings_strategy() -> SearchStrategy[ReductoSettings]:
|
|||
st.just(ReductoSettings(return_ocr_data=True)),
|
||||
return_images.map(lambda selected_images: ReductoSettings(return_images=selected_images)),
|
||||
st.just(ReductoSettings(embed_pdf_metadata=True)),
|
||||
sampled_scalar_strategy((50, 100, 250)).map(
|
||||
st.sampled_from((50, 100, 250)).map(
|
||||
lambda dpi: ReductoSettings(embed_pdf_metadata=True, embed_pdf_metadata_dpi=dpi)
|
||||
),
|
||||
sampled_scalar_strategy((300.0,)).map(lambda timeout: ReductoSettings(timeout=timeout)),
|
||||
st.just(ReductoSettings(timeout=300.0)),
|
||||
page_ranges.map(lambda page_range: ReductoSettings(page_range=page_range)),
|
||||
)
|
||||
|
||||
|
|
@ -403,20 +419,22 @@ def reducto_recording_targets(
|
|||
api_key: Final = environ.get("REDUCTO_API_KEY")
|
||||
if not api_key:
|
||||
return ()
|
||||
upstream_base: Final = environ.get("REDUCTO_API_BASE", _REDUCTO_API_BASE).rstrip("/")
|
||||
base_url: Final = environ.get("REDUCTO_API_BASE", _REDUCTO_API_BASE).rstrip("/")
|
||||
document: Final = ReductoDocumentUrlDocument(type="document_url", document_url=structured_pdf_data_uri())
|
||||
invocation: Final = invoke_with_api_key(client, api_key)
|
||||
return (
|
||||
OcrRecordingTarget(
|
||||
name="reducto-v3",
|
||||
provider_spec=ProviderSpec(upstream_base=upstream_base),
|
||||
upstream=UpstreamEndpoint(base_url=base_url),
|
||||
strategy=cast(SearchStrategy[OcrSdkInputBase], reducto_v3_input_strategy(inline_image_data_uri, document)),
|
||||
invocation=invocation,
|
||||
required_inputs=REDUCTO_V3_PROVIDER_REJECTED_INPUTS,
|
||||
),
|
||||
OcrRecordingTarget(
|
||||
name="reducto-legacy",
|
||||
provider_spec=ProviderSpec(upstream_base=upstream_base),
|
||||
upstream=UpstreamEndpoint(base_url=base_url),
|
||||
strategy=cast(SearchStrategy[OcrSdkInputBase], reducto_legacy_input_strategy(document)),
|
||||
invocation=invocation,
|
||||
required_inputs=REDUCTO_LEGACY_PROVIDER_REJECTED_INPUTS,
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -5,16 +5,14 @@ from typing import Final, Literal, cast
|
|||
|
||||
from hypothesis import strategies as st
|
||||
from hypothesis.strategies import DrawFn, SearchStrategy
|
||||
from pydantic import Field
|
||||
|
||||
from tests.route_parity.fixtures.recording import ProviderSpec
|
||||
from tests.route_parity.fixtures.recording import UpstreamEndpoint
|
||||
from tests.test_litellm.ocr.fixtures.base import OcrDocument, OcrSdkInputBase
|
||||
from tests.test_litellm.ocr.fixtures.common import (
|
||||
OcrFixtureClient,
|
||||
OcrRecordingTarget,
|
||||
image_data_document,
|
||||
invoke_with_api_key,
|
||||
sampled_scalar_strategy,
|
||||
)
|
||||
from tests.test_litellm.ocr.fixtures.mistral import (
|
||||
MistralCompatibleOcrSdkInput,
|
||||
|
|
@ -23,28 +21,60 @@ from tests.test_litellm.ocr.fixtures.mistral import (
|
|||
|
||||
VertexMistralModel = Literal["vertex_ai/mistral-ocr-2505"]
|
||||
VertexDeepSeekModel = Literal["vertex_ai/deepseek-ai/deepseek-ocr-maas"]
|
||||
VertexMistralFixtureModel = VertexMistralModel | Literal["vertex_ai/invalid-ocr-model-for-parity"]
|
||||
VertexDeepSeekFixtureModel = VertexDeepSeekModel | Literal["vertex_ai/deepseek-ai/invalid-ocr-model-for-parity"]
|
||||
|
||||
VERTEX_MISTRAL_MODELS: Final[tuple[VertexMistralModel, ...]] = ("vertex_ai/mistral-ocr-2505",)
|
||||
VERTEX_DEEPSEEK_MODELS: Final[tuple[VertexDeepSeekModel, ...]] = ("vertex_ai/deepseek-ai/deepseek-ocr-maas",)
|
||||
|
||||
|
||||
class VertexMistralOcrSdkInput(MistralCompatibleOcrSdkInput):
|
||||
boundary: str = Field(default="vertex_mistral", pattern=r"^vertex_mistral$")
|
||||
model: VertexMistralModel = "vertex_ai/mistral-ocr-2505"
|
||||
contract: Literal["vertex_mistral"] = "vertex_mistral"
|
||||
model: VertexMistralFixtureModel = "vertex_ai/mistral-ocr-2505"
|
||||
custom_llm_provider: Literal["vertex_ai"] | None = None
|
||||
vertex_project: str
|
||||
vertex_location: str = "us-central1"
|
||||
|
||||
|
||||
class VertexDeepSeekOcrSdkInput(OcrSdkInputBase):
|
||||
boundary: str = Field(default="vertex_deepseek", pattern=r"^vertex_deepseek$")
|
||||
model: VertexDeepSeekModel = "vertex_ai/deepseek-ai/deepseek-ocr-maas"
|
||||
contract: Literal["vertex_deepseek"] = "vertex_deepseek"
|
||||
model: VertexDeepSeekFixtureModel = "vertex_ai/deepseek-ai/deepseek-ocr-maas"
|
||||
document: OcrDocument
|
||||
custom_llm_provider: Literal["vertex_ai"] | None = None
|
||||
vertex_project: str
|
||||
vertex_location: str = "us-central1"
|
||||
|
||||
|
||||
def vertex_mistral_provider_rejected_inputs(
|
||||
project: str,
|
||||
location: str,
|
||||
inline_image_data_uri: str,
|
||||
) -> tuple[VertexMistralOcrSdkInput, ...]:
|
||||
return (
|
||||
VertexMistralOcrSdkInput(
|
||||
model="vertex_ai/invalid-ocr-model-for-parity",
|
||||
document=image_data_document(inline_image_data_uri),
|
||||
vertex_project=project,
|
||||
vertex_location=location,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def vertex_deepseek_provider_rejected_inputs(
|
||||
project: str,
|
||||
location: str,
|
||||
inline_image_data_uri: str,
|
||||
) -> tuple[VertexDeepSeekOcrSdkInput, ...]:
|
||||
return (
|
||||
VertexDeepSeekOcrSdkInput(
|
||||
model="vertex_ai/deepseek-ai/invalid-ocr-model-for-parity",
|
||||
document=image_data_document(inline_image_data_uri),
|
||||
vertex_project=project,
|
||||
vertex_location=location,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _as_vertex_mistral(
|
||||
values: dict[str, object],
|
||||
project: str,
|
||||
|
|
@ -65,7 +95,7 @@ def vertex_mistral_input_strategy(
|
|||
_as_vertex_mistral,
|
||||
project=st.just(project),
|
||||
location=st.just(location),
|
||||
model=sampled_scalar_strategy(VERTEX_MISTRAL_MODELS),
|
||||
model=st.sampled_from(VERTEX_MISTRAL_MODELS),
|
||||
values=mistral_input_values_strategy("2505", inline_image_data_uri),
|
||||
)
|
||||
|
||||
|
|
@ -76,10 +106,7 @@ def vertex_deepseek_input_strategy(
|
|||
) -> VertexDeepSeekOcrSdkInput:
|
||||
return VertexDeepSeekOcrSdkInput.model_validate(
|
||||
{
|
||||
"model": draw(sampled_scalar_strategy(VERTEX_DEEPSEEK_MODELS)),
|
||||
# The current Vertex model card documents image input only. Keep
|
||||
# the broader fixture model for existing recordings, but do not
|
||||
# spend a paid request on the transform's unsupported PDF branch.
|
||||
"model": draw(st.sampled_from(VERTEX_DEEPSEEK_MODELS)),
|
||||
"document": image_data_document(inline_image_data_uri),
|
||||
"vertex_project": project,
|
||||
"vertex_location": location,
|
||||
|
|
@ -95,25 +122,27 @@ def vertex_recording_targets(
|
|||
location: Final = environ.get("VERTEXAI_LOCATION") or environ.get("VERTEX_LOCATION") or "us-central1"
|
||||
if not api_key or not project:
|
||||
return ()
|
||||
upstream_base: Final = environ.get("VERTEX_AI_API_BASE") or f"https://{location}-aiplatform.googleapis.com"
|
||||
base_url: Final = environ.get("VERTEX_AI_API_BASE") or f"https://{location}-aiplatform.googleapis.com"
|
||||
invocation: Final = invoke_with_api_key(client, api_key)
|
||||
return (
|
||||
OcrRecordingTarget(
|
||||
name="vertex-mistral",
|
||||
provider_spec=ProviderSpec(upstream_base=upstream_base.rstrip("/")),
|
||||
upstream=UpstreamEndpoint(base_url=base_url.rstrip("/")),
|
||||
strategy=cast(
|
||||
SearchStrategy[OcrSdkInputBase],
|
||||
vertex_mistral_input_strategy(project, location, inline_image_data_uri),
|
||||
),
|
||||
invocation=invocation,
|
||||
required_inputs=vertex_mistral_provider_rejected_inputs(project, location, inline_image_data_uri),
|
||||
),
|
||||
OcrRecordingTarget(
|
||||
name="vertex-deepseek",
|
||||
provider_spec=ProviderSpec(upstream_base=upstream_base.rstrip("/")),
|
||||
upstream=UpstreamEndpoint(base_url=base_url.rstrip("/")),
|
||||
strategy=cast(
|
||||
SearchStrategy[OcrSdkInputBase],
|
||||
vertex_deepseek_input_strategy(project, location, inline_image_data_uri),
|
||||
),
|
||||
invocation=invocation,
|
||||
required_inputs=vertex_deepseek_provider_rejected_inputs(project, location, inline_image_data_uri),
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ from tests.test_litellm.ocr.fixtures.base import (
|
|||
OcrSdkInputBase,
|
||||
)
|
||||
from tests.test_litellm.ocr.fixtures.mistral import MISTRAL_MODELS, MistralOcrSdkInput, mistral_input_strategy
|
||||
from tests.test_litellm.ocr.fixtures.models import OcrParityCase
|
||||
from tests.test_litellm.ocr.fixtures.models import OcrParityCase, OcrSdkInput
|
||||
from tests.test_litellm.ocr.fixtures.reducto import (
|
||||
REDUCTO_LEGACY_MODELS,
|
||||
REDUCTO_V3_MODELS,
|
||||
|
|
@ -73,7 +73,7 @@ from tests.test_litellm.ocr.fixtures.vertex import (
|
|||
)
|
||||
|
||||
COMMON_FIELDS: Final = frozenset(
|
||||
{"boundary", "model", "document", "custom_llm_provider", "vertex_project", "vertex_location"}
|
||||
{"contract", "model", "document", "custom_llm_provider", "vertex_project", "vertex_location"}
|
||||
)
|
||||
SUPPORTED_OCR_PROVIDERS: Final = frozenset({"mistral", "azure_ai", "reducto", "vertex_ai"})
|
||||
ACTIVE_OCR_MODELS: Final = frozenset(
|
||||
|
|
@ -342,9 +342,23 @@ def test_fixture_fields_match_provider_config(
|
|||
),
|
||||
),
|
||||
)
|
||||
def test_provider_boundary_is_explicit_but_not_forwarded(sdk_input: OcrSdkInputBase) -> None:
|
||||
assert sdk_input.canonical_input()["boundary"] == sdk_input.boundary
|
||||
assert "boundary" not in sdk_input.as_sdk_kwargs()
|
||||
def test_provider_contract_is_explicit_but_not_forwarded(sdk_input: OcrSdkInput) -> None:
|
||||
assert sdk_input.canonical_input()["contract"] == sdk_input.contract
|
||||
assert "contract" not in sdk_input.as_sdk_kwargs()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("legacy_key", ("boundary", None))
|
||||
def test_ocr_parity_case_migrates_legacy_contract_metadata(legacy_key: str | None) -> None:
|
||||
litellm_input: Final[dict[str, object]] = {
|
||||
"model": "mistral/mistral-ocr-latest",
|
||||
"document": {"type": "document_url", "document_url": "https://example.com/document.pdf"},
|
||||
}
|
||||
if legacy_key is not None:
|
||||
litellm_input[legacy_key] = "mistral"
|
||||
|
||||
fixture: Final = OcrParityCase.model_validate({"litellm_input": litellm_input, "provider_responses": ()})
|
||||
|
||||
assert fixture.litellm_input.contract == "mistral"
|
||||
|
||||
|
||||
def test_mistral_input_preserves_omission_and_explicit_boolean_values() -> None:
|
||||
|
|
@ -478,6 +492,7 @@ def test_vertex_deepseek_request_maps_both_document_types_to_image_content(
|
|||
data: Final = cast(dict[str, object], request.data)
|
||||
messages: Final = cast(list[dict[str, object]], data["messages"])
|
||||
content: Final = cast(list[dict[str, object]], messages[0]["content"])
|
||||
assert data["model"] == "deepseek-ai/deepseek-ocr-maas"
|
||||
assert content == [{"type": "image_url", "image_url": document[source_key]}]
|
||||
|
||||
|
||||
|
|
@ -1139,7 +1154,7 @@ def test_azure_document_intelligence_strategy_only_generates_litellm_inputs(
|
|||
) -> None:
|
||||
assert sdk_input.req_format == "litellm"
|
||||
assert sdk_input.model in AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS
|
||||
assert "boundary" not in sdk_input.as_sdk_kwargs()
|
||||
assert "contract" not in sdk_input.as_sdk_kwargs()
|
||||
optional_fields: Final = frozenset(sdk_input.model_fields_set) - {"model", "document"}
|
||||
assert optional_fields in {
|
||||
frozenset[str](),
|
||||
|
|
@ -1361,7 +1376,7 @@ def test_vertex_deepseek_strategy_only_generates_litellm_inputs(
|
|||
sdk_input: VertexDeepSeekOcrSdkInput,
|
||||
) -> None:
|
||||
assert sdk_input.vertex_project == "project-1"
|
||||
assert "boundary" not in sdk_input.as_sdk_kwargs()
|
||||
assert "contract" not in sdk_input.as_sdk_kwargs()
|
||||
assert _document_transport(sdk_input.document) == ("image_url", "data")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from typing import Final, Protocol, cast
|
|||
|
||||
import pytest
|
||||
|
||||
from tests.route_parity.fixtures.store import parametrize_recorded_fixtures
|
||||
from tests.route_parity.fixtures.pytest_support import parametrize_recorded_fixtures
|
||||
from tests.test_litellm.ocr.conftest import ocr_fixture_id, ocr_fixture_marks
|
||||
from tests.test_litellm.ocr.fixtures.models import OcrParityCase
|
||||
|
||||
|
|
@ -45,7 +45,7 @@ def test_recorded_fixture_parametrization_applies_case_specific_marks() -> None:
|
|||
reducto_parameters: Final = tuple(
|
||||
parameter
|
||||
for parameter in parameters
|
||||
if parameter.values[0].litellm_input.boundary in {"reducto_v3", "reducto_legacy"}
|
||||
if parameter.values[0].litellm_input.contract in {"reducto_v3", "reducto_legacy"}
|
||||
)
|
||||
supported_parameters: Final = tuple(parameter for parameter in parameters if parameter not in reducto_parameters)
|
||||
|
||||
|
|
|
|||
|
|
@ -10,12 +10,14 @@ import pytest
|
|||
from hypothesis import find, settings
|
||||
from hypothesis.strategies import SearchStrategy
|
||||
|
||||
from tests.route_parity.fixtures.cli import parse_recording_args
|
||||
from tests.route_parity.fixtures.inputs import generate_case_inputs
|
||||
from tests.route_parity.fixtures.media import structured_pdf_data_uri
|
||||
from tests.route_parity.fixtures.pipeline import parse_recording_args
|
||||
from tests.test_litellm.ocr.fixtures.azure import (
|
||||
AZURE_DOCUMENT_INTELLIGENCE_PROVIDER_REJECTED_INPUTS,
|
||||
AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS,
|
||||
AZURE_MISTRAL_MODELS,
|
||||
AZURE_MISTRAL_PROVIDER_REJECTED_INPUTS,
|
||||
)
|
||||
from tests.test_litellm.ocr.fixtures.base import OcrSdkInputBase
|
||||
from tests.test_litellm.ocr.fixtures.common import OcrFixtureClient, OcrRecordingTarget
|
||||
|
|
@ -26,8 +28,18 @@ from tests.test_litellm.ocr.fixtures.record import (
|
|||
from tests.test_litellm.ocr.fixtures.record import (
|
||||
require_targets,
|
||||
)
|
||||
from tests.test_litellm.ocr.fixtures.reducto import REDUCTO_LEGACY_MODELS, REDUCTO_V3_MODELS
|
||||
from tests.test_litellm.ocr.fixtures.vertex import VERTEX_DEEPSEEK_MODELS, VERTEX_MISTRAL_MODELS
|
||||
from tests.test_litellm.ocr.fixtures.reducto import (
|
||||
REDUCTO_LEGACY_MODELS,
|
||||
REDUCTO_LEGACY_PROVIDER_REJECTED_INPUTS,
|
||||
REDUCTO_V3_MODELS,
|
||||
REDUCTO_V3_PROVIDER_REJECTED_INPUTS,
|
||||
)
|
||||
from tests.test_litellm.ocr.fixtures.vertex import (
|
||||
VERTEX_DEEPSEEK_MODELS,
|
||||
VERTEX_MISTRAL_MODELS,
|
||||
vertex_deepseek_provider_rejected_inputs,
|
||||
vertex_mistral_provider_rejected_inputs,
|
||||
)
|
||||
|
||||
|
||||
class _UnusedOcrClient:
|
||||
|
|
@ -186,7 +198,7 @@ def test_mistral_target_uses_canonical_model_and_normalized_base(
|
|||
assert len(targets) == 1
|
||||
target: Final = targets[0]
|
||||
assert target.name == "mistral-ocr"
|
||||
assert target.provider_spec.upstream_base == expected
|
||||
assert target.upstream.base_url == expected
|
||||
assert "mistral-secret" not in repr(target)
|
||||
case_inputs: Final = generate_case_inputs(target.strategy, examples=1)
|
||||
assert len(case_inputs) == 1
|
||||
|
|
@ -316,11 +328,21 @@ def test_only_intentional_provider_failures_are_fixed_inputs() -> None:
|
|||
_UNUSED_OCR_CLIENT,
|
||||
)
|
||||
|
||||
mistral: Final = next(target for target in targets if target.name == "mistral-ocr")
|
||||
assert mistral.required_inputs == MISTRAL_PROVIDER_REJECTED_INPUTS
|
||||
generated: Final = generate_case_inputs(mistral.strategy, examples=20)
|
||||
assert all(case_input not in mistral.required_inputs for case_input in generated)
|
||||
assert all(target.required_inputs == () for target in targets if target is not mistral)
|
||||
expected: Final[dict[str, tuple[OcrSdkInputBase, ...]]] = {
|
||||
"mistral-ocr": MISTRAL_PROVIDER_REJECTED_INPUTS,
|
||||
"azure-mistral": AZURE_MISTRAL_PROVIDER_REJECTED_INPUTS,
|
||||
"azure-document-intelligence": AZURE_DOCUMENT_INTELLIGENCE_PROVIDER_REJECTED_INPUTS,
|
||||
"vertex-mistral": vertex_mistral_provider_rejected_inputs("project-1", "us-central1", _INLINE_IMAGE_DATA_URI),
|
||||
"vertex-deepseek": vertex_deepseek_provider_rejected_inputs("project-1", "us-central1", _INLINE_IMAGE_DATA_URI),
|
||||
"reducto-v3": REDUCTO_V3_PROVIDER_REJECTED_INPUTS,
|
||||
"reducto-legacy": REDUCTO_LEGACY_PROVIDER_REJECTED_INPUTS,
|
||||
}
|
||||
|
||||
assert {target.name for target in targets} == expected.keys()
|
||||
for target in targets:
|
||||
assert target.required_inputs == expected[target.name]
|
||||
generated: Final = generate_case_inputs(target.strategy, examples=20)
|
||||
assert all(case_input not in target.required_inputs for case_input in generated)
|
||||
|
||||
|
||||
def test_mistral_adapters_preserve_omitted_optional_params() -> None:
|
||||
|
|
|
|||
|
|
@ -368,6 +368,18 @@ def test_load_rust_ocr_uses_compiled_extension(monkeypatch):
|
|||
def test_timeout_to_seconds_handles_float_timeout_and_none():
|
||||
assert rust_bridge._timeout_to_seconds(12.5) == 12.5
|
||||
assert rust_bridge._timeout_to_seconds(None) is None
|
||||
|
||||
|
||||
def test_ocr_provider_error_uses_resolved_request_url():
|
||||
error = rust_bridge._OcrProviderError(
|
||||
429,
|
||||
"rate limited",
|
||||
"https://example.azure.com/documentintelligence/documentModels/read:analyze",
|
||||
)
|
||||
|
||||
assert str(error.response.request.url) == (
|
||||
"https://example.azure.com/documentintelligence/documentModels/read:analyze"
|
||||
)
|
||||
assert rust_bridge._timeout_to_seconds(httpx.Timeout(30.0, read=42.0)) == 42.0
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
from collections.abc import Awaitable, Callable, Coroutine, Generator
|
||||
|
|
@ -32,17 +31,20 @@ from tests.route_parity.models import (
|
|||
)
|
||||
from tests.route_parity.replay import replay_server
|
||||
from tests.route_parity.runner import (
|
||||
PythonScriptRunner,
|
||||
PythonScriptWorker,
|
||||
ExecutionVariant,
|
||||
SubprocessRunner,
|
||||
SubprocessWorker,
|
||||
execution_worker_pair,
|
||||
parity_worker_main,
|
||||
run_execution,
|
||||
)
|
||||
from tests.test_litellm.ocr.fixtures.config import configured_fixture_directory
|
||||
from tests.test_litellm.ocr.fixtures.models import OcrParityCase, OcrSdkInput
|
||||
|
||||
API_KEY: Final = "test-key"
|
||||
PYTHON_HTTP_SENTINEL: Final = "python-ocr-parity-fallback"
|
||||
FIXTURE_DIR_ENV: Final = "LITELLM_OCR_FIXTURE_DIR"
|
||||
PYTHON_VARIANT: Final = ExecutionVariant(name="Python", environment=(("LITELLM_USE_RUST_OCR", "0"),))
|
||||
RUST_VARIANT: Final = ExecutionVariant(name="Rust", environment=(("LITELLM_USE_RUST_OCR", "1"),))
|
||||
|
||||
|
||||
class SDKRoute(str, Enum):
|
||||
|
|
@ -120,6 +122,15 @@ INVALID_OCR_CASES: Final = (
|
|||
expected_message="Document URL is required",
|
||||
expected_rust_calls=1,
|
||||
),
|
||||
InvalidOcrCase(
|
||||
name="missing_image_url",
|
||||
model="azure_ai/doc-intelligence/prebuilt-read",
|
||||
document={"type": "image_url"},
|
||||
expected_exception_type="litellm.exceptions.APIConnectionError",
|
||||
expected_status_code=500,
|
||||
expected_message="Document URL is required",
|
||||
expected_rust_calls=1,
|
||||
),
|
||||
InvalidOcrCase(
|
||||
name="invalid_request_format",
|
||||
model="mistral/mistral-ocr-latest",
|
||||
|
|
@ -304,22 +315,19 @@ def _native_spies() -> tuple[_RustOcrSpy, _RustAocrSpy]:
|
|||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def sdk_workers() -> Generator[tuple[PythonScriptWorker, PythonScriptWorker]]:
|
||||
runner: Final = PythonScriptRunner(
|
||||
def sdk_workers() -> Generator[tuple[SubprocessWorker, SubprocessWorker]]:
|
||||
runner: Final = SubprocessRunner(
|
||||
entrypoint=Path(__file__),
|
||||
rust_env_var="LITELLM_USE_RUST_OCR",
|
||||
python_user_agent=PYTHON_HTTP_SENTINEL,
|
||||
baseline_user_agent=PYTHON_HTTP_SENTINEL,
|
||||
route_label="OCR",
|
||||
)
|
||||
with execution_worker_pair(runner) as workers:
|
||||
with execution_worker_pair(runner, PYTHON_VARIANT, RUST_VARIANT) as workers:
|
||||
yield workers
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def startup_ocr_fixture() -> OcrParityCase:
|
||||
default_directory: Final = Path(__file__).with_name("fixtures") / "data"
|
||||
configured: Final = os.environ.get(FIXTURE_DIR_ENV)
|
||||
directory: Final = Path(configured).expanduser() if configured is not None else default_directory
|
||||
directory: Final = configured_fixture_directory()
|
||||
fixtures: Final = recorded_fixtures(directory, OcrParityCase)
|
||||
if not fixtures:
|
||||
pytest.skip(f"no recorded fixtures in {directory}")
|
||||
|
|
@ -404,7 +412,7 @@ def test_invalid_ocr_sdk_parity(case: InvalidOcrCase, route: SDKRoute) -> None:
|
|||
def test_ocr_subprocess_startup_smoke(
|
||||
startup_ocr_fixture: OcrParityCase,
|
||||
tmp_path: Path,
|
||||
sdk_workers: tuple[PythonScriptWorker, PythonScriptWorker],
|
||||
sdk_workers: tuple[SubprocessWorker, SubprocessWorker],
|
||||
) -> None:
|
||||
case_file: Final = tmp_path / "ocr-startup-smoke.json"
|
||||
case_file.write_text(startup_ocr_fixture.model_dump_json(indent=2, exclude_unset=True), encoding="utf-8")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue